You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tephra.apache.org by po...@apache.org on 2016/05/11 20:15:01 UTC
[01/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Repository: incubator-tephra
Updated Branches:
refs/heads/master b9927becd -> 82e70f5ca
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessorTest.java
new file mode 100644
index 0000000..c2187ed
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessorTest.java
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10.coprocessor;
+
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MockRegionServerServices;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.HDFSTransactionStateStorage;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.snapshot.DefaultSnapshotCodec;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.util.TxUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
+ */
+public class TransactionProcessorTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
+
+ // 8 versions, 1 hour apart, latest is current ts.
+ private static final long[] V;
+
+ static {
+ long now = System.currentTimeMillis();
+ V = new long[9];
+ for (int i = 0; i < V.length; i++) {
+ V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
+ }
+ }
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+ private static MiniDFSCluster dfsCluster;
+ private static Configuration conf;
+ private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
+ private static TransactionVisibilityState txVisibilityState;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration hConf = new Configuration();
+ String rootDir = tmpFolder.newFolder().getAbsolutePath();
+ hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
+ hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
+
+ dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
+ dfsCluster.waitActive();
+ conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
+
+ conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
+ conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ String localTestDir = tmpFolder.newFolder().getAbsolutePath();
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+
+ // write an initial transaction snapshot
+ TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(
+ System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
+ // this will set visibility upper bound to V[6]
+ Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
+ TransactionType.SHORT))),
+ new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
+ txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
+ txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
+ txSnapshot.getInProgress());
+ HDFSTransactionStateStorage tmpStorage =
+ new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
+ tmpStorage.startAndWait();
+ tmpStorage.writeSnapshot(txSnapshot);
+ tmpStorage.stopAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ @Test
+ public void testDataJanitorRegionScanner() throws Exception {
+ String tableName = "TestRegionScanner";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ for (int i = 1; i <= 8; i++) {
+ for (int k = 1; k <= i; k++) {
+ Put p = new Put(Bytes.toBytes(i));
+ p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
+ region.put(p);
+ }
+ }
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should only return the valid rows
+ // do not use a filter here to test that cleanup works on flush
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+
+ // first returned value should be "4" with version "4"
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 4, new long[]{V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 5, new long[] {V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 6, new long[]{V[6], V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 7, new long[]{V[6], V[4]});
+
+ results.clear();
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteFiltering() throws Exception {
+ String tableName = "TestDeleteFiltering";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ byte[] row = Bytes.toBytes(1);
+ for (int i = 4; i < V.length; i++) {
+ Put p = new Put(row);
+ p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
+ region.put(p);
+ }
+
+ // delete from the third entry back
+ // take that cell's timestamp + 1 to simulate a delete in a new tx
+ long deleteTs = V[5] + 1;
+ Delete d = new Delete(row, deleteTs);
+ LOG.info("Issuing delete at timestamp " + deleteTs);
+ // row deletes are not yet supported (TransactionAwareHTable normally handles this)
+ d.deleteColumns(familyBytes, columnBytes);
+ region.delete(d);
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, we should drop the deleted version, but not the others
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should return row with versions at: V[8], V[6].
+ // V[7] is invalid and V[5] and prior are deleted.
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+ // should be only one row
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 1,
+ new long[]{V[8], V[6], deleteTs},
+ new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteMarkerCleanup() throws Exception {
+ String tableName = "TestDeleteMarkerCleanup";
+ byte[] familyBytes = Bytes.toBytes("f");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+
+ // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
+ long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
+ // deletes are performed after the writes, but still before the visibility upper bound
+ long deleteTs = writeTs + 1;
+ // write separate columns to confirm that delete markers survive across flushes
+ byte[] row = Bytes.toBytes(100);
+ Put p = new Put(row);
+
+ LOG.info("Writing columns at timestamp " + writeTs);
+ for (int i = 0; i < 5; i++) {
+ byte[] iBytes = Bytes.toBytes(i);
+ p.add(familyBytes, iBytes, writeTs, iBytes);
+ }
+ region.put(p);
+ // read all back
+ Scan scan = new Scan(row);
+ RegionScanner regionScanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ for (int i = 0; i < 5; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a flush to clear the memstore
+ LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // delete the odd entries
+ for (int i = 0; i < 5; i++) {
+ if (i % 2 == 1) {
+ // deletes are performed as puts with empty values
+ Put deletePut = new Put(row);
+ deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
+ region.put(deletePut);
+ }
+ }
+
+ // read all back
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ LOG.info("Got cell " + cell);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force another flush on the delete markers
+ // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
+ LOG.info("After delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a major compaction
+ LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
+ region.compactStores(true);
+
+ // perform a raw scan (no filter) to confirm that the delete markers are now gone
+ scan = new Scan(row);
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+ } finally {
+ region.close();
+ }
+ }
+
+ /**
+ * Test that we correctly preserve the timestamp set for column family delete markers. This is not
+ * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
+ * that we make it easy to interoperate with other systems.
+ */
+ @Test
+ public void testFamilyDeleteTimestamp() throws Exception {
+ String tableName = "TestFamilyDeleteTimestamp";
+ byte[] family1Bytes = Bytes.toBytes("f1");
+ byte[] columnBytes = Bytes.toBytes("c");
+ byte[] rowBytes = Bytes.toBytes("row");
+ byte[] valBytes = Bytes.toBytes("val");
+ HRegion region = createRegion(tableName, family1Bytes, 0);
+ try {
+ region.initialize();
+
+ long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+ Put p = new Put(rowBytes);
+ p.add(family1Bytes, columnBytes, now - 10, valBytes);
+ region.put(p);
+
+ // issue a family delete with an explicit timestamp
+ Delete delete = new Delete(rowBytes, now);
+ delete.deleteFamily(family1Bytes, now - 5);
+ region.delete(delete);
+
+ // test that the delete marker preserved the timestamp
+ Scan scan = new Scan();
+ scan.setMaxVersions();
+ RegionScanner scanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(2, results.size());
+ // delete marker should appear first
+ Cell cell = results.get(0);
+ assertArrayEquals(new byte[0], cell.getQualifier());
+ assertArrayEquals(new byte[0], cell.getValue());
+ assertEquals(now - 5, cell.getTimestamp());
+ // since this is an unfiltered scan against the region, the original put should be next
+ cell = results.get(1);
+ assertArrayEquals(valBytes, cell.getValue());
+ assertEquals(now - 10, cell.getTimestamp());
+ scanner.close();
+
+
+ // with a filtered scan the original put should disappear
+ scan = new Scan();
+ scan.setMaxVersions();
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ scanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(0, results.size());
+ scanner.close();
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testPreExistingData() throws Exception {
+ String tableName = "TestPreExistingData";
+ byte[] familyBytes = Bytes.toBytes("f");
+ long ttlMillis = TimeUnit.DAYS.toMillis(14);
+ HRegion region = createRegion(tableName, familyBytes, ttlMillis);
+ try {
+ region.initialize();
+
+ // timestamps for pre-existing, non-transactional data
+ long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
+ long older = now - ttlMillis / 2;
+ long newer = now - ttlMillis / 3;
+ // timestamps for transactional data
+ long nowTx = txVisibilityState.getVisibilityUpperBound();
+ long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
+ long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
+
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ ttls.put(familyBytes, ttlMillis);
+
+ List<Cell> cells = new ArrayList<>();
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
+
+ // Write non-transactional and transactional data
+ for (Cell c : cells) {
+ region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
+ }
+
+ Scan rawScan = new Scan();
+ rawScan.setMaxVersions();
+
+ Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
+ Scan txScan = new Scan();
+ txScan.setMaxVersions();
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // read all back with raw scanner
+ scanAndAssert(region, cells, rawScan);
+
+ // read all back with transaction filter
+ scanAndAssert(region, cells, txScan);
+
+ // force a flush to clear the memstore
+ region.flushcache();
+ scanAndAssert(region, cells, txScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ scanAndAssert(region, cells, txScan);
+
+ // Reduce TTL, this should make cells with timestamps older and olderTx expire
+ long newTtl = ttlMillis / 2 - 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // Raw scan should still give all cells
+ scanAndAssert(region, cells, rawScan);
+ // However, tx scan should not return expired cells
+ scanAndAssert(region, select(cells, 1, 3, 5), txScan);
+
+ region.flushcache();
+ scanAndAssert(region, cells, rawScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ // This time raw scan too should not return expired cells, as they would be dropped during major compaction
+ scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
+
+ // Reduce TTL again to 1 ms, this should expire all cells
+ newTtl = 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // force a major compaction to remove expired cells
+ region.compactStores(true);
+ // This time raw scan should not return any cells, as all cells have expired.
+ scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
+ } finally {
+ region.close();
+ }
+ }
+
+ private List<Cell> select(List<Cell> cells, int... indexes) {
+ List<Cell> newCells = new ArrayList<>();
+ for (int i : indexes) {
+ newCells.add(cells.get(i));
+ }
+ return newCells;
+ }
+
+ @SuppressWarnings("StatementWithEmptyBody")
+ private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
+ try (RegionScanner regionScanner = region.getScanner(scan)) {
+ List<Cell> results = Lists.newArrayList();
+ while (regionScanner.next(results)) { }
+ assertEquals(expected, results);
+ }
+ }
+
+ private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
+ region.close();
+ HTableDescriptor htd = region.getTableDesc();
+ HColumnDescriptor cfd = htd.getFamily(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
+ new LocalRegionServerServices(conf, ServerName.valueOf(
+ InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
+ }
+
+ private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
+ HColumnDescriptor cfd = new HColumnDescriptor(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ htd.addFamily(cfd);
+ htd.addCoprocessor(TransactionProcessor.class.getName());
+ Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
+ FileSystem fs = FileSystem.get(conf);
+ assertTrue(fs.mkdirs(tablePath));
+ WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
+ WAL hLog = walFactory.getWAL(new byte[]{1});
+ HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
+ HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
+ return new HRegion(regionFS, hLog, conf, htd,
+ new LocalRegionServerServices(conf, ServerName.valueOf(
+ InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
+ byte[][] values = new byte[versions.length][];
+ for (int i = 0; i < versions.length; i++) {
+ values[i] = Bytes.toBytes(versions[i]);
+ }
+ assertKeyValueMatches(results, index, versions, values);
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
+ assertEquals(versions.length, results.size());
+ assertEquals(values.length, results.size());
+ for (int i = 0; i < versions.length; i++) {
+ Cell kv = results.get(i);
+ assertArrayEquals(Bytes.toBytes(index), kv.getRow());
+ assertEquals(versions[i], kv.getTimestamp());
+ assertArrayEquals(values[i], kv.getValue());
+ }
+ }
+
+ @Test
+ public void testTransactionStateCache() throws Exception {
+ TransactionStateCache cache = new TransactionStateCache();
+ cache.setConf(conf);
+ cache.startAndWait();
+ // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
+ TransactionVisibilityState cachedSnapshot = cache.getLatestState();
+ assertNotNull(cachedSnapshot);
+ assertEquals(invalidSet, cachedSnapshot.getInvalid());
+ cache.stopAndWait();
+ }
+
+ private static class LocalRegionServerServices extends MockRegionServerServices {
+ private final ServerName serverName;
+
+ public LocalRegionServerServices(Configuration conf, ServerName serverName) {
+ super(conf);
+ this.serverName = serverName;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return serverName;
+ }
+ }
+}
[48/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCache.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCache.java b/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCache.java
deleted file mode 100644
index 2745c76..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCache.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.coprocessor;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.HDFSTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.util.ConfigurationFactory;
-import com.google.common.util.concurrent.AbstractIdleService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Periodically refreshes transaction state from the latest stored snapshot. This is implemented as a singleton
- * to allow a single cache to be shared by all regions on a regionserver.
- */
-public class TransactionStateCache extends AbstractIdleService implements Configurable {
- private static final Log LOG = LogFactory.getLog(TransactionStateCache.class);
-
- // how frequently we should wake to check for changes (in seconds)
- private static final long CHECK_FREQUENCY = 15;
-
- private Configuration hConf;
-
- private TransactionStateStorage storage;
- private volatile TransactionVisibilityState latestState;
-
- private Thread refreshService;
- private long lastRefresh;
- // snapshot refresh frequency in milliseconds
- private long snapshotRefreshFrequency;
- private boolean initialized;
-
- public TransactionStateCache() {
- }
-
- @Override
- public Configuration getConf() {
- return hConf;
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.hConf = conf;
- }
-
- @Override
- protected void startUp() throws Exception {
- refreshState();
- startRefreshService();
- }
-
- @Override
- protected void shutDown() throws Exception {
- this.refreshService.interrupt();
- this.storage.stop();
- }
-
- /**
- * Try to initialize the Configuration and TransactionStateStorage instances. Obtaining the Configuration may
- * fail until ReactorServiceMain has been started.
- */
- private void tryInit() {
- try {
- Configuration conf = getSnapshotConfiguration();
- if (conf != null) {
- // Since this is only used for background loading of transaction snapshots, we use the no-op metrics collector,
- // as there are no relevant metrics to report
- this.storage = new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf),
- new TxMetricsCollector());
- this.storage.startAndWait();
- this.snapshotRefreshFrequency = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
- TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL) * 1000;
- this.initialized = true;
- } else {
- LOG.info("Could not load configuration");
- }
- } catch (Exception e) {
- LOG.info("Failed to initialize TransactionStateCache due to: " + e.getMessage());
- }
- }
-
- protected Configuration getSnapshotConfiguration() throws IOException {
- Configuration conf = new ConfigurationFactory().get(hConf);
- conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- return conf;
- }
-
- private void reset() {
- this.storage.stop();
- this.lastRefresh = 0;
- this.initialized = false;
- }
-
- private void startRefreshService() {
- this.refreshService = new Thread("tx-state-refresh") {
- @Override
- public void run() {
- while (!isInterrupted()) {
- if (latestState == null || System.currentTimeMillis() > (lastRefresh + snapshotRefreshFrequency)) {
- try {
- refreshState();
- } catch (IOException ioe) {
- LOG.info("Error refreshing transaction state cache: " + ioe.getMessage());
- }
- }
- try {
- TimeUnit.SECONDS.sleep(CHECK_FREQUENCY);
- } catch (InterruptedException ie) {
- // reset status
- interrupt();
- break;
- }
- }
- LOG.info("Exiting thread " + getName());
- }
- };
- this.refreshService.setDaemon(true);
- this.refreshService.start();
- }
-
- private void refreshState() throws IOException {
- if (!initialized) {
- tryInit();
- }
-
- // only continue if initialization was successful
- if (initialized) {
- long now = System.currentTimeMillis();
- TransactionVisibilityState currentState = storage.getLatestTransactionVisibilityState();
- if (currentState != null) {
- if (currentState.getTimestamp() < (now - 2 * snapshotRefreshFrequency)) {
- LOG.info("Current snapshot is old, will force a refresh on next run.");
- reset();
- } else {
- latestState = currentState;
- LOG.info("Transaction state reloaded with snapshot from " + latestState.getTimestamp());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Latest transaction snapshot: " + latestState.toString());
- }
- lastRefresh = now;
- }
- } else {
- LOG.info("No transaction state found.");
- }
- }
- }
-
- public TransactionVisibilityState getLatestState() {
- return latestState;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCacheSupplier.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCacheSupplier.java b/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCacheSupplier.java
deleted file mode 100644
index 9f4a778..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/coprocessor/TransactionStateCacheSupplier.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.coprocessor;
-
-import com.google.common.base.Supplier;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Supplies instances of {@link TransactionStateCache} implementations.
- */
-public class TransactionStateCacheSupplier implements Supplier<TransactionStateCache> {
- protected static volatile TransactionStateCache instance;
- protected static Object lock = new Object();
-
- protected final Configuration conf;
-
- public TransactionStateCacheSupplier(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * Returns a singleton instance of the transaction state cache, performing lazy initialization if necessary.
- * @return A shared instance of the transaction state cache.
- */
- @Override
- public TransactionStateCache get() {
- if (instance == null) {
- synchronized (lock) {
- if (instance == null) {
- instance = new TransactionStateCache();
- instance.setConf(conf);
- instance.start();
- }
- }
- }
- return instance;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/coprocessor/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/coprocessor/package-info.java b/tephra-core/src/main/java/co/cask/tephra/coprocessor/package-info.java
deleted file mode 100644
index 5fa566c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/coprocessor/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains HBase coprocessor implementations for the transaction system.
- */
-package co.cask.tephra.coprocessor;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/AbstractClientProvider.java b/tephra-core/src/main/java/co/cask/tephra/distributed/AbstractClientProvider.java
deleted file mode 100644
index d0a3603..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/AbstractClientProvider.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.TxConstants;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.TException;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.twill.discovery.Discoverable;
-import org.apache.twill.discovery.DiscoveryServiceClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * An abstract tx client provider that implements common functionality.
- */
-public abstract class AbstractClientProvider implements ThriftClientProvider {
-
- private static final Logger LOG = LoggerFactory.getLogger(AbstractClientProvider.class);
-
- // Discovery service. If null, no service discovery.
- private final DiscoveryServiceClient discoveryServiceClient;
- protected final AtomicBoolean initialized = new AtomicBoolean(false);
-
- // the configuration
- final Configuration configuration;
-
- // the endpoint strategy for service discovery.
- EndpointStrategy endpointStrategy;
-
- protected AbstractClientProvider(Configuration configuration, DiscoveryServiceClient discoveryServiceClient) {
- this.configuration = configuration;
- this.discoveryServiceClient = discoveryServiceClient;
- }
-
- public void initialize() throws TException {
- if (initialized.compareAndSet(false, true)) {
- this.initDiscovery();
- }
- }
-
- /**
- * Initialize the service discovery client, we will reuse that
- * every time we need to create a new client.
- */
- private void initDiscovery() {
- if (discoveryServiceClient == null) {
- LOG.info("No DiscoveryServiceClient provided. Skipping service discovery.");
- return;
- }
-
- endpointStrategy = new TimeLimitEndpointStrategy(
- new RandomEndpointStrategy(
- discoveryServiceClient.discover(
- configuration.get(TxConstants.Service.CFG_DATA_TX_DISCOVERY_SERVICE_NAME,
- TxConstants.Service.DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME))),
- 2, TimeUnit.SECONDS);
- }
-
- protected TransactionServiceThriftClient newClient() throws TException {
- return newClient(-1);
- }
-
- protected TransactionServiceThriftClient newClient(int timeout) throws TException {
- initialize();
- String address;
- int port;
-
- if (endpointStrategy == null) {
- // if there is no discovery service, try to read host and port directly
- // from the configuration
- LOG.info("Reading address and port from configuration.");
- address = configuration.get(TxConstants.Service.CFG_DATA_TX_BIND_ADDRESS,
- TxConstants.Service.DEFAULT_DATA_TX_BIND_ADDRESS);
- port = configuration.getInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT,
- TxConstants.Service.DEFAULT_DATA_TX_BIND_PORT);
- LOG.info("Service assumed at " + address + ":" + port);
- } else {
- Discoverable endpoint = endpointStrategy.pick();
- if (endpoint == null) {
- LOG.error("Unable to discover tx service.");
- throw new TException("Unable to discover tx service.");
- }
- address = endpoint.getSocketAddress().getHostName();
- port = endpoint.getSocketAddress().getPort();
- LOG.info("Service discovered at " + address + ":" + port);
- }
-
- // now we have an address and port, try to connect a client
- if (timeout < 0) {
- timeout = configuration.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_TIMEOUT,
- TxConstants.Service.DEFAULT_DATA_TX_CLIENT_TIMEOUT_MS);
- }
- LOG.info("Attempting to connect to tx service at " +
- address + ":" + port + " with timeout " + timeout + " ms.");
- // thrift transport layer
- TTransport transport =
- new TFramedTransport(new TSocket(address, port, timeout));
- try {
- transport.open();
- } catch (TTransportException e) {
- LOG.error("Unable to connect to tx service: " + e.getMessage());
- throw e;
- }
- // and create a thrift client
- TransactionServiceThriftClient newClient = new TransactionServiceThriftClient(transport);
-
- LOG.info("Connected to tx service at " +
- address + ":" + port);
- return newClient;
- }
-
- /**
- * This class helps picking up an endpoint from a list of Discoverable.
- */
- public interface EndpointStrategy {
-
- /**
- * Picks a {@link Discoverable} using its strategy.
- * @return A {@link Discoverable} based on the stragegy or {@code null} if no endpoint can be found.
- */
- Discoverable pick();
- }
-
- /**
- * An {@link EndpointStrategy} that make sure it picks an endpoint within the given
- * timeout limit.
- */
- public final class TimeLimitEndpointStrategy implements EndpointStrategy {
-
- private final EndpointStrategy delegate;
- private final long timeout;
- private final TimeUnit timeoutUnit;
-
- public TimeLimitEndpointStrategy(EndpointStrategy delegate, long timeout, TimeUnit timeoutUnit) {
- this.delegate = delegate;
- this.timeout = timeout;
- this.timeoutUnit = timeoutUnit;
- }
-
- @Override
- public Discoverable pick() {
- Discoverable pick = delegate.pick();
- try {
- long count = 0;
- while (pick == null && count++ < timeout) {
- timeoutUnit.sleep(1);
- pick = delegate.pick();
- }
- } catch (InterruptedException e) {
- // Simply propagate the interrupt.
- Thread.currentThread().interrupt();
- }
- return pick;
- }
- }
-
- /**
- * Randomly picks endpoint from the list of available endpoints.
- */
- public final class RandomEndpointStrategy implements EndpointStrategy {
-
- private final Iterable<Discoverable> endpoints;
-
- /**
- * Constructs a random endpoint strategy.
- * @param endpoints Endpoints for the strategy to use. Note that this strategy will
- * invoke {@link Iterable#iterator()} and traverse through it on
- * every call to the {@link #pick()} method. One could leverage this
- * behavior with the live {@link Iterable} as provided by
- * {@link org.apache.twill.discovery.DiscoveryServiceClient#discover(String)} method.
- */
- public RandomEndpointStrategy(Iterable<Discoverable> endpoints) {
- this.endpoints = endpoints;
- }
-
- @Override
- public Discoverable pick() {
- // Reservoir sampling
- Discoverable result = null;
- Iterator<Discoverable> itor = endpoints.iterator();
- Random random = new Random();
- int count = 0;
- while (itor.hasNext()) {
- Discoverable next = itor.next();
- if (random.nextInt(++count) == 0) {
- result = next;
- }
- }
- return result;
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/CloseableThriftClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/CloseableThriftClient.java b/tephra-core/src/main/java/co/cask/tephra/distributed/CloseableThriftClient.java
deleted file mode 100644
index ba6da24..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/CloseableThriftClient.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-/**
- * An {@link AutoCloseable} to automatically return the thrift client to the ThriftClientProvider.
- */
-public class CloseableThriftClient implements AutoCloseable {
-
- private final ThriftClientProvider provider;
- private final TransactionServiceThriftClient thriftClient;
-
- public CloseableThriftClient(ThriftClientProvider provider, TransactionServiceThriftClient thriftClient) {
- this.provider = provider;
- this.thriftClient = thriftClient;
- }
-
- public TransactionServiceThriftClient getThriftClient() {
- return thriftClient;
- }
-
- @Override
- public void close() {
- // in any case, the client must be returned to the pool. The pool is
- // responsible for discarding the client if it is in a bad state.
- provider.returnClient(thriftClient);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/ElasticPool.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/ElasticPool.java b/tephra-core/src/main/java/co/cask/tephra/distributed/ElasticPool.java
deleted file mode 100644
index 843ab90..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/ElasticPool.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * An Elastic Pool is an object pool that can dynamically grow.
- * Normally, an element is obtained by a client and then returned to the pool
- * after use. However, if the element gets into a bad state, the element can
- * be discarded, based upon the recycle() method returning false. This will
- * cause the element to be removed from the pool, and for a subsequent request,
- * a new element can be created on the fly to replace the discarded one.
- *
- * The pool starts with zero (active) elements. Every time a client attempts
- * to obtain an element, an element from the pool is returned if available.
- * Otherwise, if the number of active elements is less than the pool's limit,
- * a new element is created (using abstract method create(), this must be
- * overridden by all implementations), and the number of active elements is
- * increased by one. If the limit is reached, then obtain() blocks until
- * either an element is returned to the pool or, if the obtain method with timeout
- * parameters is used, a timeout occurs.
- *
- * Every time an element is returned to the pool, it is "recycled" to restore its
- * fresh state for the next use or destroyed, depending on its state.
- *
- * @param <T> the type of the elements
- * @param <E> the type of exception thrown by create()
- */
-public abstract class ElasticPool<T, E extends Exception> {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ElasticPool.class);
-
- /**
- * A method to create a new element. Will be called every time the pool
- * of available elements is empty but the limit of active elements is
- * not exceeded.
- * @return a new element
- */
- protected abstract T create() throws E;
-
- /**
- * A method to recycle an existing element when it is returned to the pool.
- * This methods ensures that the element is in a fresh state before it can
- * be reused by the next agent. If the element is not to be returned to the pool,
- * calling code is responsible for destroying it and returning false.
- *
- * @param element the element to recycle
- * @return true to reuse element, false to remove it from the pool
- */
- protected boolean recycle(T element) {
- // by default, simply return true
- return true;
- }
-
- // holds all currently available elements
- private final ConcurrentLinkedQueue<T> elements;
-
- // we keep track of elements via the permits of a semaphore, because there can
- // be elements in a queue, but also elements that are "loaned out" count towards
- // the pool's size limit
- private final Semaphore semaphore;
-
- public ElasticPool(int sizeLimit) {
- elements = new ConcurrentLinkedQueue<>();
- semaphore = new Semaphore(sizeLimit, true);
- }
-
- /**
- * Get a element from the pool. If there is an available element in
- * the pool, it will be returned. Otherwise, if the number of active
- * elements does not exceed the limit, a new element is created with
- * create() and returned. Otherwise, blocks until an element is either
- * released and returned to the pool, or an element is discarded,
- * allowing for the creation of a new element.
- *
- * @return an element
- */
- public T obtain() throws E, InterruptedException {
- semaphore.acquire();
- return getOrCreate();
- }
-
- /**
- * Get a element from the pool. If there is an available element in
- * the pool, it will be returned. Otherwise, if the number of active
- * elements does ot exceed the limit, a new element is created with
- * create() and returned. Otherwise, blocks until an element is either
- * released and returned to the pool, an element is discarded,
- * allowing for the creation of a new element, or a timeout occurs.
- *
- * @param timeout the timeout for trying to obtain an element
- * @param unit the timeout unit for trying to obtain an element
- * @return an element
- * @throws TimeoutException if a client is not able to be obtained within the given timeout
- */
- public T obtain(long timeout, TimeUnit unit) throws E, TimeoutException, InterruptedException {
- if (!semaphore.tryAcquire(1, timeout, unit)) {
- throw new TimeoutException(String.format("Failed to obtain client within %d %s.",
- timeout, unit));
- }
- return getOrCreate();
- }
-
- // gets an element from the queue, or creates one if there is none available in the queue.
- // the semaphore must be acquired before calling this method. The semaphore will be released from within
- // this method if it throws any exception
- private T getOrCreate() throws E {
- try {
- T client = elements.poll();
- // a client was available, all good. otherwise, create one
- if (client != null) {
- return client;
- }
- return create();
- } catch (Exception e) {
- // if an exception is thrown after acquiring the semaphore, release the
- // semaphore before propagating the exception
- semaphore.release();
- throw e;
- }
- }
-
- /**
- * Returns an element to the pool of available elements. The element must
- * have been obtained from this pool through obtain(). The recycle() method
- * is called before the element is available for obtain().
- *
- * @param element the element to be returned
- */
- public void release(T element) {
- try {
- if (recycle(element)) {
- elements.add(element);
- }
- } finally {
- semaphore.release();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/PooledClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/PooledClientProvider.java b/tephra-core/src/main/java/co/cask/tephra/distributed/PooledClientProvider.java
deleted file mode 100644
index 3394811..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/PooledClientProvider.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.TxConstants;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.TException;
-import org.apache.twill.discovery.DiscoveryServiceClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * This is an tx client provider that uses a bounded size pool of connections.
- */
-public class PooledClientProvider extends AbstractClientProvider {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(PooledClientProvider.class);
-
- // we will use this as a pool of tx clients
- class TxClientPool extends ElasticPool<TransactionServiceThriftClient, TException> {
- TxClientPool(int sizeLimit) {
- super(sizeLimit);
- }
-
- @Override
- protected TransactionServiceThriftClient create() throws TException {
- return newClient();
- }
-
- @Override
- protected boolean recycle(TransactionServiceThriftClient client) {
- if (!client.isValid()) {
- client.close();
- return false;
- }
- return true;
- }
- }
-
- // we will use this as a pool of tx clients
- private volatile TxClientPool clients;
-
- // the limit for the number of active clients
- private int maxClients;
- // timeout, for obtaining a client
- private long obtainClientTimeoutMs;
-
- public PooledClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient) {
- super(conf, discoveryServiceClient);
- }
-
- private void initializePool() throws TException {
- // initialize the super class (needed for service discovery)
- super.initialize();
-
- // create a (empty) pool of tx clients
- maxClients = configuration.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT,
- TxConstants.Service.DEFAULT_DATA_TX_CLIENT_COUNT);
- if (maxClients < 1) {
- LOG.warn("Configuration of " + TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT +
- " is invalid: value is " + maxClients + " but must be at least 1. " +
- "Using 1 as a fallback. ");
- maxClients = 1;
- }
-
- obtainClientTimeoutMs =
- configuration.getLong(TxConstants.Service.CFG_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS,
- TxConstants.Service.DEFAULT_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS);
- if (obtainClientTimeoutMs < 0) {
- LOG.warn("Configuration of " + TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT +
- " is invalid: value is " + obtainClientTimeoutMs + " but must be at least 0. " +
- "Using 0 as a fallback. ");
- obtainClientTimeoutMs = 0;
- }
- this.clients = new TxClientPool(maxClients);
- }
-
- @Override
- public CloseableThriftClient getCloseableClient() throws TException, TimeoutException, InterruptedException {
- TransactionServiceThriftClient client = getClientPool().obtain(obtainClientTimeoutMs, TimeUnit.MILLISECONDS);
- return new CloseableThriftClient(this, client);
- }
-
- @Override
- public void returnClient(TransactionServiceThriftClient client) {
- getClientPool().release(client);
- }
-
- @Override
- public String toString() {
- return "Elastic pool of size " + this.maxClients +
- ", with timeout (in milliseconds): " + this.obtainClientTimeoutMs;
- }
-
- private TxClientPool getClientPool() {
- if (clients != null) {
- return clients;
- }
-
- synchronized (this) {
- if (clients == null) {
- try {
- initializePool();
- } catch (TException e) {
- LOG.error("Failed to initialize Tx client provider", e);
- throw Throwables.propagate(e);
- }
- }
- }
- return clients;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/RetryNTimes.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryNTimes.java b/tephra-core/src/main/java/co/cask/tephra/distributed/RetryNTimes.java
deleted file mode 100644
index 22a4015..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryNTimes.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.TxConstants;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A retry strategy that makes N attempts and then gives up. This does
- * not do anything before the re-attempt - extend this class to add a
- * sleep or similar.
- */
-public class RetryNTimes extends RetryStrategy {
-
- int attempts = 0;
- int limit;
-
- /**
- * @param maxAttempts the number of attempts after which to stop
- */
- protected RetryNTimes(int maxAttempts) {
- limit = maxAttempts;
- }
-
- @Override
- boolean failOnce() {
- ++attempts;
- return attempts < limit;
- }
-
- /**
- * A retry strategy provider for this strategy.
- */
- public static class Provider implements RetryStrategyProvider {
-
- int nTimes;
-
- public Provider() {
- this.nTimes = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_ATTEMPTS;
- }
-
- @Override
- public void configure(Configuration config) {
- nTimes = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, nTimes);
- }
-
- @Override
- public RetryStrategy newRetryStrategy() {
- return new RetryNTimes(nTimes);
- }
-
- @Override
- public String toString() {
- return nTimes + " attempts without delay";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategy.java b/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategy.java
deleted file mode 100644
index 80ae779..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategy.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-/**
- * A retry strategy is an abstraction over how the remote tx client shuold retry operations after connection
- * failures.
- */
-public abstract class RetryStrategy {
-
- /**
- * Increments the number of failed attempts.
- * @return whether another attempt should be made
- */
- abstract boolean failOnce();
-
- /**
- * Should be called before re-attempting. This can, for instance
- * inject a sleep time between retries. Default implementation is
- * to do nothing.
- */
- void beforeRetry() {
- // do nothinhg
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategyProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategyProvider.java b/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategyProvider.java
deleted file mode 100644
index 3ea2e94..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryStrategyProvider.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A retry strategy provider is used by the tx client to get a new retry strategy for every call.
- */
-public interface RetryStrategyProvider {
-
- /**
- * Provides a new instance of a retry strategy.
- * @return a retry strategy
- */
- RetryStrategy newRetryStrategy();
-
- /**
- * Configure the strategy.
- * @param config the configuration
- */
- void configure(Configuration config);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/RetryWithBackoff.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryWithBackoff.java b/tephra-core/src/main/java/co/cask/tephra/distributed/RetryWithBackoff.java
deleted file mode 100644
index 3226ba7..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/RetryWithBackoff.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.TxConstants;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A retry strategy that makes N attempts and then gives up. This does
- * not do anything before the re-attempt - extend this class to add a
- * sleep or similar.
- */
-public class RetryWithBackoff extends RetryStrategy {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RetryWithBackoff.class);
-
- int initialSleep; // initial sleep time
- int backoffFactor; // factor by which to increase sleep for each retry
- int maxSleep; // max sleep time. stop retrying when we exceed this
- int sleep; // current sleep time
-
- /**
- * @param initial the initial sleep time (before first retry)
- * @param backoff the backoff factor by which sleep time is multiplied
- * after each retry
- * @param limit the max sleep time. if sleep time reaches this limit, we
- * stop retrying
- */
- protected RetryWithBackoff(int initial, int backoff, int limit) {
- initialSleep = initial;
- backoffFactor = backoff;
- maxSleep = limit;
- sleep = initialSleep;
- }
-
- @Override
- boolean failOnce() {
- return sleep < maxSleep;
- }
-
- @Override
- void beforeRetry() {
- LOG.info("Sleeping " + sleep + " ms before retry.");
- long current = System.currentTimeMillis();
- long end = current + sleep;
- while (current < end) {
- try {
- Thread.sleep(end - current);
- } catch (InterruptedException e) {
- // do nothing
- }
- current = System.currentTimeMillis();
- }
- sleep = sleep * backoffFactor;
- }
-
- /**
- * A provider for this retry strategy.
- */
- public static class Provider implements RetryStrategyProvider {
-
- int initialSleep; // initial sleep time
- int backoffFactor; // factor by which to increase sleep for each retry
- int maxSleep; // max sleep time. stop retrying when we exceed this
-
- public Provider() {
- initialSleep = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_BACKOFF_INITIAL;
- backoffFactor = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_BACKOFF_FACTOR;
- maxSleep = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_BACKOFF_LIMIT;
- }
-
- public void configure(Configuration config) {
- initialSleep = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_BACKOFF_INITIAL, initialSleep);
- backoffFactor = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_BACKOFF_FACTOR, backoffFactor);
- maxSleep = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_BACKOFF_LIMIT, maxSleep);
- }
-
- @Override
- public RetryStrategy newRetryStrategy() {
- return new RetryWithBackoff(initialSleep, backoffFactor, maxSleep);
- }
-
- @Override
- public String toString() {
- return "sleep " + initialSleep + " ms with back off factor " +
- backoffFactor + " and limit " + maxSleep + " ms";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/SingleUseClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/SingleUseClientProvider.java b/tephra-core/src/main/java/co/cask/tephra/distributed/SingleUseClientProvider.java
deleted file mode 100644
index 8673ca4..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/SingleUseClientProvider.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.TException;
-import org.apache.twill.discovery.DiscoveryServiceClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeoutException;
-
-/**
- * An tx client provider that creates a new connection every time.
- */
-public class SingleUseClientProvider extends AbstractClientProvider {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SingleUseClientProvider.class);
-
- public SingleUseClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient, int timeout) {
- super(conf, discoveryServiceClient);
- this.timeout = timeout;
- }
-
- final int timeout;
-
- @Override
- public CloseableThriftClient getCloseableClient() throws TException, TimeoutException, InterruptedException {
- try {
- return new CloseableThriftClient(this, newClient(timeout));
- } catch (TException e) {
- LOG.error("Unable to create new tx client: " + e.getMessage());
- throw e;
- }
- }
-
- @Override
- public void returnClient(TransactionServiceThriftClient client) {
- client.close();
- }
-
- @Override
- public String toString() {
- return "Single-use(timeout = " + timeout + ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/ThreadLocalClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/ThreadLocalClientProvider.java b/tephra-core/src/main/java/co/cask/tephra/distributed/ThreadLocalClientProvider.java
deleted file mode 100644
index eac75bd..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/ThreadLocalClientProvider.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.TException;
-import org.apache.twill.discovery.DiscoveryServiceClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeoutException;
-
-/**
- * An tx client provider that uses thread local to maintain at most one open connection per thread.
- * Note that there can be a connection leak if the threads are recycled.
- */
-public class ThreadLocalClientProvider extends AbstractClientProvider {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ThreadLocalClientProvider.class);
-
- ThreadLocal<TransactionServiceThriftClient> clients = new ThreadLocal<>();
-
- public ThreadLocalClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient) {
- super(conf, discoveryServiceClient);
- }
-
- @Override
- public CloseableThriftClient getCloseableClient() throws TException, TimeoutException, InterruptedException {
- TransactionServiceThriftClient client = this.clients.get();
- if (client == null) {
- try {
- client = this.newClient();
- clients.set(client);
- } catch (TException e) {
- LOG.error("Unable to create new tx client for thread: "
- + e.getMessage());
- throw e;
- }
- }
- return new CloseableThriftClient(this, client);
- }
-
- @Override
- public void returnClient(TransactionServiceThriftClient client) {
- if (!client.isValid()) {
- client.close();
- clients.remove();
- }
- }
-
- @Override
- public String toString() {
- return "Thread-local";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/ThriftClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/ThriftClientProvider.java b/tephra-core/src/main/java/co/cask/tephra/distributed/ThriftClientProvider.java
deleted file mode 100644
index 1e86c08..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/ThriftClientProvider.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import org.apache.thrift.TException;
-
-import java.util.concurrent.TimeoutException;
-
-/**
- * This interface is used to provide thrift tx service clients:
- * there is only one (singleton)
- * tx service per JVM, but many threads may use it concurrently.
- * However, being a thrift client, it is not thread-safe. In
- * order to avoid serializing all tx calls by synchronizing
- * on the tx service client, we employ a pool of clients. But
- * in different scenarios there are different strategies for
- * pooling: If there are many short-lived threads, it is wise
- * to have a shared pool between all threads. But if there are
- * few long-lived threads, it may be better to have thread-local
- * client for each thread.
- *
- * This interface provides an abstraction of the pooling strategy.
- */
-public interface ThriftClientProvider {
-
- /**
- * Initialize the provider. At this point, it should be verified
- * that tx service is up and running and getClient() can
- * create new clients when necessary.
- */
- void initialize() throws TException;
-
- /**
- * Retrieve an AutoCloseable wrapper around tx client for exclusive use by the
- * current thread. The client must be closed (returned) to the provider after use.
- * @return an tx client, connected and fully functional
- */
- CloseableThriftClient getCloseableClient() throws TException,
- TimeoutException, InterruptedException;
-
- /**
- * Release an tx client back to the provider's pool, if the client is valid.
- * If the client becomes disfunctional, for instance, due to a socket
- * exception. The provider must make sure to close the client, and it
- * must remove the client from its arsenal and be prepared to create
- * a new client subsequently.
- *
- * @param client The client to return
- */
- void returnClient(TransactionServiceThriftClient client);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionConverterUtils.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionConverterUtils.java b/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionConverterUtils.java
deleted file mode 100644
index bda39fd..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionConverterUtils.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.distributed.thrift.TTransaction;
-import co.cask.tephra.distributed.thrift.TTransactionType;
-import co.cask.tephra.distributed.thrift.TVisibilityLevel;
-import com.google.common.primitives.Longs;
-
-/**
- * Utility methods to convert to thrift and back.
- */
-public final class TransactionConverterUtils {
- private static final long[] EMPTY_LONG_ARRAY = {};
-
- public static TTransaction wrap(Transaction tx) {
- return new TTransaction(tx.getTransactionId(), tx.getReadPointer(),
- Longs.asList(tx.getInvalids()), Longs.asList(tx.getInProgress()),
- tx.getFirstShortInProgress(), getTTransactionType(tx.getType()),
- tx.getWritePointer(), Longs.asList(tx.getCheckpointWritePointers()),
- getTVisibilityLevel(tx.getVisibilityLevel()));
- }
-
- public static Transaction unwrap(TTransaction thriftTx) {
- return new Transaction(thriftTx.getReadPointer(), thriftTx.getTransactionId(), thriftTx.getWritePointer(),
- thriftTx.getInvalids() == null ? EMPTY_LONG_ARRAY : Longs.toArray(thriftTx.getInvalids()),
- thriftTx.getInProgress() == null ? EMPTY_LONG_ARRAY :
- Longs.toArray(thriftTx.getInProgress()),
- thriftTx.getFirstShort(), getTransactionType(thriftTx.getType()),
- thriftTx.getCheckpointWritePointers() == null ? EMPTY_LONG_ARRAY :
- Longs.toArray(thriftTx.getCheckpointWritePointers()),
- getVisibilityLevel(thriftTx.getVisibilityLevel()));
- }
-
- private static TransactionType getTransactionType(TTransactionType tType) {
- return tType == TTransactionType.SHORT ? TransactionType.SHORT : TransactionType.LONG;
- }
-
- private static TTransactionType getTTransactionType(TransactionType type) {
- return type == TransactionType.SHORT ? TTransactionType.SHORT : TTransactionType.LONG;
- }
-
- private static Transaction.VisibilityLevel getVisibilityLevel(TVisibilityLevel tLevel) {
- // default to SNAPSHOT
- if (tLevel == null) {
- return Transaction.VisibilityLevel.SNAPSHOT;
- }
-
- switch (tLevel) {
- case SNAPSHOT:
- return Transaction.VisibilityLevel.SNAPSHOT;
- case SNAPSHOT_EXCLUDE_CURRENT:
- return Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT;
- case SNAPSHOT_ALL:
- return Transaction.VisibilityLevel.SNAPSHOT_ALL;
- default:
- throw new IllegalArgumentException("Unknown TVisibilityLevel: " + tLevel);
- }
- }
-
- private static TVisibilityLevel getTVisibilityLevel(Transaction.VisibilityLevel level) {
- switch (level) {
- case SNAPSHOT:
- return TVisibilityLevel.SNAPSHOT;
- case SNAPSHOT_EXCLUDE_CURRENT:
- return TVisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT;
- case SNAPSHOT_ALL:
- return TVisibilityLevel.SNAPSHOT_ALL;
- default:
- throw new IllegalArgumentException("Unknown VisibilityLevel: " + level);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionService.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionService.java b/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionService.java
deleted file mode 100644
index 22e986d..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionService.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.distributed.thrift.TTransactionServer;
-import co.cask.tephra.inmemory.InMemoryTransactionService;
-import co.cask.tephra.rpc.ThriftRPCServer;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.api.ElectionHandler;
-import org.apache.twill.discovery.DiscoveryService;
-import org.apache.twill.internal.ServiceListenerAdapter;
-import org.apache.twill.internal.zookeeper.LeaderElection;
-import org.apache.twill.zookeeper.ZKClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- *
- */
-public final class TransactionService extends InMemoryTransactionService {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionService.class);
- private LeaderElection leaderElection;
- private final ZKClient zkClient;
-
- private ThriftRPCServer<TransactionServiceThriftHandler, TTransactionServer> server;
-
- @Inject
- public TransactionService(Configuration conf,
- ZKClient zkClient,
- DiscoveryService discoveryService,
- Provider<TransactionManager> txManagerProvider) {
- super(conf, discoveryService, txManagerProvider);
- this.zkClient = zkClient;
- }
-
- @Override
- protected InetSocketAddress getAddress() {
- if (address.equals("0.0.0.0")) {
- // resolve hostname
- try {
- return new InetSocketAddress(InetAddress.getLocalHost().getHostName(), server.getBindAddress().getPort());
- } catch (UnknownHostException x) {
- LOG.error("Cannot resolve hostname for 0.0.0.0", x);
- }
- }
- return server.getBindAddress();
- }
-
- @Override
- protected void doStart() {
- leaderElection = new LeaderElection(zkClient, "/tx.service/leader", new ElectionHandler() {
- @Override
- public void leader() {
- // if the txManager fails, we should stop the server
- txManager = txManagerProvider.get();
- txManager.addListener(new ServiceListenerAdapter() {
- @Override
- public void failed(State from, Throwable failure) {
- LOG.error("Transaction manager aborted, stopping transaction service");
- TransactionService.this.abort(failure);
- }
- }, MoreExecutors.sameThreadExecutor());
-
- server = ThriftRPCServer.builder(TTransactionServer.class)
- .setHost(address)
- .setPort(port)
- .setWorkerThreads(threads)
- .setMaxReadBufferBytes(maxReadBufferBytes)
- .setIOThreads(ioThreads)
- .build(new TransactionServiceThriftHandler(txManager));
- try {
- server.startAndWait();
- doRegister();
- LOG.info("Transaction Thrift Service started successfully on " + getAddress());
- } catch (Throwable t) {
- LOG.info("Transaction Thrift Service didn't start on " + server.getBindAddress());
- leaderElection.stop();
- notifyFailed(t);
- }
- }
-
- @Override
- public void follower() {
- // First stop the transaction server as un-registering from discovery can block sometimes.
- // That can lead to multiple transaction servers being active at the same time.
- if (server != null && server.isRunning()) {
- server.stopAndWait();
- }
- undoRegister();
- }
- });
- leaderElection.start();
-
- notifyStarted();
- }
-
- @VisibleForTesting
- State thriftRPCServerState() {
- return server.state();
- }
-
- @Override
- protected void doStop() {
- internalStop();
- notifyStopped();
- }
-
- protected void abort(Throwable cause) {
- // try to clear leader status and shutdown RPC
- internalStop();
- notifyFailed(cause);
- }
-
- protected void internalStop() {
- if (leaderElection != null) {
- // NOTE: if was a leader this will cause loosing of leadership which in callback above will
- // de-register service in discovery service and stop the service if needed
- try {
- Uninterruptibles.getUninterruptibly(leaderElection.stop(), 5, TimeUnit.SECONDS);
- } catch (TimeoutException te) {
- LOG.warn("Timed out waiting for leader election cancellation to complete");
- } catch (ExecutionException e) {
- LOG.error("Exception when cancelling leader election.", e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceClient.java b/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceClient.java
deleted file mode 100644
index 4f29730..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceClient.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.InvalidTruncateTimeException;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCouldNotTakeSnapshotException;
-import co.cask.tephra.TransactionNotInProgressException;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import co.cask.tephra.util.ConfigurationFactory;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.thrift.TException;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-
-/**
- * A tx service client
- */
-public class TransactionServiceClient implements TransactionSystemClient {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TransactionServiceClient.class);
-
- // we will use this to provide every call with an tx client
- private ThriftClientProvider clientProvider;
-
- // the retry strategy we will use
- private final RetryStrategyProvider retryStrategyProvider;
-
- /**
- * Utility to be used for basic verification of transaction system availability and functioning
- * @param args arguments list, accepts single option "-v" that makes it to print out more details about started tx
- * @throws Exception
- */
- public static void main(String[] args) throws Exception {
- if (args.length > 1 || (args.length == 1 && !"-v".equals(args[0]))) {
- System.out.println("USAGE: TransactionServiceClient [-v]");
- }
-
- boolean verbose = false;
- if (args.length == 1 && "-v".equals(args[0])) {
- verbose = true;
- }
- doMain(verbose, new ConfigurationFactory().get());
- }
-
- @VisibleForTesting
- public static void doMain(boolean verbose, Configuration conf) throws Exception {
- LOG.info("Starting tx server client test.");
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- new TransactionModules().getDistributedModules(),
- new TransactionClientModule()
- );
-
- ZKClientService zkClient = injector.getInstance(ZKClientService.class);
- zkClient.startAndWait();
-
- try {
- TransactionServiceClient client = injector.getInstance(TransactionServiceClient.class);
- LOG.info("Starting tx...");
- Transaction tx = client.startShort();
- if (verbose) {
- LOG.info("Started tx details: " + tx.toString());
- } else {
- LOG.info("Started tx: " + tx.getTransactionId() +
- ", readPointer: " + tx.getReadPointer() +
- ", invalids: " + tx.getInvalids().length +
- ", inProgress: " + tx.getInProgress().length);
- }
- LOG.info("Checking if canCommit tx...");
- boolean canCommit = client.canCommit(tx, Collections.<byte[]>emptyList());
- LOG.info("canCommit: " + canCommit);
- if (canCommit) {
- LOG.info("Committing tx...");
- boolean committed = client.commit(tx);
- LOG.info("Committed tx: " + committed);
- if (!committed) {
- LOG.info("Aborting tx...");
- client.abort(tx);
- LOG.info("Aborted tx...");
- }
- } else {
- LOG.info("Aborting tx...");
- client.abort(tx);
- LOG.info("Aborted tx...");
- }
- } finally {
- zkClient.stopAndWait();
- }
- }
-
- /**
- * Create from a configuration. This will first attempt to find a zookeeper
- * for service discovery. Otherwise it will look for the port in the
- * config and use localhost.
- * @param config a configuration containing the zookeeper properties
- */
- @Inject
- public TransactionServiceClient(Configuration config,
- ThriftClientProvider clientProvider) {
-
- // initialize the retry logic
- String retryStrat = config.get(
- TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY,
- TxConstants.Service.DEFAULT_DATA_TX_CLIENT_RETRY_STRATEGY);
- if ("backoff".equals(retryStrat)) {
- this.retryStrategyProvider = new RetryWithBackoff.Provider();
- } else if ("n-times".equals(retryStrat)) {
- this.retryStrategyProvider = new RetryNTimes.Provider();
- } else {
- String message = "Unknown Retry Strategy '" + retryStrat + "'.";
- LOG.error(message);
- throw new IllegalArgumentException(message);
- }
- this.retryStrategyProvider.configure(config);
- LOG.debug("Retry strategy is " + this.retryStrategyProvider);
-
- this.clientProvider = clientProvider;
- }
-
- /**
- * This is an abstract class that encapsulates an operation. It provides a
- * method to attempt the actual operation, and it can throw an operation
- * exception.
- * @param <T> The return type of the operation
- */
- abstract static class Operation<T> {
-
- /** the name of the operation. */
- String name;
-
- /** constructor with name of operation. */
- Operation(String name) {
- this.name = name;
- }
-
- /** return the name of the operation. */
- String getName() {
- return name;
- }
-
- /** execute the operation, given an tx client. */
- abstract T execute(TransactionServiceThriftClient client)
- throws Exception;
- }
-
- /** see execute(operation, client). */
- private <T> T execute(Operation<T> operation) throws Exception {
- return execute(operation, null);
- }
-
- /**
- * This is a generic method implementing the somewhat complex execution
- * and retry logic for operations, to avoid repetitive code.
- *
- * Attempts to execute one operation, by obtaining an tx client from
- * the client provider and passing the operation to the client. If the
- * call fails with a Thrift exception, apply the retry strategy. If no
- * more retries are to be made according to the strategy, call the
- * operation's error method to obtain a value to return. Note that error()
- * may also throw an exception. Note also that the retry logic is only
- * applied for thrift exceptions.
- *
- * @param operation The operation to be executed
- * @param provider An tx client provider. If null, then a client will be
- * obtained using the client provider
- * @param <T> The return type of the operation
- * @return the result of the operation, or a value returned by error()
- */
- private <T> T execute(Operation<T> operation, ThriftClientProvider provider) throws Exception {
- RetryStrategy retryStrategy = retryStrategyProvider.newRetryStrategy();
- while (true) {
- // did we get a custom client provider or do we use the default?
- if (provider == null) {
- provider = this.clientProvider;
- }
- // this will throw a TException if it cannot get a client
- try (CloseableThriftClient closeable = provider.getCloseableClient()) {
- // note that this can throw exceptions other than TException
- return operation.execute(closeable.getThriftClient());
-
- } catch (TException te) {
- // determine whether we should retry
- boolean retry = retryStrategy.failOnce();
- if (!retry) {
- // retry strategy is exceeded, throw an operation exception
- String message =
- "Thrift error for " + operation + ": " + te.getMessage();
- LOG.error(message);
- LOG.debug(message, te);
- throw new Exception(message, te);
- } else {
- // call retry strategy before retrying
- retryStrategy.beforeRetry();
- String msg = "Retrying " + operation.getName() + " after Thrift error: " + te.getMessage();
- LOG.info(msg);
- LOG.debug(msg, te);
- }
-
- }
- }
- }
-
- @Override
- public Transaction startLong() {
- try {
- return execute(
- new Operation<Transaction>("startLong") {
- @Override
- public Transaction execute(TransactionServiceThriftClient client)
- throws TException {
- return client.startLong();
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public Transaction startShort() {
- try {
- return execute(
- new Operation<Transaction>("startShort") {
- @Override
- public Transaction execute(TransactionServiceThriftClient client)
- throws TException {
- return client.startShort();
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public Transaction startShort(final int timeout) {
- try {
- return execute(
- new Operation<Transaction>("startShort") {
- @Override
- public Transaction execute(TransactionServiceThriftClient client)
- throws TException {
- return client.startShort(timeout);
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public boolean canCommit(final Transaction tx, final Collection<byte[]> changeIds)
- throws TransactionNotInProgressException {
-
- try {
- return execute(
- new Operation<Boolean>("canCommit") {
- @Override
- public Boolean execute(TransactionServiceThriftClient client)
- throws Exception {
- return client.canCommit(tx, changeIds);
- }
- });
- } catch (TransactionNotInProgressException e) {
- throw e;
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public boolean commit(final Transaction tx) throws TransactionNotInProgressException {
- try {
- return this.execute(
- new Operation<Boolean>("commit") {
- @Override
- public Boolean execute(TransactionServiceThriftClient client)
- throws Exception {
- return client.commit(tx);
- }
- });
- } catch (TransactionNotInProgressException e) {
- throw e;
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public void abort(final Transaction tx) {
- try {
- this.execute(
- new Operation<Boolean>("abort") {
- @Override
- public Boolean execute(TransactionServiceThriftClient client)
- throws TException {
- client.abort(tx);
- return true;
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public boolean invalidate(final long tx) {
- try {
- return this.execute(
- new Operation<Boolean>("invalidate") {
- @Override
- public Boolean execute(TransactionServiceThriftClient client)
- throws TException {
- return client.invalidate(tx);
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public Transaction checkpoint(final Transaction tx) throws TransactionNotInProgressException {
- try {
- return this.execute(
- new Operation<Transaction>("checkpoint") {
- @Override
- Transaction execute(TransactionServiceThriftClient client) throws Exception {
- return client.checkpoint(tx);
- }
- }
- );
- } catch (TransactionNotInProgressException e) {
- throw e;
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
- try {
- return this.execute(
- new Operation<InputStream>("takeSnapshot") {
- @Override
- public InputStream execute(TransactionServiceThriftClient client)
- throws Exception {
- return client.getSnapshotStream();
- }
- });
- } catch (TransactionCouldNotTakeSnapshotException e) {
- throw e;
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public String status() {
- try {
- return this.execute(
- new Operation<String>("status") {
- @Override
- public String execute(TransactionServiceThriftClient client) throws Exception {
- return client.status();
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public void resetState() {
- try {
- this.execute(
- new Operation<Boolean>("resetState") {
- @Override
- public Boolean execute(TransactionServiceThriftClient client)
- throws TException {
- client.resetState();
- return true;
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public boolean truncateInvalidTx(final Set<Long> invalidTxIds) {
- try {
- return this.execute(
- new Operation<Boolean>("truncateInvalidTx") {
- @Override
- public Boolean execute(TransactionServiceThriftClient client) throws TException {
- return client.truncateInvalidTx(invalidTxIds);
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public boolean truncateInvalidTxBefore(final long time) throws InvalidTruncateTimeException {
- try {
- return this.execute(
- new Operation<Boolean>("truncateInvalidTxBefore") {
- @Override
- public Boolean execute(TransactionServiceThriftClient client) throws Exception {
- return client.truncateInvalidTxBefore(time);
- }
- });
- } catch (InvalidTruncateTimeException e) {
- throw e;
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public int getInvalidSize() {
- try {
- return this.execute(
- new Operation<Integer>("getInvalidSize") {
- @Override
- public Integer execute(TransactionServiceThriftClient client) throws TException {
- return client.getInvalidSize();
- }
- });
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftClient.java b/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftClient.java
deleted file mode 100644
index 01f9b4c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftClient.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.InvalidTruncateTimeException;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCouldNotTakeSnapshotException;
-import co.cask.tephra.TransactionNotInProgressException;
-import co.cask.tephra.distributed.thrift.TInvalidTruncateTimeException;
-import co.cask.tephra.distributed.thrift.TTransactionCouldNotTakeSnapshotException;
-import co.cask.tephra.distributed.thrift.TTransactionNotInProgressException;
-import co.cask.tephra.distributed.thrift.TTransactionServer;
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.transport.TTransport;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * This class is a wrapper around the thrift tx service client, it takes
- * Operations, converts them into thrift objects, calls the thrift
- * client, and converts the results back to data fabric classes.
- * This class also instruments the thrift calls with metrics.
- */
-public class TransactionServiceThriftClient {
- private static final Function<byte[], ByteBuffer> BYTES_WRAPPER = new Function<byte[], ByteBuffer>() {
- @Override
- public ByteBuffer apply(byte[] input) {
- return ByteBuffer.wrap(input);
- }
- };
-
- /**
- * The thrift transport layer. We need this when we close the connection.
- */
- TTransport transport;
-
- /**
- * The actual thrift client.
- */
- TTransactionServer.Client client;
-
- /**
- * Whether this client is valid for use.
- */
- private final AtomicBoolean isValid = new AtomicBoolean(true);
-
- /**
- * Constructor from an existing, connected thrift transport.
- *
- * @param transport the thrift transport layer. It must already be connected
- */
- public TransactionServiceThriftClient(TTransport transport) {
- this.transport = transport;
- // thrift protocol layer, we use binary because so does the service
- TProtocol protocol = new TBinaryProtocol(transport);
- // and create a thrift client
- this.client = new TTransactionServer.Client(protocol);
- }
-
- /**
- * close this client. may be called multiple times
- */
- public void close() {
- if (this.transport.isOpen()) {
- this.transport.close();
- }
- }
-
- public Transaction startLong() throws TException {
- try {
- return TransactionConverterUtils.unwrap(client.startLong());
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public Transaction startShort() throws TException {
- try {
- return TransactionConverterUtils.unwrap(client.startShort());
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public Transaction startShort(int timeout) throws TException {
- try {
- return TransactionConverterUtils.unwrap(client.startShortTimeout(timeout));
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public boolean canCommit(Transaction tx, Collection<byte[]> changeIds)
- throws TException, TransactionNotInProgressException {
- try {
- return client.canCommitTx(TransactionConverterUtils.wrap(tx),
- ImmutableSet.copyOf(Iterables.transform(changeIds, BYTES_WRAPPER))).isValue();
- } catch (TTransactionNotInProgressException e) {
- throw new TransactionNotInProgressException(e.getMessage());
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
-
-
- public boolean commit(Transaction tx) throws TException, TransactionNotInProgressException {
- try {
- return client.commitTx(TransactionConverterUtils.wrap(tx)).isValue();
- } catch (TTransactionNotInProgressException e) {
- throw new TransactionNotInProgressException(e.getMessage());
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public void abort(Transaction tx) throws TException {
- try {
- client.abortTx(TransactionConverterUtils.wrap(tx));
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public boolean invalidate(long tx) throws TException {
- try {
- return client.invalidateTx(tx);
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public Transaction checkpoint(Transaction tx) throws TException {
- try {
- return TransactionConverterUtils.unwrap(client.checkpoint(TransactionConverterUtils.wrap(tx)));
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public InputStream getSnapshotStream() throws TException, TransactionCouldNotTakeSnapshotException {
- try {
- ByteBuffer buffer = client.getSnapshot();
- if (buffer.hasArray()) {
- return new ByteArrayInputStream(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
- }
-
- // The ByteBuffer is not backed by array. Read the content to a new byte array and return an InputStream of that.
- byte[] snapshot = new byte[buffer.remaining()];
- buffer.get(snapshot);
- return new ByteArrayInputStream(snapshot);
- } catch (TTransactionCouldNotTakeSnapshotException e) {
- throw new TransactionCouldNotTakeSnapshotException(e.getMessage());
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public String status() throws TException {
- try {
- return client.status();
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public void resetState() throws TException {
- try {
- client.resetState();
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public boolean truncateInvalidTx(Set<Long> invalidTxIds) throws TException {
- try {
- return client.truncateInvalidTx(invalidTxIds).isValue();
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public boolean truncateInvalidTxBefore(long time) throws TException, InvalidTruncateTimeException {
- try {
- return client.truncateInvalidTxBefore(time).isValue();
- } catch (TInvalidTruncateTimeException e) {
- throw new InvalidTruncateTimeException(e.getMessage());
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public int getInvalidSize() throws TException {
- try {
- return client.invalidTxSize();
- } catch (TException e) {
- isValid.set(false);
- throw e;
- }
- }
-
- public boolean isValid() {
- return isValid.get();
- }
-}
[28/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/persist/TransactionEditTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/persist/TransactionEditTest.java b/tephra-core/src/test/java/co/cask/tephra/persist/TransactionEditTest.java
deleted file mode 100644
index 4a3af3c..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/persist/TransactionEditTest.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.TransactionType;
-import com.google.common.collect.Sets;
-import com.google.common.io.ByteArrayDataOutput;
-import com.google.common.io.ByteStreams;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.DataInput;
-import java.io.IOException;
-
-/**
- * test for {@link TransactionEdit}
- */
-public class TransactionEditTest {
- private static final byte[] COL = new byte[] {'c'};
-
- @Test
- public void testV1SerdeCompat() throws Exception {
- TransactionEditCodecs.TransactionEditCodec olderCodec = new TransactionEditCodecs.TransactionEditCodecV1();
- // start tx edit and committed tx edit cover all fields of tx edit
- // NOTE: set visibilityUpperBound to 0 and transaction type to null as this is expected default
- // for decoding older versions that doesn't store it
- verifyDecodingSupportsOlderVersion(TransactionEdit.createStarted(2L, 0L, 1000L, null), olderCodec);
- verifyDecodingSupportsOlderVersion(
- TransactionEdit.createCommitted(2L, Sets.newHashSet(new ChangeId(COL)), 3L, true), olderCodec);
- }
-
- @Test
- public void testV2SerdeCompat() throws Exception {
- TransactionEditCodecs.TransactionEditCodec olderCodec = new TransactionEditCodecs.TransactionEditCodecV2();
- // start tx edit and committed tx edit cover all fields of tx edit
- // NOTE: transaction type to null as this is expected default for decoding older versions that doesn't store it
- verifyDecodingSupportsOlderVersion(TransactionEdit.createStarted(2L, 100L, 1000L, null), olderCodec);
- verifyDecodingSupportsOlderVersion(
- TransactionEdit.createCommitted(2L, Sets.newHashSet(new ChangeId(COL)), 3L, true), olderCodec);
- }
-
- @SuppressWarnings("deprecation")
- private void verifyDecodingSupportsOlderVersion(TransactionEdit edit,
- TransactionEditCodecs.TransactionEditCodec olderCodec)
- throws IOException {
- // encoding with older version of codec
- ByteArrayDataOutput out = ByteStreams.newDataOutput();
- TransactionEditCodecs.encode(edit, out, olderCodec);
-
- // decoding
- TransactionEdit decodedEdit = new TransactionEdit();
- DataInput in = ByteStreams.newDataInput(out.toByteArray());
- decodedEdit.readFields(in);
-
- Assert.assertEquals(edit, decodedEdit);
- }
-
- @Test
- public void testSerialization() throws Exception {
- assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.SHORT, new long[0]));
- assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.SHORT, new long[]{ 2L, 3L }));
- assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.LONG, new long[0]));
- assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.LONG, new long[]{ 2L, 3L }));
-
- assertSerializedEdit(TransactionEdit.createCheckpoint(2L, 1L));
-
- assertSerializedEdit(TransactionEdit.createCommitted(1L, Sets.<ChangeId>newHashSet(), 2L, false));
- assertSerializedEdit(TransactionEdit.createCommitted(1L, Sets.<ChangeId>newHashSet(), 2L, true));
- assertSerializedEdit(TransactionEdit.createCommitted(1L,
- Sets.newHashSet(new ChangeId(new byte[]{'a', 'b', 'c'})), 2L, false));
- assertSerializedEdit(TransactionEdit.createCommitted(1L,
- Sets.newHashSet(new ChangeId(new byte[]{ 'a', 'b', 'c' }), new ChangeId(new byte[]{ 'd', 'e', 'f' })),
- 2L, true));
-
- assertSerializedEdit(TransactionEdit.createCommitting(1L, Sets.<ChangeId>newHashSet()));
- assertSerializedEdit(TransactionEdit.createCommitting(1L,
- Sets.newHashSet(new ChangeId(new byte[]{'a', 'b', 'c'}))));
- assertSerializedEdit(TransactionEdit.createCommitting(1L,
- Sets.newHashSet(new ChangeId(new byte[]{'a', 'b', 'c'}), new ChangeId(new byte[]{'d', 'e', 'f'}))));
-
- assertSerializedEdit(TransactionEdit.createInvalid(1L));
-
- assertSerializedEdit(TransactionEdit.createMoveWatermark(10L));
-
- assertSerializedEdit(TransactionEdit.createStarted(2L, 1L, System.currentTimeMillis() + 1000,
- TransactionType.SHORT));
- assertSerializedEdit(TransactionEdit.createStarted(2L, 1L, System.currentTimeMillis() + 10000,
- TransactionType.LONG));
-
- assertSerializedEdit(TransactionEdit.createTruncateInvalidTx(Sets.newHashSet(new Long(1))));
- assertSerializedEdit(TransactionEdit.createTruncateInvalidTx(
- Sets.newHashSet(new Long(1), new Long(2), new Long(3))));
-
- assertSerializedEdit(TransactionEdit.createTruncateInvalidTxBefore(System.currentTimeMillis()));
- }
-
- private void assertSerializedEdit(TransactionEdit originalEdit) throws IOException {
- ByteArrayDataOutput out = ByteStreams.newDataOutput();
- originalEdit.write(out);
-
- TransactionEdit decodedEdit = new TransactionEdit();
- DataInput in = ByteStreams.newDataInput(out.toByteArray());
- decodedEdit.readFields(in);
-
- Assert.assertEquals(originalEdit, decodedEdit);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/snapshot/SnapshotCodecTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/snapshot/SnapshotCodecTest.java b/tephra-core/src/test/java/co/cask/tephra/snapshot/SnapshotCodecTest.java
deleted file mode 100644
index f3fe2e1..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/snapshot/SnapshotCodecTest.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionNotInProgressException;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionModules;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Assert;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests related to {@link SnapshotCodec} implementations.
- */
-public class SnapshotCodecTest {
- @ClassRule
- public static TemporaryFolder tmpDir = new TemporaryFolder();
-
- @Test
- public void testMinimalDeserilization() throws Exception {
- long now = System.currentTimeMillis();
- long nowWritePointer = now * TxConstants.MAX_TX_PER_MS;
- /*
- * Snapshot consisting of transactions at:
- */
- long tInvalid = nowWritePointer - 5; // t1 - invalid
- long readPtr = nowWritePointer - 4; // t2 - here and earlier committed
- long tLong = nowWritePointer - 3; // t3 - in-progress LONG
- long tCommitted = nowWritePointer - 2; // t4 - committed, changeset (r1, r2)
- long tShort = nowWritePointer - 1; // t5 - in-progress SHORT, canCommit called, changeset (r3, r4)
-
- TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap(ImmutableSortedMap.of(
- tLong, new TransactionManager.InProgressTx(readPtr,
- TransactionManager.getTxExpirationFromWritePointer(
- tLong, TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT),
- TransactionType.LONG),
- tShort, new TransactionManager.InProgressTx(readPtr, now + 1000, TransactionType.SHORT)));
-
- TransactionSnapshot snapshot = new TransactionSnapshot(now, readPtr, nowWritePointer,
- Lists.newArrayList(tInvalid), // invalid
- inProgress, ImmutableMap.<Long, Set<ChangeId>>of(
- tShort, Sets.<ChangeId>newHashSet()),
- ImmutableMap.<Long, Set<ChangeId>>of(
- tCommitted, Sets.<ChangeId>newHashSet()));
-
- Configuration conf1 = new Configuration();
- conf1.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
- SnapshotCodecProvider provider1 = new SnapshotCodecProvider(conf1);
-
- byte[] byteArray;
- try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
- provider1.encode(out, snapshot);
- byteArray = out.toByteArray();
- }
-
- // TransactionSnapshot and TransactionVisibilityState decode should pass now
- TransactionSnapshot txSnapshot = provider1.decode(new ByteArrayInputStream(byteArray));
- TransactionVisibilityState txVisibilityState =
- provider1.decodeTransactionVisibilityState(new ByteArrayInputStream(byteArray));
- assertTransactionVisibilityStateEquals(txSnapshot, txVisibilityState);
-
- // Corrupt the serialization byte array so that full deserialization will fail
- byteArray[byteArray.length - 1] = 'a';
-
- // TransactionVisibilityState decoding should pass since it doesn't decode the committing and committed changesets.
- TransactionVisibilityState txVisibilityState2 = provider1.decodeTransactionVisibilityState(
- new ByteArrayInputStream(byteArray));
- Assert.assertNotNull(txVisibilityState2);
- Assert.assertEquals(txVisibilityState, txVisibilityState2);
- Assert.assertEquals(readPtr, txVisibilityState2.getReadPointer());
- try {
- provider1.decode(new ByteArrayInputStream(byteArray));
- Assert.fail();
- } catch (RuntimeException e) {
- // expected since we modified the serialization bytes
- }
- }
-
- /**
- * In-progress LONG transactions written with DefaultSnapshotCodec will not have the type serialized as part of
- * the data. Since these transactions also contain a non-negative expiration, we need to ensure we reset the type
- * correctly when the snapshot is loaded.
- */
- @Test
- public void testDefaultToV3Compatibility() throws Exception {
- long now = System.currentTimeMillis();
- long nowWritePointer = now * TxConstants.MAX_TX_PER_MS;
- /*
- * Snapshot consisting of transactions at:
- */
- long tInvalid = nowWritePointer - 5; // t1 - invalid
- long readPtr = nowWritePointer - 4; // t2 - here and earlier committed
- long tLong = nowWritePointer - 3; // t3 - in-progress LONG
- long tCommitted = nowWritePointer - 2; // t4 - committed, changeset (r1, r2)
- long tShort = nowWritePointer - 1; // t5 - in-progress SHORT, canCommit called, changeset (r3, r4)
-
- TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap(ImmutableSortedMap.of(
- tLong, new TransactionManager.InProgressTx(readPtr,
- TransactionManager.getTxExpirationFromWritePointer(tLong, TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT),
- TransactionType.LONG),
- tShort, new TransactionManager.InProgressTx(readPtr, now + 1000, TransactionType.SHORT)));
-
- TransactionSnapshot snapshot = new TransactionSnapshot(now, readPtr, nowWritePointer,
- Lists.newArrayList(tInvalid), // invalid
- inProgress,
- ImmutableMap.<Long, Set<ChangeId>>of(
- tShort, Sets.newHashSet(new ChangeId(new byte[]{'r', '3'}), new ChangeId(new byte[]{'r', '4'}))),
- ImmutableMap.<Long, Set<ChangeId>>of(
- tCommitted, Sets.newHashSet(new ChangeId(new byte[]{'r', '1'}), new ChangeId(new byte[]{'r', '2'}))));
-
- Configuration conf1 = new Configuration();
- conf1.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
- SnapshotCodecProvider provider1 = new SnapshotCodecProvider(conf1);
-
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- try {
- provider1.encode(out, snapshot);
- } finally {
- out.close();
- }
-
- TransactionSnapshot snapshot2 = provider1.decode(new ByteArrayInputStream(out.toByteArray()));
- TransactionVisibilityState minTxSnapshot = provider1.decodeTransactionVisibilityState(
- new ByteArrayInputStream(out.toByteArray()));
- assertTransactionVisibilityStateEquals(snapshot2, minTxSnapshot);
-
- assertEquals(snapshot.getReadPointer(), snapshot2.getReadPointer());
- assertEquals(snapshot.getWritePointer(), snapshot2.getWritePointer());
- assertEquals(snapshot.getInvalid(), snapshot2.getInvalid());
- // in-progress transactions will have missing types
- assertNotEquals(snapshot.getInProgress(), snapshot2.getInProgress());
- assertEquals(snapshot.getCommittingChangeSets(), snapshot2.getCommittingChangeSets());
- assertEquals(snapshot.getCommittedChangeSets(), snapshot2.getCommittedChangeSets());
-
- // after fixing in-progress, full snapshot should match
- Map<Long, TransactionManager.InProgressTx> fixedInProgress = TransactionManager.txnBackwardsCompatCheck(
- TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT, 10000L, snapshot2.getInProgress());
- assertEquals(snapshot.getInProgress(), fixedInProgress);
- assertEquals(snapshot, snapshot2);
- }
-
- /**
- * Test full stack serialization for a TransactionManager migrating from DefaultSnapshotCodec to SnapshotCodecV3.
- */
- @Test
- public void testDefaultToV3Migration() throws Exception {
- File testDir = tmpDir.newFolder("testDefaultToV3Migration");
- Configuration conf = new Configuration();
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
-
- Injector injector = Guice.createInjector(new ConfigModule(conf),
- new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());
-
- TransactionManager txManager = injector.getInstance(TransactionManager.class);
- txManager.startAndWait();
-
- txManager.startLong();
-
- // shutdown to force a snapshot
- txManager.stopAndWait();
-
- TransactionStateStorage txStorage = injector.getInstance(TransactionStateStorage.class);
- txStorage.startAndWait();
-
- // confirm that the in-progress entry is missing a type
- TransactionSnapshot snapshot = txStorage.getLatestSnapshot();
- TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState();
- assertTransactionVisibilityStateEquals(snapshot, txVisibilityState);
- assertNotNull(snapshot);
- assertEquals(1, snapshot.getInProgress().size());
- Map.Entry<Long, TransactionManager.InProgressTx> entry =
- snapshot.getInProgress().entrySet().iterator().next();
- assertNull(entry.getValue().getType());
- txStorage.stopAndWait();
-
-
- // start a new Tx manager to test fixup
- Configuration conf2 = new Configuration();
- conf2.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
- conf2.setStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES,
- DefaultSnapshotCodec.class.getName(), SnapshotCodecV3.class.getName());
- Injector injector2 = Guice.createInjector(new ConfigModule(conf2),
- new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());
-
- TransactionManager txManager2 = injector2.getInstance(TransactionManager.class);
- txManager2.startAndWait();
-
- // state should be recovered
- TransactionSnapshot snapshot2 = txManager2.getCurrentState();
- assertEquals(1, snapshot2.getInProgress().size());
- Map.Entry<Long, TransactionManager.InProgressTx> inProgressTx =
- snapshot2.getInProgress().entrySet().iterator().next();
- assertEquals(TransactionType.LONG, inProgressTx.getValue().getType());
-
- // save a new snapshot
- txManager2.stopAndWait();
-
- TransactionStateStorage txStorage2 = injector2.getInstance(TransactionStateStorage.class);
- txStorage2.startAndWait();
-
- TransactionSnapshot snapshot3 = txStorage2.getLatestSnapshot();
- // full snapshot should have deserialized correctly without any fixups
- assertEquals(snapshot2.getInProgress(), snapshot3.getInProgress());
- assertEquals(snapshot2, snapshot3);
- txStorage2.stopAndWait();
- }
-
- @Test
- public void testSnapshotCodecProviderConfiguration() throws Exception {
- Configuration conf = new Configuration(false);
- StringBuilder buf = new StringBuilder();
- for (Class c : TxConstants.Persist.DEFAULT_TX_SNAPHOT_CODEC_CLASSES) {
- if (buf.length() > 0) {
- buf.append(",\n ");
- }
- buf.append(c.getName());
- }
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, buf.toString());
-
- SnapshotCodecProvider codecProvider = new SnapshotCodecProvider(conf);
- SnapshotCodec v1codec = codecProvider.getCodecForVersion(new DefaultSnapshotCodec().getVersion());
- assertNotNull(v1codec);
- assertTrue(v1codec instanceof DefaultSnapshotCodec);
-
- SnapshotCodec v2codec = codecProvider.getCodecForVersion(new SnapshotCodecV2().getVersion());
- assertNotNull(v2codec);
- assertTrue(v2codec instanceof SnapshotCodecV2);
-
- SnapshotCodec v3codec = codecProvider.getCodecForVersion(new SnapshotCodecV3().getVersion());
- assertNotNull(v3codec);
- assertTrue(v3codec instanceof SnapshotCodecV3);
-
- SnapshotCodec v4codec = codecProvider.getCodecForVersion(new SnapshotCodecV4().getVersion());
- assertNotNull(v4codec);
- assertTrue(v4codec instanceof SnapshotCodecV4);
- }
-
- @Test
- public void testSnapshotCodecV4() throws IOException, TransactionNotInProgressException {
- File testDir = tmpDir.newFolder("testSnapshotCodecV4");
- Configuration conf = new Configuration();
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
-
- Injector injector = Guice.createInjector(new ConfigModule(conf),
- new DiscoveryModules().getSingleNodeModules(),
- new TransactionModules().getSingleNodeModules());
-
- TransactionManager txManager = injector.getInstance(TransactionManager.class);
- txManager.startAndWait();
-
- // Create a transaction and a checkpoint transaction
- Transaction transaction = txManager.startLong();
- Transaction checkpointTx = txManager.checkpoint(transaction);
-
- // shutdown to force a snapshot
- txManager.stopAndWait();
-
- // Validate the snapshot on disk
- TransactionStateStorage txStorage = injector.getInstance(TransactionStateStorage.class);
- txStorage.startAndWait();
-
- TransactionSnapshot snapshot = txStorage.getLatestSnapshot();
- TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState();
- assertTransactionVisibilityStateEquals(snapshot, txVisibilityState);
-
- Map<Long, TransactionManager.InProgressTx> inProgress = snapshot.getInProgress();
- Assert.assertEquals(1, inProgress.size());
-
- TransactionManager.InProgressTx inProgressTx = inProgress.get(transaction.getTransactionId());
- Assert.assertNotNull(inProgressTx);
- Assert.assertArrayEquals(checkpointTx.getCheckpointWritePointers(),
- inProgressTx.getCheckpointWritePointers().toLongArray());
-
- txStorage.stopAndWait();
-
- // start a new Tx manager to see if the transaction is restored correctly.
- Injector injector2 = Guice.createInjector(new ConfigModule(conf),
- new DiscoveryModules().getSingleNodeModules(),
- new TransactionModules().getSingleNodeModules());
-
- txManager = injector2.getInstance(TransactionManager.class);
- txManager.startAndWait();
-
- // state should be recovered
- snapshot = txManager.getCurrentState();
- inProgress = snapshot.getInProgress();
- Assert.assertEquals(1, inProgress.size());
-
- inProgressTx = inProgress.get(transaction.getTransactionId());
- Assert.assertNotNull(inProgressTx);
- Assert.assertArrayEquals(checkpointTx.getCheckpointWritePointers(),
- inProgressTx.getCheckpointWritePointers().toLongArray());
-
- // Should be able to commit the transaction
- Assert.assertTrue(txManager.canCommit(checkpointTx, Collections.<byte[]>emptyList()));
- Assert.assertTrue(txManager.commit(checkpointTx));
-
- // save a new snapshot
- txManager.stopAndWait();
-
- TransactionStateStorage txStorage2 = injector2.getInstance(TransactionStateStorage.class);
- txStorage2.startAndWait();
-
- snapshot = txStorage2.getLatestSnapshot();
- Assert.assertTrue(snapshot.getInProgress().isEmpty());
- txStorage2.stopAndWait();
- }
-
- private void assertTransactionVisibilityStateEquals(TransactionVisibilityState expected,
- TransactionVisibilityState input) {
- Assert.assertEquals(expected.getTimestamp(), input.getTimestamp());
- Assert.assertEquals(expected.getReadPointer(), input.getReadPointer());
- Assert.assertEquals(expected.getWritePointer(), input.getWritePointer());
- Assert.assertEquals(expected.getInProgress(), input.getInProgress());
- Assert.assertEquals(expected.getInvalid(), input.getInvalid());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/util/AbstractConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/util/AbstractConfigurationProviderTest.java b/tephra-core/src/test/java/co/cask/tephra/util/AbstractConfigurationProviderTest.java
deleted file mode 100644
index 0ae9ed4..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/util/AbstractConfigurationProviderTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- *
- */
-public abstract class AbstractConfigurationProviderTest {
- @Test
- public void testVersionFactory() {
- HBaseVersion.Version foundVersion = HBaseVersion.get();
- assertEquals(getExpectedVersion(), foundVersion);
- }
-
- protected abstract HBaseVersion.Version getExpectedVersion();
-
- @Test
- public void testConfigurationProvider() {
- Configuration conf = new Configuration();
- conf.set("foo", "bar");
- Configuration newConf = new ConfigurationFactory().get(conf);
- assertNotNull(newConf);
- assertEquals("bar", newConf.get("foo"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/util/HBaseVersionTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/util/HBaseVersionTest.java b/tephra-core/src/test/java/co/cask/tephra/util/HBaseVersionTest.java
deleted file mode 100644
index 1bb1fe2..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/util/HBaseVersionTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import org.junit.Test;
-
-import java.text.ParseException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-/**
- * Tests for HBase version parsing.
- */
-public class HBaseVersionTest {
- @Test
- public void testVersionNumber() throws Exception {
- HBaseVersion.VersionNumber ver = HBaseVersion.VersionNumber.create("1");
- assertVersionNumber(ver, 1, null, null, null, false);
-
- ver = HBaseVersion.VersionNumber.create("1-SNAPSHOT");
- assertVersionNumber(ver, 1, null, null, null, true);
-
- ver = HBaseVersion.VersionNumber.create("1-foo");
- assertVersionNumber(ver, 1, null, null, "foo", false);
-
- ver = HBaseVersion.VersionNumber.create("1-foo-SNAPSHOT");
- assertVersionNumber(ver, 1, null, null, "foo", true);
-
- ver = HBaseVersion.VersionNumber.create("10.0");
- assertVersionNumber(ver, 10, 0, null, null, false);
-
- ver = HBaseVersion.VersionNumber.create("10.0-bar");
- assertVersionNumber(ver, 10, 0, null, "bar", false);
-
- ver = HBaseVersion.VersionNumber.create("3.2.1");
- assertVersionNumber(ver, 3, 2, 1, null, false);
-
- ver = HBaseVersion.VersionNumber.create("3.2.1-SNAPSHOT");
- assertVersionNumber(ver, 3, 2, 1, null, true);
-
- ver = HBaseVersion.VersionNumber.create("3.2.1-baz");
- assertVersionNumber(ver, 3, 2, 1, "baz", false);
-
- ver = HBaseVersion.VersionNumber.create("3.2.1-baz1.2.3");
- assertVersionNumber(ver, 3, 2, 1, "baz1.2.3", false);
-
- ver = HBaseVersion.VersionNumber.create("3.2.1-baz1.2.3-SNAPSHOT");
- assertVersionNumber(ver, 3, 2, 1, "baz1.2.3", true);
-
- try {
- ver = HBaseVersion.VersionNumber.create("abc");
- fail("Invalid verison number 'abc' should have thrown a ParseException");
- } catch (ParseException pe) {
- // expected
- }
-
- try {
- ver = HBaseVersion.VersionNumber.create("1.a.b");
- fail("Invalid verison number '1.a.b' should have thrown a ParseException");
- } catch (ParseException pe) {
- // expected
- }
-
- ver = HBaseVersion.VersionNumber.create("1.2.0-CDH5.7.0");
- assertVersionNumber(ver, 1, 2, 0, "CDH5.7.0", false);
- }
-
- private void assertVersionNumber(HBaseVersion.VersionNumber version, Integer expectedMajor, Integer expectedMinor,
- Integer expectedPatch, String expectedClassifier, boolean snapshot) {
- if (expectedMajor == null) {
- assertNull(version.getMajor());
- } else {
- assertEquals(expectedMajor, version.getMajor());
- }
- if (expectedMinor == null) {
- assertNull(version.getMinor());
- } else {
- assertEquals(expectedMinor, version.getMinor());
- }
- if (expectedPatch == null) {
- assertNull(version.getPatch());
- } else {
- assertEquals(expectedPatch, version.getPatch());
- }
- if (expectedClassifier == null) {
- assertNull(version.getClassifier());
- } else {
- assertEquals(expectedClassifier, version.getClassifier());
- }
- assertEquals(snapshot, version.isSnapshot());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/util/TransactionEditUtil.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/util/TransactionEditUtil.java b/tephra-core/src/test/java/co/cask/tephra/util/TransactionEditUtil.java
deleted file mode 100644
index c3269b0..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/util/TransactionEditUtil.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.persist.TransactionEdit;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
-/**
- * Util class for {@link TransactionEdit} related tests.
- */
-public final class TransactionEditUtil {
- private static Random random = new Random();
-
- /**
- * Generates a number of semi-random {@link TransactionEdit} instances.
- * These are just randomly selected from the possible states, so would not necessarily reflect a real-world
- * distribution.
- *
- * @param numEntries how many entries to generate in the returned list.
- * @return a list of randomly generated transaction log edits.
- */
- public static List<TransactionEdit> createRandomEdits(int numEntries) {
- List<TransactionEdit> edits = Lists.newArrayListWithCapacity(numEntries);
- for (int i = 0; i < numEntries; i++) {
- TransactionEdit.State nextType = TransactionEdit.State.values()[random.nextInt(6)];
- long writePointer = Math.abs(random.nextLong());
- switch (nextType) {
- case INPROGRESS:
- edits.add(
- TransactionEdit.createStarted(writePointer, writePointer - 1,
- System.currentTimeMillis() + 300000L, TransactionType.SHORT));
- break;
- case COMMITTING:
- edits.add(TransactionEdit.createCommitting(writePointer, generateChangeSet(10)));
- break;
- case COMMITTED:
- edits.add(TransactionEdit.createCommitted(writePointer, generateChangeSet(10), writePointer + 1,
- random.nextBoolean()));
- break;
- case INVALID:
- edits.add(TransactionEdit.createInvalid(writePointer));
- break;
- case ABORTED:
- edits.add(TransactionEdit.createAborted(writePointer, TransactionType.SHORT, null));
- break;
- case MOVE_WATERMARK:
- edits.add(TransactionEdit.createMoveWatermark(writePointer));
- break;
- }
- }
- return edits;
- }
-
- private static Set<ChangeId> generateChangeSet(int numEntries) {
- Set<ChangeId> changes = Sets.newHashSet();
- for (int i = 0; i < numEntries; i++) {
- byte[] bytes = new byte[8];
- random.nextBytes(bytes);
- changes.add(new ChangeId(bytes));
- }
- return changes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/util/TxUtilsTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/util/TxUtilsTest.java b/tephra-core/src/test/java/co/cask/tephra/util/TxUtilsTest.java
deleted file mode 100644
index b8188e0..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/util/TxUtilsTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import co.cask.tephra.Transaction;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test cases for {@link TxUtils} utility methods.
- */
-public class TxUtilsTest {
- @Test
- public void testMaxVisibleTimestamp() {
- // make sure we don't overflow with MAX_VALUE write pointer
- assertEquals(Long.MAX_VALUE, TxUtils.getMaxVisibleTimestamp(Transaction.ALL_VISIBLE_LATEST));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/visibility/VisibilityFenceTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/visibility/VisibilityFenceTest.java b/tephra-core/src/test/java/co/cask/tephra/visibility/VisibilityFenceTest.java
deleted file mode 100644
index 750fe28..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/visibility/VisibilityFenceTest.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.visibility;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TransactionConflictException;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import com.google.common.base.Charsets;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * The following are all the possible cases when using {@link VisibilityFence}.
- *
- * In the below table,
- * "Read Txn" refers to the transaction that contains the read fence
- * "Before Write", "During Write" and "After Write" refer to the write transaction time
- * "Before Write Fence", "During Write Fence", "After Write Fence" refer to the write fence transaction time
- *
- * Timeline is: Before Write < During Write < After Write < Before Write Fence < During Write Fence <
- * After Write Fence
- *
- * +------+----------------------+----------------------+--------------------+--------------------+
- * | Case | Read Txn Start | Read Txn Commit | Conflict on Commit | Conflict on Commit |
- * | | | | of Read Txn | of Write Fence |
- * +------+----------------------+----------------------+--------------------+--------------------+
- * | 1 | Before Write | Before Write | No | No |
- * | 2 | Before Write | During Write | No | No |
- * | 3 | Before Write | After Write | No | No |
- * | 4 | Before Write | Before Write Fence | No | No |
- * | 5 | Before Write | During Write Fence | No | Yes |
- * | 6 | Before Write | After Write Fence | Yes | No |
- * | | | | | |
- * | 7 | During Write | During Write | No | No |
- * | 8 | During Write | After Write | No | No |
- * | 9 | During Write | Before Write Fence | No | No |
- * | 10 | During Write | During Write Fence | No | Yes |
- * | 11 | During Write | After Write Fence | Yes | No |
- * | | | | | |
- * | 12 | After Write | After Write | No | No |
- * | 13 | After Write | Before Write Fence | No | No |
- * | 14 | After Write | During Write Fence | No | Yes # |
- * | 15 | After Write | After Write Fence | Yes # | No |
- * | | | | | |
- * | 16 | Before Write Fence | Before Write Fence | No | No |
- * | 17 | Before Write Fence | During Write Fence | No | Yes # |
- * | 18 | Before Write Fence | After Write Fence | Yes # | No |
- * | | | | | |
- * | 19 | During Write Fence | During Write Fence | No | No |
- * | 20 | During Write Fence | After Write Fence | No | No |
- * | | | | | |
- * | 21 | After Write Fence | After Write Fence | No | No |
- * +------+----------------------+----------------------+--------------------+--------------------+
- *
- * Note: Cases marked with '#' in conflict column should not conflict, however current implementation causes
- * them to conflict. The remaining conflicts are a result of the fence.
- *
- * In the current implementation of VisibilityFence, read txns that start "Before Write", "During Write",
- * and "After Write" can be represented by read txns that start "Before Write Fence".
- * Verifying cases 16, 17, 18, 20 and 21 will effectively cover all other cases.
- */
-public class VisibilityFenceTest {
- private static Configuration conf = new Configuration();
-
- private static TransactionManager txManager = null;
-
- @BeforeClass
- public static void before() {
- txManager = new TransactionManager(conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
- txManager.startAndWait();
- }
-
- @AfterClass
- public static void after() {
- txManager.stopAndWait();
- }
-
- @Test
- public void testFence1() throws Exception {
- byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
-
- // Writer updates data here in a separate transaction (code not shown)
- // start tx
- // update
- // commit tx
-
- // Readers use fence to indicate that they are interested in changes to specific data
- TransactionAware readFenceCase16 = VisibilityFence.create(fenceId);
- TransactionContext readTxContextCase16 =
- new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase16);
- readTxContextCase16.start();
- readTxContextCase16.finish();
-
- TransactionAware readFenceCase17 = VisibilityFence.create(fenceId);
- TransactionContext readTxContextCase17 =
- new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase17);
- readTxContextCase17.start();
-
- TransactionAware readFenceCase18 = VisibilityFence.create(fenceId);
- TransactionContext readTxContextCase18 =
- new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase18);
- readTxContextCase18.start();
-
- // Now writer needs to wait for in-progress readers to see the change, it uses write fence to do so
- // Start write fence txn
- TransactionAware writeFence = new WriteFence(fenceId);
- TransactionContext writeTxContext = new TransactionContext(new InMemoryTxSystemClient(txManager), writeFence);
- writeTxContext.start();
-
- TransactionAware readFenceCase20 = VisibilityFence.create(fenceId);
- TransactionContext readTxContextCase20 =
- new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase20);
- readTxContextCase20.start();
-
- readTxContextCase17.finish();
-
- assertTxnConflict(writeTxContext);
- writeTxContext.start();
-
- // Commit write fence txn can commit without conflicts at this point
- writeTxContext.finish();
-
- TransactionAware readFenceCase21 = VisibilityFence.create(fenceId);
- TransactionContext readTxContextCase21 =
- new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase21);
- readTxContextCase21.start();
-
- assertTxnConflict(readTxContextCase18);
- readTxContextCase20.finish();
- readTxContextCase21.finish();
- }
-
- private void assertTxnConflict(TransactionContext txContext) throws Exception {
- try {
- txContext.finish();
- Assert.fail("Expected transaction to fail");
- } catch (TransactionConflictException e) {
- // Expected
- txContext.abort();
- }
- }
-
- @Test
- public void testFence2() throws Exception {
- byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
-
- // Readers use fence to indicate that they are interested in changes to specific data
- // Reader 1
- TransactionAware readFence1 = VisibilityFence.create(fenceId);
- TransactionContext readTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence1);
- readTxContext1.start();
-
- // Reader 2
- TransactionAware readFence2 = VisibilityFence.create(fenceId);
- TransactionContext readTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence2);
- readTxContext2.start();
-
- // Reader 3
- TransactionAware readFence3 = VisibilityFence.create(fenceId);
- TransactionContext readTxContext3 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence3);
- readTxContext3.start();
-
- // Writer updates data here in a separate transaction (code not shown)
- // start tx
- // update
- // commit tx
-
- // Now writer needs to wait for readers 1, 2, and 3 to see the change, it uses write fence to do so
- TransactionAware writeFence = new WriteFence(fenceId);
- TransactionContext writeTxContext = new TransactionContext(new InMemoryTxSystemClient(txManager), writeFence);
- writeTxContext.start();
-
- // Reader 1 commits before writeFence is committed
- readTxContext1.finish();
-
- try {
- // writeFence will throw exception since Reader 1 committed without seeing changes
- writeTxContext.finish();
- Assert.fail("Expected transaction to fail");
- } catch (TransactionConflictException e) {
- // Expected
- writeTxContext.abort();
- }
-
- // Start over writeFence again
- writeTxContext.start();
-
- // Now, Reader 3 commits before writeFence
- // Note that Reader 3 does not conflict with Reader 1
- readTxContext3.finish();
-
- try {
- // writeFence will throw exception again since Reader 3 committed without seeing changes
- writeTxContext.finish();
- Assert.fail("Expected transaction to fail");
- } catch (TransactionConflictException e) {
- // Expected
- writeTxContext.abort();
- }
-
- // Start over writeFence again
- writeTxContext.start();
- // This time writeFence commits before the other readers
- writeTxContext.finish();
-
- // After this point all readers will see the change
-
- try {
- // Reader 2 commits after writeFence, hence this commit with throw exception
- readTxContext2.finish();
- Assert.fail("Expected transaction to fail");
- } catch (TransactionConflictException e) {
- // Expected
- readTxContext2.abort();
- }
-
- // Reader 2 has to abort and start over again. It will see the changes now.
- readTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence2);
- readTxContext2.start();
- readTxContext2.finish();
- }
-
- @Test
- public void testFenceAwait() throws Exception {
- byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
-
- final TransactionContext fence1 = new TransactionContext(new InMemoryTxSystemClient(txManager),
- VisibilityFence.create(fenceId));
- fence1.start();
- final TransactionContext fence2 = new TransactionContext(new InMemoryTxSystemClient(txManager),
- VisibilityFence.create(fenceId));
- fence2.start();
- TransactionContext fence3 = new TransactionContext(new InMemoryTxSystemClient(txManager),
- VisibilityFence.create(fenceId));
- fence3.start();
-
- final AtomicInteger attempts = new AtomicInteger();
- TransactionSystemClient customTxClient = new InMemoryTxSystemClient(txManager) {
- @Override
- public Transaction startShort() {
- Transaction transaction = super.startShort();
- try {
- switch (attempts.getAndIncrement()) {
- case 0:
- fence1.finish();
- break;
- case 1:
- fence2.finish();
- break;
- case 2:
- break;
- default:
- throw new IllegalStateException("Unexpected state");
- }
- } catch (TransactionFailureException e) {
- Throwables.propagate(e);
- }
- return transaction;
- }
- };
-
- FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, customTxClient);
- fenceWait.await(1000, TimeUnit.MILLISECONDS);
- Assert.assertEquals(3, attempts.get());
-
- try {
- fence3.finish();
- Assert.fail("Expected transaction to fail");
- } catch (TransactionConflictException e) {
- // Expected exception
- fence3.abort();
- }
-
- fence3.start();
- fence3.finish();
- }
-
- @Test
- public void testFenceTimeout() throws Exception {
- byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
-
- final TransactionContext fence1 = new TransactionContext(new InMemoryTxSystemClient(txManager),
- VisibilityFence.create(fenceId));
- fence1.start();
-
- final long timeout = 100;
- final TimeUnit timeUnit = TimeUnit.MILLISECONDS;
- final AtomicInteger attempts = new AtomicInteger();
- TransactionSystemClient customTxClient = new InMemoryTxSystemClient(txManager) {
- @Override
- public Transaction startShort() {
- Transaction transaction = super.startShort();
- try {
- switch (attempts.getAndIncrement()) {
- case 0:
- fence1.finish();
- break;
- }
- timeUnit.sleep(timeout + 1);
- } catch (InterruptedException | TransactionFailureException e) {
- Throwables.propagate(e);
- }
- return transaction;
- }
- };
-
- try {
- FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, customTxClient);
- fenceWait.await(timeout, timeUnit);
- Assert.fail("Expected await to fail");
- } catch (TimeoutException e) {
- // Expected exception
- }
- Assert.assertEquals(1, attempts.get());
-
- FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, customTxClient);
- fenceWait.await(timeout, timeUnit);
- Assert.assertEquals(2, attempts.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/ThriftTransactionSystemTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/ThriftTransactionSystemTest.java b/tephra-core/src/test/java/org/apache/tephra/ThriftTransactionSystemTest.java
new file mode 100644
index 0000000..28000ff
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/ThriftTransactionSystemTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.util.Modules;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.distributed.TransactionService;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ThriftTransactionSystemTest extends TransactionSystemTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ThriftTransactionSystemTest.class);
+
+ private static InMemoryZKServer zkServer;
+ private static ZKClientService zkClientService;
+ private static TransactionService txService;
+ private static TransactionStateStorage storage;
+ private static TransactionSystemClient txClient;
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @BeforeClass
+ public static void start() throws Exception {
+ zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
+ zkServer.startAndWait();
+
+ Configuration conf = new Configuration();
+ conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
+ conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
+ conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
+ conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
+
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ Modules.override(new TransactionModules().getDistributedModules())
+ .with(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class).in(Scopes.SINGLETON);
+ }
+ }),
+ new TransactionClientModule()
+ );
+
+ zkClientService = injector.getInstance(ZKClientService.class);
+ zkClientService.startAndWait();
+
+ // start a tx server
+ txService = injector.getInstance(TransactionService.class);
+ storage = injector.getInstance(TransactionStateStorage.class);
+ txClient = injector.getInstance(TransactionSystemClient.class);
+ try {
+ LOG.info("Starting transaction service");
+ txService.startAndWait();
+ } catch (Exception e) {
+ LOG.error("Failed to start service: ", e);
+ }
+ }
+
+ @Before
+ public void reset() throws Exception {
+ getClient().resetState();
+ }
+
+ @AfterClass
+ public static void stop() throws Exception {
+ txService.stopAndWait();
+ storage.stopAndWait();
+ zkClientService.stopAndWait();
+ zkServer.stopAndWait();
+ }
+
+ @Override
+ protected TransactionSystemClient getClient() throws Exception {
+ return txClient;
+ }
+
+ @Override
+ protected TransactionStateStorage getStateStorage() throws Exception {
+ return storage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/TransactionAdminTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/TransactionAdminTest.java b/tephra-core/src/test/java/org/apache/tephra/TransactionAdminTest.java
new file mode 100644
index 0000000..9305229
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/TransactionAdminTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.util.Modules;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.distributed.TransactionService;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+
+public class TransactionAdminTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAdminTest.class);
+
+ private static Configuration conf;
+ private static InMemoryZKServer zkServer;
+ private static ZKClientService zkClientService;
+ private static TransactionService txService;
+ private static TransactionSystemClient txClient;
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @BeforeClass
+ public static void start() throws Exception {
+ zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
+ zkServer.startAndWait();
+
+ conf = new Configuration();
+ conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
+ conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
+ conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
+ conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
+
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ Modules.override(new TransactionModules().getDistributedModules())
+ .with(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class).in(Scopes.SINGLETON);
+ }
+ }),
+ new TransactionClientModule()
+ );
+
+ zkClientService = injector.getInstance(ZKClientService.class);
+ zkClientService.startAndWait();
+
+ // start a tx server
+ txService = injector.getInstance(TransactionService.class);
+ txClient = injector.getInstance(TransactionSystemClient.class);
+ try {
+ LOG.info("Starting transaction service");
+ txService.startAndWait();
+ } catch (Exception e) {
+ LOG.error("Failed to start service: ", e);
+ }
+ }
+
+ @Before
+ public void reset() throws Exception {
+ txClient.resetState();
+ }
+
+ @AfterClass
+ public static void stop() throws Exception {
+ txService.stopAndWait();
+ zkClientService.stopAndWait();
+ zkServer.stopAndWait();
+ }
+
+ @Test
+ public void testPrintUsage() throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(out), new PrintStream(err));
+ int status = txAdmin.doMain(new String[0], conf);
+ Assert.assertEquals(1, status);
+ //noinspection ConstantConditions
+ Assert.assertTrue(err.toString("UTF-8").startsWith("Usage:"));
+ Assert.assertEquals(0, out.toByteArray().length);
+ }
+
+ @Test
+ public void testTruncateInvalidTx() throws Exception {
+ Transaction tx1 = txClient.startLong();
+ Transaction tx2 = txClient.startShort();
+ txClient.invalidate(tx1.getTransactionId());
+ txClient.invalidate(tx2.getTransactionId());
+ Assert.assertEquals(2, txClient.getInvalidSize());
+
+ TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(System.out), new PrintStream(System.err));
+ int status = txAdmin.doMain(new String[]{"--truncate-invalid-tx", String.valueOf(tx2.getTransactionId())}, conf);
+ Assert.assertEquals(0, status);
+ Assert.assertEquals(1, txClient.getInvalidSize());
+ }
+
+ @Test
+ public void testTruncateInvalidTxBefore() throws Exception {
+ Transaction tx1 = txClient.startLong();
+ TimeUnit.MILLISECONDS.sleep(1);
+ long beforeTx2 = System.currentTimeMillis();
+ Transaction tx2 = txClient.startLong();
+
+ // Try before invalidation
+ Assert.assertEquals(0, txClient.getInvalidSize());
+ TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(System.out), new PrintStream(System.err));
+ int status = txAdmin.doMain(new String[]{"--truncate-invalid-tx-before", String.valueOf(beforeTx2)}, conf);
+ // Assert command failed due to in-progress transactions
+ Assert.assertEquals(1, status);
+ // Assert no change to invalid size
+ Assert.assertEquals(0, txClient.getInvalidSize());
+
+ txClient.invalidate(tx1.getTransactionId());
+ txClient.invalidate(tx2.getTransactionId());
+ Assert.assertEquals(2, txClient.getInvalidSize());
+
+ status = txAdmin.doMain(new String[]{"--truncate-invalid-tx-before", String.valueOf(beforeTx2)}, conf);
+ Assert.assertEquals(0, status);
+ Assert.assertEquals(1, txClient.getInvalidSize());
+ }
+
+ @Test
+ public void testGetInvalidTxSize() throws Exception {
+ Transaction tx1 = txClient.startShort();
+ txClient.startLong();
+ txClient.invalidate(tx1.getTransactionId());
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(out), new PrintStream(err));
+ int status = txAdmin.doMain(new String[]{"--get-invalid-tx-size"}, conf);
+ Assert.assertEquals(0, status);
+ //noinspection ConstantConditions
+ Assert.assertTrue(out.toString("UTF-8").contains("Invalid list size: 1\n"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/TransactionContextTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/TransactionContextTest.java b/tephra-core/src/test/java/org/apache/tephra/TransactionContextTest.java
new file mode 100644
index 0000000..20f6944
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/TransactionContextTest.java
@@ -0,0 +1,676 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import com.google.inject.util.Modules;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.snapshot.SnapshotCodecV4;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Tests the transaction executor.
+ */
+public class TransactionContextTest {
+ private static DummyTxClient txClient;
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ final Configuration conf = new Configuration();
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new DiscoveryModules().getInMemoryModules(),
+ Modules.override(
+ new TransactionModules().getInMemoryModules()).with(new AbstractModule() {
+ @Override
+ protected void configure() {
+ TransactionManager txManager = new TransactionManager(conf);
+ txManager.startAndWait();
+ bind(TransactionManager.class).toInstance(txManager);
+ bind(TransactionSystemClient.class).to(DummyTxClient.class).in(Singleton.class);
+ }
+ }));
+
+ txClient = (DummyTxClient) injector.getInstance(TransactionSystemClient.class);
+ }
+
+ final DummyTxAware ds1 = new DummyTxAware(), ds2 = new DummyTxAware();
+
+ static final byte[] A = { 'a' };
+ static final byte[] B = { 'b' };
+
+ private static TransactionContext newTransactionContext(TransactionAware... txAwares) {
+ return new TransactionContext(txClient, txAwares);
+ }
+
+ @Before
+ public void resetTxAwares() {
+ ds1.reset();
+ ds2.reset();
+ }
+
+ @Test
+ public void testSuccessful() throws TransactionFailureException, InterruptedException {
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction
+ context.finish();
+ // verify both are committed and post-committed
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertTrue(ds1.postCommitted);
+ Assert.assertTrue(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
+ }
+
+ @Test
+ public void testPostCommitFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failPostCommitTxOnce = InduceFailure.ThrowException;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail but without rollback as the failure happens post-commit
+ try {
+ context.finish();
+ Assert.fail("post commit failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("post failure", e.getCause().getMessage());
+ }
+ // verify both are committed and post-committed
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertTrue(ds1.postCommitted);
+ Assert.assertTrue(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
+ }
+
+ @Test
+ public void testPersistFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ThrowException;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("Persist should have failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("persist failure", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testPersistFalse() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("Persist should have failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testPersistAndRollbackFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ThrowException;
+ ds1.failRollbackTxOnce = InduceFailure.ThrowException;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("Persist should have failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("persist failure", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is invalidated
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
+ }
+
+ @Test
+ public void testPersistAndRollbackFalse() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
+ ds1.failRollbackTxOnce = InduceFailure.ReturnFalse;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("Persist should have failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
+ }
+ // verify both are rolled back and tx is invalidated
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
+ }
+
+ @Test
+ public void testCommitFalse() throws TransactionFailureException, InterruptedException {
+ txClient.failCommits = 1;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("commit failed - exception should be thrown");
+ } catch (TransactionConflictException e) {
+ Assert.assertNull(e.getCause());
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testCanCommitFalse() throws TransactionFailureException, InterruptedException {
+ txClient.failCanCommitOnce = true;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("commit failed - exception should be thrown");
+ } catch (TransactionConflictException e) {
+ Assert.assertNull(e.getCause());
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertFalse(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testChangesAndRollbackFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failChangesTxOnce = InduceFailure.ThrowException;
+ ds1.failRollbackTxOnce = InduceFailure.ThrowException;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ context.start();
+ // add a change to ds1 and ds2
+ ds1.addChange(A);
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("get changes failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("changes failure", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is invalidated
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertFalse(ds2.checked);
+ Assert.assertFalse(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
+ }
+
+ @Test
+ public void testStartAndRollbackFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failStartTxOnce = InduceFailure.ThrowException;
+ TransactionContext context = newTransactionContext(ds1, ds2);
+ // start transaction
+ try {
+ context.start();
+ Assert.fail("start failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("start failure", e.getCause().getMessage());
+ }
+ // verify both are not rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertFalse(ds2.started);
+ Assert.assertFalse(ds1.checked);
+ Assert.assertFalse(ds2.checked);
+ Assert.assertFalse(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testAddThenSuccess() throws TransactionFailureException, InterruptedException {
+ TransactionContext context = newTransactionContext(ds1);
+ // start transaction
+ context.start();
+ // add a change to ds1
+ ds1.addChange(A);
+ // add ds2 to the tx
+ context.addTransactionAware(ds2);
+ // add a change to ds2
+ ds2.addChange(B);
+ // commit transaction
+ context.finish();
+ // verify both are committed and post-committed
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertTrue(ds1.postCommitted);
+ Assert.assertTrue(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
+ }
+
+ @Test
+ public void testAddThenFailure() throws TransactionFailureException, InterruptedException {
+ ds2.failCommitTxOnce = InduceFailure.ThrowException;
+
+ TransactionContext context = newTransactionContext(ds1);
+ // start transaction
+ context.start();
+ // add a change to ds1
+ ds1.addChange(A);
+ // add ds2 to the tx
+ context.addTransactionAware(ds2);
+ // add a change to ds2
+ ds2.addChange(B);
+ // commit transaction should fail and cause rollback
+ try {
+ context.finish();
+ Assert.fail("Persist should have failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("persist failure", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testAddThenRemoveSuccess() throws TransactionFailureException {
+ TransactionContext context = newTransactionContext();
+
+ context.start();
+ Assert.assertTrue(context.addTransactionAware(ds1));
+ ds1.addChange(A);
+
+ try {
+ context.removeTransactionAware(ds1);
+ Assert.fail("Removal of TransactionAware should fails when there is active transaction.");
+ } catch (IllegalStateException e) {
+ // Expected
+ }
+
+ context.finish();
+
+ Assert.assertTrue(context.removeTransactionAware(ds1));
+ // Removing a TransactionAware not added before should returns false
+ Assert.assertFalse(context.removeTransactionAware(ds2));
+
+ // Verify ds1 is committed and post-committed
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds1.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+
+ // Verify nothing happen to ds2
+ Assert.assertFalse(ds2.started);
+ Assert.assertFalse(ds2.checked);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertFalse(ds2.rolledBack);
+
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
+ }
+
+ @Test
+ public void testAndThenRemoveOnFailure() throws TransactionFailureException {
+ ds1.failCommitTxOnce = InduceFailure.ThrowException;
+ TransactionContext context = newTransactionContext();
+
+ context.start();
+ Assert.assertTrue(context.addTransactionAware(ds1));
+ ds1.addChange(A);
+
+ try {
+ context.finish();
+ Assert.fail("Persist should have failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("persist failure", e.getCause().getMessage());
+ }
+
+ Assert.assertTrue(context.removeTransactionAware(ds1));
+
+ // Verify ds1 is rolled back
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ enum InduceFailure { NoFailure, ReturnFalse, ThrowException }
+
+ static class DummyTxAware implements TransactionAware {
+
+ Transaction tx;
+ boolean started = false;
+ boolean committed = false;
+ boolean checked = false;
+ boolean rolledBack = false;
+ boolean postCommitted = false;
+ List<byte[]> changes = Lists.newArrayList();
+
+ InduceFailure failStartTxOnce = InduceFailure.NoFailure;
+ InduceFailure failChangesTxOnce = InduceFailure.NoFailure;
+ InduceFailure failCommitTxOnce = InduceFailure.NoFailure;
+ InduceFailure failPostCommitTxOnce = InduceFailure.NoFailure;
+ InduceFailure failRollbackTxOnce = InduceFailure.NoFailure;
+
+ void addChange(byte[] key) {
+ changes.add(key);
+ }
+
+ void reset() {
+ tx = null;
+ started = false;
+ checked = false;
+ committed = false;
+ rolledBack = false;
+ postCommitted = false;
+ changes.clear();
+ }
+
+ @Override
+ public void startTx(Transaction tx) {
+ reset();
+ started = true;
+ this.tx = tx;
+ if (failStartTxOnce == InduceFailure.ThrowException) {
+ failStartTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("start failure");
+ }
+ }
+
+ @Override
+ public void updateTx(Transaction tx) {
+ this.tx = tx;
+ }
+
+ @Override
+ public Collection<byte[]> getTxChanges() {
+ checked = true;
+ if (failChangesTxOnce == InduceFailure.ThrowException) {
+ failChangesTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("changes failure");
+ }
+ return ImmutableList.copyOf(changes);
+ }
+
+ @Override
+ public boolean commitTx() throws Exception {
+ committed = true;
+ if (failCommitTxOnce == InduceFailure.ThrowException) {
+ failCommitTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("persist failure");
+ }
+ if (failCommitTxOnce == InduceFailure.ReturnFalse) {
+ failCommitTxOnce = InduceFailure.NoFailure;
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postTxCommit() {
+ postCommitted = true;
+ if (failPostCommitTxOnce == InduceFailure.ThrowException) {
+ failPostCommitTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("post failure");
+ }
+ }
+
+ @Override
+ public boolean rollbackTx() throws Exception {
+ rolledBack = true;
+ if (failRollbackTxOnce == InduceFailure.ThrowException) {
+ failRollbackTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("rollback failure");
+ }
+ if (failRollbackTxOnce == InduceFailure.ReturnFalse) {
+ failRollbackTxOnce = InduceFailure.NoFailure;
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String getTransactionAwareName() {
+ return "dummy";
+ }
+ }
+
+ static class DummyTxClient extends InMemoryTxSystemClient {
+
+ boolean failCanCommitOnce = false;
+ int failCommits = 0;
+ enum CommitState {
+ Started, Committed, Aborted, Invalidated
+ }
+ CommitState state = CommitState.Started;
+
+ @Inject
+ DummyTxClient(TransactionManager txmgr) {
+ super(txmgr);
+ }
+
+ @Override
+ public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
+ if (failCanCommitOnce) {
+ failCanCommitOnce = false;
+ return false;
+ } else {
+ return super.canCommit(tx, changeIds);
+ }
+ }
+
+ @Override
+ public boolean commit(Transaction tx) throws TransactionNotInProgressException {
+ if (failCommits-- > 0) {
+ return false;
+ } else {
+ state = CommitState.Committed;
+ return super.commit(tx);
+ }
+ }
+
+ @Override
+ public Transaction startLong() {
+ state = CommitState.Started;
+ return super.startLong();
+ }
+
+ @Override
+ public Transaction startShort() {
+ state = CommitState.Started;
+ return super.startShort();
+ }
+
+ @Override
+ public Transaction startShort(int timeout) {
+ state = CommitState.Started;
+ return super.startShort(timeout);
+ }
+
+ @Override
+ public void abort(Transaction tx) {
+ state = CommitState.Aborted;
+ super.abort(tx);
+ }
+
+ @Override
+ public boolean invalidate(long tx) {
+ state = CommitState.Invalidated;
+ return super.invalidate(tx);
+ }
+ }
+}
[19/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessorTest.java
new file mode 100644
index 0000000..5d49ac1
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessorTest.java
@@ -0,0 +1,771 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96.coprocessor;
+
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.ipc.RpcServerInterface;
+import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.Leases;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.HDFSTransactionStateStorage;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.snapshot.DefaultSnapshotCodec;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.util.TxUtils;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
+ */
+public class TransactionProcessorTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
+
+ // 8 versions, 1 hour apart, latest is current ts.
+ private static final long[] V;
+
+ static {
+ long now = System.currentTimeMillis();
+ V = new long[9];
+ for (int i = 0; i < V.length; i++) {
+ V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
+ }
+ }
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+ private static MiniDFSCluster dfsCluster;
+ private static Configuration conf;
+ private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
+ private static TransactionVisibilityState txVisibilityState;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration hConf = new Configuration();
+ String rootDir = tmpFolder.newFolder().getAbsolutePath();
+ hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
+ hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
+
+ dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
+ dfsCluster.waitActive();
+ conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
+
+ conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
+ conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ String localTestDir = tmpFolder.newFolder().getAbsolutePath();
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+
+ // write an initial transaction snapshot
+ TransactionSnapshot txSnapshot =
+ TransactionSnapshot.copyFrom(
+ System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
+ // this will set visibility upper bound to V[6]
+ Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
+ TransactionType.SHORT))),
+ new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
+ txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
+ txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
+ txSnapshot.getInProgress());
+ HDFSTransactionStateStorage tmpStorage =
+ new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
+ tmpStorage.startAndWait();
+ tmpStorage.writeSnapshot(txSnapshot);
+ tmpStorage.stopAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ @Test
+ public void testDataJanitorRegionScanner() throws Exception {
+ String tableName = "TestRegionScanner";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ for (int i = 1; i <= 8; i++) {
+ for (int k = 1; k <= i; k++) {
+ Put p = new Put(Bytes.toBytes(i));
+ p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
+ region.put(p);
+ }
+ }
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should only return the valid rows - testing that cleanup works on flush
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+
+ // first returned value should be "4" with version "4"
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 4, new long[] {V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 5, new long[] {V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});
+
+ results.clear();
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteFiltering() throws Exception {
+ String tableName = "TestDeleteFiltering";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ byte[] row = Bytes.toBytes(1);
+ for (int i = 4; i < V.length; i++) {
+ Put p = new Put(row);
+ p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
+ region.put(p);
+ }
+
+ // delete from the third entry back
+ // take that cell's timestamp + 1 to simulate a delete in a new tx
+ long deleteTs = V[5] + 1;
+ Delete d = new Delete(row, deleteTs);
+ LOG.info("Issuing delete at timestamp " + deleteTs);
+ // row deletes are not yet supported (TransactionAwareHTable normally handles this)
+ d.deleteColumns(familyBytes, columnBytes, deleteTs);
+ region.delete(d);
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, we should drop the deleted version, but not the others
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should return row with versions at: V[8], V[6].
+ // V[7] is invalid and V[5] and prior are deleted.
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+ // should be only one row
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 1,
+ new long[]{V[8], V[6], deleteTs},
+ new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteMarkerCleanup() throws Exception {
+ String tableName = "TestDeleteMarkerCleanup";
+ byte[] familyBytes = Bytes.toBytes("f");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+
+ // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
+ long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
+ // deletes are performed after the writes, but still before the visibility upper bound
+ long deleteTs = writeTs + 1;
+ // write separate columns to confirm that delete markers survive across flushes
+ byte[] row = Bytes.toBytes(100);
+ Put p = new Put(row);
+
+ LOG.info("Writing columns at timestamp " + writeTs);
+ for (int i = 0; i < 5; i++) {
+ byte[] iBytes = Bytes.toBytes(i);
+ p.add(familyBytes, iBytes, writeTs, iBytes);
+ }
+ region.put(p);
+ // read all back
+ Scan scan = new Scan(row);
+ RegionScanner regionScanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ for (int i = 0; i < 5; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a flush to clear the memstore
+ LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // delete the odd entries
+ for (int i = 0; i < 5; i++) {
+ if (i % 2 == 1) {
+ // deletes are performed as puts with empty values
+ Put deletePut = new Put(row);
+ deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
+ region.put(deletePut);
+ }
+ }
+
+ // read all back
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ LOG.info("Got cell " + cell);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force another flush on the delete markers
+ // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
+ LOG.info("After delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a major compaction
+ LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
+ region.compactStores(true);
+
+ // perform a raw scan (no filter) to confirm that the delete markers are now gone
+ scan = new Scan(row);
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+ } finally {
+ region.close();
+ }
+ }
+
+ /**
+ * Test that we correctly preserve the timestamp set for column family delete markers. This is not
+ * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
+ * that we make it easy to interoperate with other systems.
+ */
+ @Test
+ public void testFamilyDeleteTimestamp() throws Exception {
+ String tableName = "TestFamilyDeleteTimestamp";
+ byte[] family1Bytes = Bytes.toBytes("f1");
+ byte[] columnBytes = Bytes.toBytes("c");
+ byte[] rowBytes = Bytes.toBytes("row");
+ byte[] valBytes = Bytes.toBytes("val");
+ HRegion region = createRegion(tableName, family1Bytes, 0);
+ try {
+ region.initialize();
+
+ long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+ Put p = new Put(rowBytes);
+ p.add(family1Bytes, columnBytes, now - 10, valBytes);
+ region.put(p);
+
+ // issue a family delete with an explicit timestamp
+ Delete delete = new Delete(rowBytes, now);
+ delete.deleteFamily(family1Bytes, now - 5);
+ region.delete(delete);
+
+ // test that the delete marker preserved the timestamp
+ Scan scan = new Scan();
+ scan.setMaxVersions();
+ RegionScanner scanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(2, results.size());
+ // delete marker should appear first
+ Cell cell = results.get(0);
+ assertArrayEquals(new byte[0], cell.getQualifier());
+ assertArrayEquals(new byte[0], cell.getValue());
+ assertEquals(now - 5, cell.getTimestamp());
+ // since this is an unfiltered scan against the region, the original put should be next
+ cell = results.get(1);
+ assertArrayEquals(valBytes, cell.getValue());
+ assertEquals(now - 10, cell.getTimestamp());
+ scanner.close();
+
+
+ // with a filtered scan the original put should disappear
+ scan = new Scan();
+ scan.setMaxVersions();
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+
+ scanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(0, results.size());
+ scanner.close();
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testPreExistingData() throws Exception {
+ String tableName = "TestPreExistingData";
+ byte[] familyBytes = Bytes.toBytes("f");
+ long ttlMillis = TimeUnit.DAYS.toMillis(14);
+ HRegion region = createRegion(tableName, familyBytes, ttlMillis);
+ try {
+ region.initialize();
+
+ // timestamps for pre-existing, non-transactional data
+ long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
+ long older = now - ttlMillis / 2;
+ long newer = now - ttlMillis / 3;
+ // timestamps for transactional data
+ long nowTx = txVisibilityState.getVisibilityUpperBound();
+ long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
+ long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
+
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ ttls.put(familyBytes, ttlMillis);
+
+ List<Cell> cells = new ArrayList<>();
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
+
+ // Write non-transactional and transactional data
+ for (Cell c : cells) {
+ region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
+ }
+
+ Scan rawScan = new Scan();
+ rawScan.setMaxVersions();
+
+ Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
+ Scan txScan = new Scan();
+ txScan.setMaxVersions();
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // read all back with raw scanner
+ scanAndAssert(region, cells, rawScan);
+
+ // read all back with transaction filter
+ scanAndAssert(region, cells, txScan);
+
+ // force a flush to clear the memstore
+ region.flushcache();
+ scanAndAssert(region, cells, txScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ scanAndAssert(region, cells, txScan);
+
+ // Reduce TTL, this should make cells with timestamps older and olderTx expire
+ long newTtl = ttlMillis / 2 - 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // Raw scan should still give all cells
+ scanAndAssert(region, cells, rawScan);
+ // However, tx scan should not return expired cells
+ scanAndAssert(region, select(cells, 1, 3, 5), txScan);
+
+ region.flushcache();
+ scanAndAssert(region, cells, rawScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ // This time raw scan too should not return expired cells, as they would be dropped during major compaction
+ scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
+
+ // Reduce TTL again to 1 ms, this should expire all cells
+ newTtl = 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // force a major compaction to remove expired cells
+ region.compactStores(true);
+ // This time raw scan should not return any cells, as all cells have expired.
+ scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
+ } finally {
+ region.close();
+ }
+ }
+
+ private List<Cell> select(List<Cell> cells, int... indexes) {
+ List<Cell> newCells = new ArrayList<>();
+ for (int i : indexes) {
+ newCells.add(cells.get(i));
+ }
+ return newCells;
+ }
+
+ @SuppressWarnings("StatementWithEmptyBody")
+ private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
+ try (RegionScanner regionScanner = region.getScanner(scan)) {
+ List<Cell> results = Lists.newArrayList();
+ while (regionScanner.next(results)) { }
+ assertEquals(expected, results);
+ }
+ }
+
+ private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
+ region.close();
+ HTableDescriptor htd = region.getTableDesc();
+ HColumnDescriptor cfd = new HColumnDescriptor(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ htd.addFamily(cfd);
+ return HRegion.openHRegion(region.getRegionInfo(), htd, region.getLog(), conf,
+ new MockRegionServerServices(conf, null), null);
+ }
+
+ private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
+ HColumnDescriptor cfd = new HColumnDescriptor(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ htd.addFamily(cfd);
+ htd.addCoprocessor(TransactionProcessor.class.getName());
+ Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
+ Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
+ FileSystem fs = FileSystem.get(conf);
+ assertTrue(fs.mkdirs(tablePath));
+ HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
+ HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
+ HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
+ return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
+ byte[][] values = new byte[versions.length][];
+ for (int i = 0; i < versions.length; i++) {
+ values[i] = Bytes.toBytes(versions[i]);
+ }
+ assertKeyValueMatches(results, index, versions, values);
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
+ assertEquals(versions.length, results.size());
+ assertEquals(values.length, results.size());
+ for (int i = 0; i < versions.length; i++) {
+ Cell kv = results.get(i);
+ assertArrayEquals(Bytes.toBytes(index), kv.getRow());
+ assertEquals(versions[i], kv.getTimestamp());
+ assertArrayEquals(values[i], kv.getValue());
+ }
+ }
+
+ @Test
+ public void testTransactionStateCache() throws Exception {
+ TransactionStateCache cache = new TransactionStateCache();
+ cache.setConf(conf);
+ cache.startAndWait();
+ // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
+ TransactionVisibilityState cachedSnapshot = cache.getLatestState();
+ assertNotNull(cachedSnapshot);
+ assertEquals(invalidSet, cachedSnapshot.getInvalid());
+ cache.stopAndWait();
+ }
+
+ private static class MockRegionServerServices implements RegionServerServices {
+ private final Configuration hConf;
+ private final ZooKeeperWatcher zookeeper;
+ private final Map<String, HRegion> regions = new HashMap<String, HRegion>();
+ private boolean stopping = false;
+ private final ConcurrentSkipListMap<byte[], Boolean> rit = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
+ private HFileSystem hfs = null;
+ private ServerName serverName = null;
+ private RpcServerInterface rpcServer = null;
+ private volatile boolean abortRequested;
+
+
+ public MockRegionServerServices(Configuration hConf, ZooKeeperWatcher zookeeper) {
+ this.hConf = hConf;
+ this.zookeeper = zookeeper;
+ }
+
+ @Override
+ public boolean isStopping() {
+ return stopping;
+ }
+
+ @Override
+ public HLog getWAL(HRegionInfo regionInfo) throws IOException {
+ return null;
+ }
+
+ @Override
+ public CompactionRequestor getCompactionRequester() {
+ return null;
+ }
+
+ @Override
+ public FlushRequester getFlushRequester() {
+ return null;
+ }
+
+ @Override
+ public RegionServerAccounting getRegionServerAccounting() {
+ return null;
+ }
+
+ @Override
+ public TableLockManager getTableLockManager() {
+ return new TableLockManager.NullTableLockManager();
+ }
+
+ @Override
+ public void postOpenDeployTasks(HRegion r, CatalogTracker ct) throws KeeperException, IOException {
+ }
+
+ @Override
+ public RpcServerInterface getRpcServer() {
+ return rpcServer;
+ }
+
+ @Override
+ public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
+ return rit;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return hfs;
+ }
+
+ @Override
+ public Leases getLeases() {
+ return null;
+ }
+
+ @Override
+ public ExecutorService getExecutorService() {
+ return null;
+ }
+
+ @Override
+ public CatalogTracker getCatalogTracker() {
+ return null;
+ }
+
+ @Override
+ public Map<String, HRegion> getRecoveringRegions() {
+ return null;
+ }
+
+ @Override
+ public void updateRegionFavoredNodesMapping(String encodedRegionName, List<HBaseProtos.ServerName> favoredNodes) {
+ }
+
+ @Override
+ public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
+ return new InetSocketAddress[0];
+ }
+
+ @Override
+ public void addToOnlineRegions(HRegion r) {
+ regions.put(r.getRegionNameAsString(), r);
+ }
+
+ @Override
+ public boolean removeFromOnlineRegions(HRegion r, ServerName destination) {
+ return regions.remove(r.getRegionInfo().getEncodedName()) != null;
+ }
+
+ @Override
+ public HRegion getFromOnlineRegions(String encodedRegionName) {
+ return regions.get(encodedRegionName);
+ }
+
+ @Override
+ public List<HRegion> getOnlineRegions(TableName tableName) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hConf;
+ }
+
+ @Override
+ public ZooKeeperWatcher getZooKeeper() {
+ return zookeeper;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return serverName;
+ }
+
+ @Override
+ public void abort(String why, Throwable e) {
+ this.abortRequested = true;
+ }
+
+ @Override
+ public boolean isAborted() {
+ return abortRequested;
+ }
+
+ @Override
+ public void stop(String why) {
+ this.stopping = true;
+ }
+
+ @Override
+ public boolean isStopped() {
+ return stopping;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java
new file mode 100644
index 0000000..678ba5a
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96.coprocessor;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.hbase.AbstractTransactionVisibilityFilterTest;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * HBase 0.96 specific test for filtering logic applied when reading data transactionally.
+ */
+public class TransactionVisibilityFilterTest extends AbstractTransactionVisibilityFilterTest {
+ /**
+ * Test filtering of KeyValues for in-progress and invalid transactions.
+ * @throws Exception
+ */
+ @Test
+ public void testFiltering() throws Exception {
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+ }
+
+ @Test
+ public void testSubFilter() throws Exception {
+ final FilterBase includeFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE;
+ }
+ };
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter skipFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.SKIP;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ final Filter includeNextFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter nextColFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ }
+
+ @Test
+ public void testSubFilterOverride() throws Exception {
+ final FilterBase includeFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE;
+ }
+ };
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.INCLUDE));
+
+ final Filter skipFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.SKIP;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ final Filter includeNextFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter nextColFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ }
+
+ private void runFilteringTest(TxFilterFactory txFilterFactory,
+ List<Filter.ReturnCode> assertCodes) throws Exception {
+
+ /*
+ * Start and stop some transactions. This will give us a transaction state something like the following
+ * (numbers only reflect ordering, not actual transaction IDs):
+ * 6 - in progress
+ * 5 - committed
+ * 4 - invalid
+ * 3 - in-progress
+ * 2 - committed
+ * 1 - committed
+ *
+ * read ptr = 5
+ * write ptr = 6
+ */
+
+ Transaction tx1 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx1, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx1));
+
+ Transaction tx2 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx2, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx2));
+
+ Transaction tx3 = txManager.startShort();
+ Transaction tx4 = txManager.startShort();
+ txManager.invalidate(tx4.getTransactionId());
+
+ Transaction tx5 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx5, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx5));
+
+ Transaction tx6 = txManager.startShort();
+
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ Filter filter = txFilterFactory.getTxFilter(tx6, ttls);
+
+ assertEquals(assertCodes.get(5),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx6.getTransactionId())));
+ assertEquals(assertCodes.get(4),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx5.getTransactionId())));
+ assertEquals(assertCodes.get(3),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx4.getTransactionId())));
+ assertEquals(assertCodes.get(2),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx3.getTransactionId())));
+ assertEquals(assertCodes.get(1),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx2.getTransactionId())));
+ assertEquals(assertCodes.get(0),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx1.getTransactionId())));
+ }
+
+ /**
+ * Test filtering for TTL settings.
+ * @throws Exception
+ */
+ @Test
+ public void testTTLFiltering() throws Exception {
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ ttls.put(FAM, 10L);
+ ttls.put(FAM2, 30L);
+ ttls.put(FAM3, 0L);
+
+ Transaction tx = txManager.startShort();
+ long now = tx.getVisibilityUpperBound();
+ Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 21 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 1001 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
+
+ // Verify ttl for pre-existing, non-transactional data
+ long preNow = now / TxConstants.MAX_TX_PER_MS;
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 9L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 10L)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 11L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 9L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 10L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 1001L)));
+ }
+
+ protected KeyValue newKeyValue(String rowkey, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), FAM, COL, timestamp, Bytes.toBytes(value));
+ }
+
+ protected KeyValue newKeyValue(String rowkey, byte[] family, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), family, COL, timestamp, Bytes.toBytes(value));
+ }
+
+ private interface TxFilterFactory {
+ Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs);
+ }
+
+ private class CustomTxFilter extends TransactionVisibilityFilter {
+ public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
+ @Nullable Filter cellFilter) {
+ super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+ }
+
+ @Override
+ protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
+ switch (subFilterCode) {
+ case INCLUDE:
+ return ReturnCode.INCLUDE;
+ case INCLUDE_AND_NEXT_COL:
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ case SKIP:
+ return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
+ default:
+ return subFilterCode;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/HBase98ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/HBase98ConfigurationProvider.java b/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/HBase98ConfigurationProvider.java
deleted file mode 100644
index 7fb068b..0000000
--- a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/HBase98ConfigurationProvider.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98;
-
-import co.cask.tephra.util.ConfigurationProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-/**
- * HBase 0.98 version of {@link co.cask.tephra.util.ConfigurationProvider}.
- */
-public class HBase98ConfigurationProvider extends ConfigurationProvider {
- @Override
- public Configuration get() {
- return HBaseConfiguration.create();
- }
-
- @Override
- public Configuration get(Configuration baseConf) {
- return HBaseConfiguration.create(baseConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/SecondaryIndexTable.java b/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/SecondaryIndexTable.java
deleted file mode 100644
index 1dd4df8..0000000
--- a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/SecondaryIndexTable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase98;
-
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * A Transactional SecondaryIndexTable.
- */
-public class SecondaryIndexTable {
- private byte[] secondaryIndex;
- private TransactionAwareHTable transactionAwareHTable;
- private TransactionAwareHTable secondaryIndexTable;
- private TransactionContext transactionContext;
- private final TableName secondaryIndexTableName;
- private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
- private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
- private static final byte[] DELIMITER = new byte[] {0};
-
- public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
- byte[] secondaryIndex) {
- secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
- HTable secondaryIndexHTable = null;
- HBaseAdmin hBaseAdmin = null;
- try {
- hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
- if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
- hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
- }
- secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
- } catch (Exception e) {
- Throwables.propagate(e);
- } finally {
- try {
- hBaseAdmin.close();
- } catch (Exception e) {
- Throwables.propagate(e);
- }
- }
-
- this.secondaryIndex = secondaryIndex;
- this.transactionAwareHTable = new TransactionAwareHTable(hTable);
- this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
- this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
- secondaryIndexTable);
- }
-
- public Result get(Get get) throws IOException {
- return get(Collections.singletonList(get))[0];
- }
-
- public Result[] get(List<Get> gets) throws IOException {
- try {
- transactionContext.start();
- Result[] result = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return result;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public Result[] getByIndex(byte[] value) throws IOException {
- try {
- transactionContext.start();
- Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
- scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
- ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
-
- ArrayList<Get> gets = new ArrayList<Get>();
- for (Result result : indexScanner) {
- for (Cell cell : result.listCells()) {
- gets.add(new Get(cell.getValue()));
- }
- }
- Result[] results = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return results;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public void put(Put put) throws IOException {
- put(Collections.singletonList(put));
- }
-
-
- public void put(List<Put> puts) throws IOException {
- try {
- transactionContext.start();
- ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
- for (Put put : puts) {
- List<Put> indexPuts = new ArrayList<Put>();
- Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
- for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
- for (KeyValue value : family.getValue()) {
- if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
- secondaryIndex, 0, secondaryIndex.length)) {
- byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
- Bytes.add(value.getValue(), DELIMITER,
- value.getRow()));
- Put indexPut = new Put(secondaryRow);
- indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
- indexPuts.add(indexPut);
- }
- }
- }
- secondaryIndexPuts.addAll(indexPuts);
- }
- transactionAwareHTable.put(puts);
- secondaryIndexTable.put(secondaryIndexPuts);
- transactionContext.finish();
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/TransactionAwareHTable.java b/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/TransactionAwareHTable.java
deleted file mode 100644
index b9087ab..0000000
--- a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/TransactionAwareHTable.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase98;
-
-import co.cask.tephra.AbstractTransactionAwareTable;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TxConstants;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.Batch;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-
-/**
- * A Transaction Aware HTable implementation for HBase 0.98. Operations are committed as usual,
- * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
- * was started.
- */
-public class TransactionAwareHTable extends AbstractTransactionAwareTable
- implements HTableInterface, TransactionAware {
-
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
- private final HTableInterface hTable;
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- */
- public TransactionAwareHTable(HTableInterface hTable) {
- this(hTable, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
- this(hTable, conflictLevel, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
- this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
- boolean allowNonTransactional) {
- super(conflictLevel, allowNonTransactional);
- this.hTable = hTable;
- }
-
- /* AbstractTransactionAwareTable implementation */
-
- @Override
- protected byte[] getTableKey() {
- return getTableName();
- }
-
- @Override
- protected boolean doCommit() throws IOException {
- hTable.flushCommits();
- return true;
- }
-
- @Override
- protected boolean doRollback() throws Exception {
- try {
- // pre-size arraylist of deletes
- int size = 0;
- for (Set<ActionChange> cs : changeSets.values()) {
- size += cs.size();
- }
- List<Delete> rollbackDeletes = new ArrayList<>(size);
- for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
- long transactionTimestamp = entry.getKey();
- for (ActionChange change : entry.getValue()) {
- byte[] row = change.getRow();
- byte[] family = change.getFamily();
- byte[] qualifier = change.getQualifier();
- Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- switch (conflictLevel) {
- case ROW:
- case NONE:
- // issue family delete for the tx write pointer
- rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
- break;
- case COLUMN:
- if (family != null && qualifier == null) {
- rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
- } else if (family != null && qualifier != null) {
- rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
- }
- break;
- default:
- throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
- }
- rollbackDeletes.add(rollbackDelete);
- }
- }
- hTable.delete(rollbackDeletes);
- return true;
- } finally {
- try {
- hTable.flushCommits();
- } catch (Exception e) {
- LOG.error("Could not flush HTable commits", e);
- }
- tx = null;
- changeSets.clear();
- }
- }
-
- /* HTableInterface implementation */
-
- @Override
- public byte[] getTableName() {
- return hTable.getTableName();
- }
-
- @Override
- public TableName getName() {
- return hTable.getName();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hTable.getConfiguration();
- }
-
- @Override
- public HTableDescriptor getTableDescriptor() throws IOException {
- return hTable.getTableDescriptor();
- }
-
- @Override
- public boolean exists(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.exists(transactionalizeAction(get));
- }
-
- @Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.exists(transactionalizedGets);
- }
-
- @Override
- public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batch(transactionalizeActions(actions), results);
- }
-
- @Override
- public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batch(transactionalizeActions(actions));
- }
-
- @Override
- public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
- IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batchCallback(transactionalizeActions(actions), results, callback);
- }
-
- @Override
- public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
- InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batchCallback(transactionalizeActions(actions), callback);
- }
-
- @Override
- public Result get(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.get(transactionalizeAction(get));
- }
-
- @Override
- public Result[] get(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.get(transactionalizedGets);
- }
-
- @Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- if (allowNonTransactional) {
- return hTable.getRowOrBefore(row, family);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public ResultScanner getScanner(Scan scan) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addFamily(family);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addColumn(family, qualifier);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public void put(Put put) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Put txPut = transactionalizeAction(put);
- hTable.put(txPut);
- }
-
- @Override
- public void put(List<Put> puts) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
- for (Put put : puts) {
- Put txPut = transactionalizeAction(put);
- transactionalizedPuts.add(txPut);
- }
- hTable.put(transactionalizedPuts);
- }
-
- @Override
- public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndPut(row, family, qualifier, value, put);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public void delete(Delete delete) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.delete(transactionalizeAction(delete));
- }
-
- @Override
- public void delete(List<Delete> deletes) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
- for (Delete delete : deletes) {
- Delete txDelete = transactionalizeAction(delete);
- transactionalizedDeletes.add(txDelete);
- }
- hTable.delete(transactionalizedDeletes);
- }
-
- @Override
- public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndDelete(row, family, qualifier, value, delete);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
- CompareFilter.CompareOp compareOp, byte[] value, RowMutations rowMutations)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndMutate(row, family, qualifier, compareOp, value, rowMutations);
- }
-
- throw new UnsupportedOperationException("checkAndMutate operation is not supported transactionally");
- }
-
- @Override
- public void mutateRow(RowMutations rm) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- RowMutations transactionalMutations = new RowMutations();
- for (Mutation mutation : rm.getMutations()) {
- if (mutation instanceof Put) {
- transactionalMutations.add(transactionalizeAction((Put) mutation));
- } else if (mutation instanceof Delete) {
- transactionalMutations.add(transactionalizeAction((Delete) mutation));
- }
- }
- hTable.mutateRow(transactionalMutations);
- }
-
- @Override
- public Result append(Append append) throws IOException {
- if (allowNonTransactional) {
- return hTable.append(append);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public Result increment(Increment increment) throws IOException {
- if (allowNonTransactional) {
- return hTable.increment(increment);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean isAutoFlush() {
- return hTable.isAutoFlush();
- }
-
- @Override
- public void flushCommits() throws IOException {
- hTable.flushCommits();
- }
-
- @Override
- public void close() throws IOException {
- hTable.close();
- }
-
- @Override
- public CoprocessorRpcChannel coprocessorService(byte[] row) {
- return hTable.coprocessorService(row);
- }
-
- @Override
- public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable)
- throws ServiceException, Throwable {
- return hTable.coprocessorService(service, startKey, endKey, callable);
- }
-
- @Override
- public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable, Batch.Callback<R> callback)
- throws ServiceException, Throwable {
- hTable.coprocessorService(service, startKey, endKey, callable, callback);
- }
-
- @Override
- public <R extends Message> Map<byte[], R> batchCoprocessorService(
- MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey,
- R responsePrototype) throws ServiceException, Throwable {
- return hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
- }
-
- @Override
- public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
- Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
- throws ServiceException, Throwable {
- hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush) {
- setAutoFlushTo(autoFlush);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- hTable.setAutoFlush(autoFlush, clearBufferOnFail);
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- hTable.setAutoFlushTo(autoFlush);
- }
-
- @Override
- public long getWriteBufferSize() {
- return hTable.getWriteBufferSize();
- }
-
- @Override
- public void setWriteBufferSize(long writeBufferSize) throws IOException {
- hTable.setWriteBufferSize(writeBufferSize);
- }
-
- // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
-
- private Get transactionalizeAction(Get get) throws IOException {
- addToOperation(get, tx);
- return get;
- }
-
- private Scan transactionalizeAction(Scan scan) throws IOException {
- addToOperation(scan, tx);
- return scan;
- }
-
- private Put transactionalizeAction(Put put) throws IOException {
- Put txPut = new Put(put.getRow(), tx.getWritePointer());
- Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
- if (!familyMap.isEmpty()) {
- for (Map.Entry<byte[], List<Cell>> family : familyMap) {
- List<Cell> familyValues = family.getValue();
- if (!familyValues.isEmpty()) {
- for (Cell value : familyValues) {
- txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
- addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
- txPut.setAttribute(entry.getKey(), entry.getValue());
- }
- txPut.setDurability(put.getDurability());
- addToOperation(txPut, tx);
- return txPut;
- }
-
- private Delete transactionalizeAction(Delete delete) throws IOException {
- long transactionTimestamp = tx.getWritePointer();
-
- byte[] deleteRow = delete.getRow();
- Delete txDelete = new Delete(deleteRow, transactionTimestamp);
-
- Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
- if (familyToDelete.isEmpty()) {
- // perform a row delete if we are using row-level conflict detection
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // Row delete leaves delete markers in all column families of the table
- // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
- for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
- // no need to identify individual columns deleted
- addToChangeSet(deleteRow, columnDescriptor.getName(), null);
- }
- } else {
- Result result = get(new Get(delete.getRow()));
- // Delete everything
- NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
- for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
- }
- }
- }
- } else {
- for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
- byte[] family = familyEntry.getKey();
- List<Cell> entries = familyEntry.getValue();
- boolean isFamilyDelete = false;
- if (entries.size() == 1) {
- Cell cell = entries.get(0);
- isFamilyDelete = CellUtil.isDeleteFamily(cell);
- }
- if (isFamilyDelete) {
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // no need to identify individual columns deleted
- txDelete.deleteFamily(family);
- addToChangeSet(deleteRow, family, null);
- } else {
- Result result = get(new Get(delete.getRow()).addFamily(family));
- // Delete entire family
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, family, column.getKey());
- }
- }
- } else {
- for (Cell value : entries) {
- txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
- addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- txDelete.setAttribute(entry.getKey(), entry.getValue());
- }
- txDelete.setDurability(delete.getDurability());
- return txDelete;
- }
-
- private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
- List<Row> transactionalizedActions = new ArrayList<>(actions.size());
- for (Row action : actions) {
- if (action instanceof Get) {
- transactionalizedActions.add(transactionalizeAction((Get) action));
- } else if (action instanceof Put) {
- transactionalizedActions.add(transactionalizeAction((Put) action));
- } else if (action instanceof Delete) {
- transactionalizedActions.add(transactionalizeAction((Delete) action));
- } else {
- transactionalizedActions.add(action);
- }
- }
- return transactionalizedActions;
- }
-
- public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
- op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilter.java
deleted file mode 100644
index dcf29d6..0000000
--- a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilter.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
- * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
- * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
- * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
- * Please see TEPHRA-169 for more details.
- */
-public class CellSkipFilter extends FilterBase {
- private final Filter filter;
- // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
- private KeyValue skipColumn = null;
-
- public CellSkipFilter(Filter filter) {
- this.filter = filter;
- }
-
- /**
- * Determines whether the current cell should be skipped. The cell will be skipped
- * if the previous keyvalue had the same key as the current cell. This means filter already responded
- * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
- * @param cell the {@link Cell} to be tested for skipping
- * @return true is current cell should be skipped, false otherwise
- */
- private boolean skipCellVersion(Cell cell) {
- return skipColumn != null
- && skipColumn.matchingRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
- && skipColumn.matchingColumn(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (skipCellVersion(cell)) {
- return ReturnCode.NEXT_COL;
- }
-
- ReturnCode code = filter.filterKeyValue(cell);
- if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
- // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
- skipColumn = KeyValue.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
- cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(),
- cell.getQualifierLength());
- } else {
- skipColumn = null;
- }
- return code;
- }
-
- @Override
- public boolean filterRow() throws IOException {
- return filter.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- return filter.transformCell(cell);
- }
-
- @Override
- public void reset() throws IOException {
- filter.reset();
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- return filter.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- return filter.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- filter.filterRowCells(kvs);
- }
-
- @Override
- public boolean hasFilterRow() {
- return filter.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- return filter.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- return filter.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- return filter.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return filter.toByteArray();
- }
-}
-
[10/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/TransactionAwareHTableTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/TransactionAwareHTableTest.java
deleted file mode 100644
index 1725a40..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/TransactionAwareHTableTest.java
+++ /dev/null
@@ -1,1552 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase10cdh;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionConflictException;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase10cdh.coprocessor.TransactionProcessor;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.LongComparator;
-import org.apache.hadoop.hbase.filter.ValueFilter;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Tests for TransactionAwareHTables.
- */
-public class TransactionAwareHTableTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
-
- private static HBaseTestingUtility testUtil;
- private static HBaseAdmin hBaseAdmin;
- private static TransactionStateStorage txStateStorage;
- private static TransactionManager txManager;
- private static Configuration conf;
- private TransactionContext transactionContext;
- private TransactionAwareHTable transactionAwareHTable;
- private HTable hTable;
-
- private static final class TestBytes {
- private static final byte[] table = Bytes.toBytes("testtable");
- private static final byte[] family = Bytes.toBytes("f1");
- private static final byte[] family2 = Bytes.toBytes("f2");
- private static final byte[] qualifier = Bytes.toBytes("col1");
- private static final byte[] qualifier2 = Bytes.toBytes("col2");
- private static final byte[] row = Bytes.toBytes("row");
- private static final byte[] row2 = Bytes.toBytes("row2");
- private static final byte[] row3 = Bytes.toBytes("row3");
- private static final byte[] row4 = Bytes.toBytes("row4");
- private static final byte[] value = Bytes.toBytes("value");
- private static final byte[] value2 = Bytes.toBytes("value2");
- private static final byte[] value3 = Bytes.toBytes("value3");
- }
-
- private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
-
- public static class TestRegionObserver extends BaseRegionObserver {
- @Override
- public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Put put, final WALEdit edit,
- final Durability durability) throws IOException {
- if (put.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Put should preserve attributes");
- }
- if (put.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
-
- @Override
- public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Delete delete, final WALEdit edit,
- final Durability durability) throws IOException {
- if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Delete should preserve attributes");
- }
- if (delete.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
- }
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- testUtil = new HBaseTestingUtility();
- conf = testUtil.getConfiguration();
-
- // Tune down the connection thread pool size
- conf.setInt("hbase.hconnection.threads.core", 5);
- conf.setInt("hbase.hconnection.threads.max", 10);
- // Tunn down handler threads in regionserver
- conf.setInt("hbase.regionserver.handler.count", 10);
-
- // Set to random port
- conf.setInt("hbase.master.port", 0);
- conf.setInt("hbase.master.info.port", 0);
- conf.setInt("hbase.regionserver.port", 0);
- conf.setInt("hbase.regionserver.info.port", 0);
-
- testUtil.startMiniCluster();
- hBaseAdmin = testUtil.getHBaseAdmin();
- txStateStorage = new InMemoryTransactionStateStorage();
- txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
- txManager.startAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- testUtil.shutdownMiniCluster();
- hBaseAdmin.close();
- }
-
- @Before
- public void setupBeforeTest() throws Exception {
- hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
- transactionAwareHTable = new TransactionAwareHTable(hTable);
- transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
- }
-
- @After
- public void shutdownAfterTest() throws IOException {
- hBaseAdmin.disableTable(TestBytes.table);
- hBaseAdmin.deleteTable(TestBytes.table);
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
- return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
- List<String> coprocessors) throws Exception {
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
- for (byte[] family : columnFamilies) {
- HColumnDescriptor columnDesc = new HColumnDescriptor(family);
- columnDesc.setMaxVersions(Integer.MAX_VALUE);
- columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
- desc.addFamily(columnDesc);
- }
- if (existingData) {
- desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
- }
- // Divide individually to prevent any overflow
- int priority = Coprocessor.PRIORITY_USER;
- desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
- // order in list is the same order that coprocessors will be invoked
- for (String coprocessor : coprocessors) {
- desc.addCoprocessor(coprocessor, null, ++priority, null);
- }
- hBaseAdmin.createTable(desc);
- testUtil.waitTableAvailable(tableName, 5000);
- return new HTable(testUtil.getConfiguration(), tableName);
- }
-
- /**
- * Test transactional put and get requests.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
-
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- /**
- * Test aborted put requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
-
- transactionContext.abort();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(value, null);
- }
-
- /**
- * Test transactional delete operations.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
- new byte[][]{TestBytes.family, TestBytes.family2});
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
-
- // test column delete
- // load 10 rows
- txContext.start();
- int rowCount = 10;
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
- }
- txTable.put(p);
- }
- txContext.finish();
-
- // verify loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertFalse(r.isEmpty());
- for (int j = 0; j < 10; j++) {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
- }
- }
- txContext.finish();
-
- // delete odds columns from odd rows and even columns from even rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- if (i % 2 == j % 2) {
- LOG.info("Deleting row={}, column={}", i, j);
- d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
- }
- }
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted columns
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertEquals(5, r.size());
- for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
- int col = Bytes.toInt(entry.getKey());
- LOG.info("Got row={}, col={}", i, col);
- // each row should only have the opposite mod (odd=even, even=odd)
- assertNotEquals(i % 2, col % 2);
- assertArrayEquals(TestBytes.value, entry.getValue());
- }
- }
- txContext.finish();
-
- // test family delete
- // load 10 rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("famrow" + i));
- p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
- txTable.put(p);
- }
- txContext.finish();
-
- // verify all loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(2, r.size());
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- txContext.finish();
-
- // delete family1 for even rows, family2 for odd rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("famrow" + i));
- d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted families
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(1, r.size());
- if (i % 2 == 0) {
- assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- } else {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- }
- txContext.finish();
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test that put and delete attributes are preserved
- *
- * @throws Exception
- */
- @Test
- public void testAttributesPreserved() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
- new byte[][]{TestBytes.family, TestBytes.family2}, false,
- Lists.newArrayList(TestRegionObserver.class.getName()));
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- // set an attribute on the put, TestRegionObserver will verify it still exists
- put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete, TestRegionObserver will verify it still exists
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test aborted transactional delete requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionalDelete() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
-
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row);
- transactionAwareHTable.delete(delete);
- transactionContext.abort();
-
- transactionContext.start();
- result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
- String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
- HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- // Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- // Start a tx, delete a column family and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
- txContext.abort();
-
- // Above operations should have no effect on the row, since they were aborted
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
- }
- }
-
- @Test
- public void testDeleteRollback() throws Exception {
- testDeleteRollback(TxConstants.ConflictDetection.ROW);
- testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
- testDeleteRollback(TxConstants.ConflictDetection.NONE);
- }
-
- @Test
- public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- txContext.start();
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- Result result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
-
- //Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- //Start a tx and scan all the col families to make sure none of them have delete markers
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
- }
- }
-
- @Test
- public void testRowDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][]{TestBytes.family, TestBytes.family2});
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
- try {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Test 1: full row delete
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // delete entire row
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.finish();
-
- // verify row is now empty
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertTrue(result.isEmpty());
-
- // verify row is empty for explicit column retrieval
- result = txTable.get(new Get(TestBytes.row)
- .addColumn(TestBytes.family, TestBytes.qualifier)
- .addFamily(TestBytes.family2));
- assertTrue(result.isEmpty());
-
- // verify row is empty for scan
- ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
- assertNull(scanner.next());
- scanner.close();
-
- // verify row is empty for scan with explicit column
- scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
- assertNull(scanner.next());
- scanner.close();
- txContext.finish();
-
- // write swapped values to one column per family
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
- txContext.finish();
-
- // verify new values appear
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertFalse(result.isEmpty());
- assertEquals(2, result.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
-
- scanner = txTable.getScanner(new Scan(TestBytes.row));
- Result result1 = scanner.next();
- assertNotNull(result1);
- assertFalse(result1.isEmpty());
- assertEquals(2, result1.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- scanner.close();
- txContext.finish();
-
- // Test 2: delete of first column family
- txContext.start();
- txTable.put(new Put(TestBytes.row2)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
- txContext.finish();
-
- txContext.start();
- Result fam1Result = txTable.get(new Get(TestBytes.row2));
- assertFalse(fam1Result.isEmpty());
- assertEquals(2, fam1Result.size());
- assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 3: delete of second column family
- txContext.start();
- txTable.put(new Put(TestBytes.row3)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
- txContext.finish();
-
- txContext.start();
- Result fam2Result = txTable.get(new Get(TestBytes.row3));
- assertFalse(fam2Result.isEmpty());
- assertEquals(2, fam2Result.size());
- assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 4: delete specific rows in a range
- txContext.start();
- for (int i = 0; i < 10; i++) {
- txTable.put(new Put(Bytes.toBytes("z" + i))
- .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
- .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
- }
- txContext.finish();
-
- txContext.start();
- // delete odd rows
- for (int i = 1; i < 10; i += 2) {
- txTable.delete(new Delete(Bytes.toBytes("z" + i)));
- }
- txContext.finish();
-
- txContext.start();
- int cnt = 0;
- ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
- Result res;
- while ((res = zScanner.next()) != null) {
- assertFalse(res.isEmpty());
- assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
- cnt += 2;
- }
-
- // Test 5: delete prior writes in the same transaction
- txContext.start();
- txTable.put(new Put(TestBytes.row4)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txTable.delete(new Delete(TestBytes.row4));
- txContext.finish();
-
- txContext.start();
- Result row4Result = txTable.get(new Get(TestBytes.row4));
- assertTrue(row4Result.isEmpty());
- txContext.finish();
- } finally {
- txTable.close();
- }
- }
-
- /**
- * Expect an exception since a transaction hasn't been started.
- *
- * @throws Exception
- */
- @Test(expected = IOException.class)
- public void testTransactionlessFailure() throws Exception {
- transactionAwareHTable.get(new Get(TestBytes.row));
- }
-
- /**
- * Tests that each transaction can see its own persisted writes, while not seeing writes from other
- * in-progress transactions.
- */
- @Test
- public void testReadYourWrites() throws Exception {
- // In-progress tx1: started before our main transaction
- HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
- TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
-
- // In-progress tx2: started while our main transaction is running
- HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
- TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
-
- // create an in-progress write that should be ignored
- byte[] col2 = Bytes.toBytes("col2");
- inprogressTxContext1.start();
- Put putCol2 = new Put(TestBytes.row);
- byte[] valueCol2 = Bytes.toBytes("writing in progress");
- putCol2.add(TestBytes.family, col2, valueCol2);
- txHTable1.put(putCol2);
-
- // start a tx and write a value to test reading in same tx
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] value = Bytes.toBytes("writing");
- put.add(TestBytes.family, TestBytes.qualifier, value);
- transactionAwareHTable.put(put);
-
- // test that a write from a tx started after the first is not visible
- inprogressTxContext2.start();
- Put put2 = new Put(TestBytes.row);
- byte[] value2 = Bytes.toBytes("writing2");
- put2.add(TestBytes.family, TestBytes.qualifier, value2);
- txHTable2.put(put2);
-
- Get get = new Get(TestBytes.row);
- Result row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
- Assert.assertNotNull(col1Value);
- Assert.assertArrayEquals(value, col1Value);
- // write from in-progress transaction should not be visible
- byte[] col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- // commit in-progress transaction, should still not be visible
- inprogressTxContext1.finish();
-
- get = new Get(TestBytes.row);
- row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- transactionContext.finish();
-
- inprogressTxContext2.abort();
- }
-
- @Test
- public void testRowLevelConflictDetection() throws Exception {
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- byte[] row1 = Bytes.toBytes("row1");
- byte[] row2 = Bytes.toBytes("row2");
- byte[] col1 = Bytes.toBytes("c1");
- byte[] col2 = Bytes.toBytes("c2");
- byte[] val1 = Bytes.toBytes("val1");
- byte[] val2 = Bytes.toBytes("val2");
-
- // test that concurrent writing to different rows succeeds
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
-
- // should be no conflicts
- txContext1.finish();
- txContext2.finish();
-
- transactionContext.start();
- Result res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
-
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // test that writing to different columns in the same row fails
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
-
- txContext2.start();
- txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row-level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- // should now be val2
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
-
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- // col2 should not be visible due to conflict
- assertNull(cell);
- transactionContext.finish();
-
- // test that writing to the same column in the same row fails
- txContext1.start();
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row and column level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- assertNotNull(cell);
- // should now be val1
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // verify change set that is being reported only on rows
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- Collection<byte[]> changeSet = txTable1.getTxChanges();
- assertNotNull(changeSet);
- assertEquals(2, changeSet.size());
- assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
- assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
- txContext1.finish();
- }
-
- @Test
- public void testNoneLevelConflictDetection() throws Exception {
- InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
-
- // overlapping writes to the same row + column should not conflict
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // changes should not be visible yet
- txContext2.start();
- Result row = txTable2.get(new Get(TestBytes.row));
- assertTrue(row.isEmpty());
-
- txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // both commits should succeed
- txContext1.finish();
- txContext2.finish();
-
- txContext1.start();
- row = txTable1.get(new Get(TestBytes.row));
- assertFalse(row.isEmpty());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
- txContext1.finish();
-
- // transaction abort should still rollback changes
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext1.abort();
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row2 = txTable2.get(new Get(TestBytes.row2));
- assertTrue(row2.isEmpty());
- txContext2.finish();
-
- // transaction invalidate should still make changes invisible
-
- txContext1.start();
- Transaction tx1 = txContext1.getCurrentTransaction();
- txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- assertNotNull(tx1);
- txClient.invalidate(tx1.getWritePointer());
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row3 = txTable2.get(new Get(TestBytes.row3));
- assertTrue(row3.isEmpty());
- txContext2.finish();
- }
-
- @Test
- public void testCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionContext.checkpoint();
- Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
- assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
- long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
- assertEquals(1, checkpointPtrs.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
-
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
- assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
- long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
- assertEquals(2, checkpointPtrs2.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
- assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
-
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // by default, all rows should be visible with Read-Your-Writes
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
-
- // when disabling current write pointer, only the previous checkpoints should be visible
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- Get get = new Get(TestBytes.row);
- verifyRow(transactionAwareHTable, get, TestBytes.value);
- get = new Get(TestBytes.row2);
- verifyRow(transactionAwareHTable, get, TestBytes.value2);
- get = new Get(TestBytes.row3);
- verifyRow(transactionAwareHTable, get, null);
-
- // test scan results excluding current write pointer
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
-
- Result row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row2, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNull(row);
- scanner.close();
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
-
- // commit transaction, verify writes are visible
- transactionContext.finish();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
- transactionContext.finish();
- }
-
- @Test
- public void testInProgressCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // check that writes are still not visible to other clients
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- txContext2.start();
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- txContext2.finish();
- txTable2.close();
-
- transactionContext.finish();
-
- // verify writes are visible after commit
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointRollback() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- transactionContext.abort();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, null);
- verifyRow(transactionAwareHTable, TestBytes.row2, null);
- verifyRow(transactionAwareHTable, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointInvalidate() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
- txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
-
- // check that writes are not visible
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
- txContext2.start();
- Transaction newTx = txContext2.getCurrentTransaction();
-
- // all 3 writes pointers from the previous transaction should now be excluded
- assertTrue(newTx.isExcluded(origTx.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
-
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- verifyRow(txTable2, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = txTable2.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- txContext2.finish();
- }
-
- @Test
- public void testExistingData() throws Exception {
- byte[] val11 = Bytes.toBytes("val11");
- byte[] val12 = Bytes.toBytes("val12");
- byte[] val21 = Bytes.toBytes("val21");
- byte[] val22 = Bytes.toBytes("val22");
- byte[] val31 = Bytes.toBytes("val31");
- byte[] val111 = Bytes.toBytes("val111");
-
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
- Collections.<String>emptyList()));
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Add some pre-existing, non-transactional data
- HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
- HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.flushCommits();
-
- // Add transactional data
- txContext.start();
- txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
- txContext.finish();
-
- txContext.start();
- // test get
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
-
- // test scan
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
-
- // test update and delete
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
- txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
-
- txContext.start();
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
- txContext.finish();
-
- // test scan
- txContext.start();
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
- }
-
- private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
- verifyRow(table, new Get(rowkey), expectedValue);
- }
-
- private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
- verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
- }
-
- private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
- Result result = table.get(get);
- if (expectedValues == null) {
- assertTrue(result.isEmpty());
- } else {
- assertFalse(result.isEmpty());
- byte[] family = TestBytes.family;
- byte[] col = TestBytes.qualifier;
- if (get.hasFamilies()) {
- family = get.getFamilyMap().keySet().iterator().next();
- col = get.getFamilyMap().get(family).first();
- }
- Iterator<Cell> it = result.getColumnCells(family, col).iterator();
- for (byte[] expectedValue : expectedValues) {
- Assert.assertTrue(it.hasNext());
- assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
- }
- }
- }
-
- private Cell[] getRow(HTableInterface table, Get get) throws Exception {
- Result result = table.get(get);
- return result.rawCells();
- }
-
- private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
- List<Cell> actualCells = new ArrayList<>();
- try (ResultScanner scanner = table.getScanner(scan)) {
- Result[] results = scanner.next(expectedCells.size() + 1);
- for (Result result : results) {
- actualCells.addAll(Lists.newArrayList(result.rawCells()));
- }
- Assert.assertEquals(expectedCells, actualCells);
- }
- }
-
- @Test
- public void testVisibilityAll() throws Exception {
- HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
- new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(nonTxTable,
- TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // start a transaction and create a delete marker
- txContext.start();
- //noinspection ConstantConditions
- long txWp0 = txContext.getCurrentTransaction().getWritePointer();
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // start a new transaction and write some values
- txContext.start();
- @SuppressWarnings("ConstantConditions")
- long txWp1 = txContext.getCurrentTransaction().getWritePointer();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
-
- // verify written data
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- TestBytes.value2);
-
- // checkpoint and make changes to written data now
- txContext.checkpoint();
- long txWp2 = txContext.getCurrentTransaction().getWritePointer();
- // delete a column
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
- // no change to a column
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- // update a column
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
- // delete column family
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
-
- // verify changed values
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- null);
-
- // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
- // and the raw change by prior transaction
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- List<KeyValue> expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // verify a Get is also able to return all snapshot versions
- Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
- Cell[] cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(3, cells.length);
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
- Assert.assertTrue(CellUtil.isDeleteColumns(cells[2]));
-
- verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- ImmutableList.of(TestBytes.value3, TestBytes.value));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
-
- // Verify VisibilityLevel.SNAPSHOT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
- txContext.finish();
-
- // finally verify values once more after commit, this time we should get only committed raw values for
- // all visibility levels
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- txContext.finish();
-
- // Test with regular HBase deletes in pre-existing data
- long now = System.currentTimeMillis();
- Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
- // to prevent Tephra from replacing delete with delete marker
- deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteColumn);
- Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
- // to prevent Tephra from replacing delete with delete marker
- deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteFamily);
- nonTxTable.flushCommits();
-
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
- new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
- );
- // test scan
- Scan scan = new Scan();
- scan.setRaw(true);
- verifyScan(txTable, scan, expected);
- txContext.finish();
- }
-
- @Test
- public void testFilters() throws Exception {
- // Add some values to table
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] val1 = Bytes.toBytes(1L);
- put.add(TestBytes.family, TestBytes.qualifier, val1);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row2);
- byte[] val2 = Bytes.toBytes(2L);
- put.add(TestBytes.family, TestBytes.qualifier, val2);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row3);
- byte[] val3 = Bytes.toBytes(3L);
- put.add(TestBytes.family, TestBytes.qualifier, val3);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row4);
- byte[] val4 = Bytes.toBytes(4L);
- put.add(TestBytes.family, TestBytes.qualifier, val4);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Delete cell with value 2
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row2);
- delete.addColumn(TestBytes.family, TestBytes.qualifier);
- transactionAwareHTable.delete(delete);
- transactionContext.finish();
-
- // Scan for values less than 4, should get only values 1 and 3
- transactionContext.start();
- Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(4)));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run a Get with a filter for less than 10 on row4, should get value 4
- transactionContext.start();
- Get get = new Get(TestBytes.row4);
- get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
- Result result = transactionAwareHTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
- transactionContext.finish();
-
- // Change value of row4 to 40
- transactionContext.start();
- put = new Put(TestBytes.row4);
- byte[] val40 = Bytes.toBytes(40L);
- put.add(TestBytes.family, TestBytes.qualifier, val40);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Scan for values less than 10, should get only values 1 and 3
- transactionContext.start();
- scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run the Get again with a filter for less than 10 on row4, this time should not get any results
- transactionContext.start();
- result = transactionAwareHTable.get(get);
- assertTrue(result.isEmpty());
- transactionContext.finish();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java
deleted file mode 100644
index bc34f43..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * HBase 1.0-cdh specific test for testing {@link CellSkipFilter}.
- */
-public class CellSkipFilterTest {
-
- private static final String ROW1KEY = "row1";
- private static final String ROW2KEY = "row2";
- private static final String FAM1KEY = "fam1";
- private static final String COL1KEY = "col1";
- private static final String FAM2KEY = "fam2";
- private static final String COL2KEY = "col2";
- private static final String VALUE = "value";
-
- @Test
- public void testSkipFiltering() throws Exception {
- long timestamp = System.currentTimeMillis();
- // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
- Filter filter = new CellSkipFilter(new MyFilter(0));
- assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 1)));
-
- // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give SKIP from MyFilter
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
- timestamp - 4)));
-
- // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
- filter = new CellSkipFilter(new MyFilter(2));
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 1)));
-
- // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give NEXT_ROW from MyFilter
- assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
- timestamp - 4)));
-
- // Next call with the new key should returned the SEEK_NEXT_USING_HINT
- assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 5)));
- }
-
- private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
- timestamp, Bytes.toBytes(value));
- }
-
- /**
- * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
- * start index in the list and start serving the return codes corresponding that that index. Every time the
- * return code is served, index is incremented.
- */
- class MyFilter extends FilterBase {
-
- private final List<ReturnCode> returnCodes;
- private int counter;
-
- public MyFilter(int startIndex) {
- returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
- ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
- counter = startIndex;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- ReturnCode code = returnCodes.get(counter % returnCodes.size());
- counter++;
- return code;
- }
- }
-}
[08/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java
new file mode 100644
index 0000000..d31b972
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java
@@ -0,0 +1,1552 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase10cdh;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.LongComparator;
+import org.apache.hadoop.hbase.filter.ValueFilter;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionConflictException;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.hbase10cdh.coprocessor.TransactionProcessor;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests for TransactionAwareHTables.
+ */
+public class TransactionAwareHTableTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
+
+ private static HBaseTestingUtility testUtil;
+ private static HBaseAdmin hBaseAdmin;
+ private static TransactionStateStorage txStateStorage;
+ private static TransactionManager txManager;
+ private static Configuration conf;
+ private TransactionContext transactionContext;
+ private TransactionAwareHTable transactionAwareHTable;
+ private HTable hTable;
+
+ private static final class TestBytes {
+ private static final byte[] table = Bytes.toBytes("testtable");
+ private static final byte[] family = Bytes.toBytes("f1");
+ private static final byte[] family2 = Bytes.toBytes("f2");
+ private static final byte[] qualifier = Bytes.toBytes("col1");
+ private static final byte[] qualifier2 = Bytes.toBytes("col2");
+ private static final byte[] row = Bytes.toBytes("row");
+ private static final byte[] row2 = Bytes.toBytes("row2");
+ private static final byte[] row3 = Bytes.toBytes("row3");
+ private static final byte[] row4 = Bytes.toBytes("row4");
+ private static final byte[] value = Bytes.toBytes("value");
+ private static final byte[] value2 = Bytes.toBytes("value2");
+ private static final byte[] value3 = Bytes.toBytes("value3");
+ }
+
+ private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
+
+ public static class TestRegionObserver extends BaseRegionObserver {
+ @Override
+ public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Put put, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (put.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Put should preserve attributes");
+ }
+ if (put.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+
+ @Override
+ public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Delete delete, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Delete should preserve attributes");
+ }
+ if (delete.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ conf = testUtil.getConfiguration();
+
+ // Tune down the connection thread pool size
+ conf.setInt("hbase.hconnection.threads.core", 5);
+ conf.setInt("hbase.hconnection.threads.max", 10);
+ // Tunn down handler threads in regionserver
+ conf.setInt("hbase.regionserver.handler.count", 10);
+
+ // Set to random port
+ conf.setInt("hbase.master.port", 0);
+ conf.setInt("hbase.master.info.port", 0);
+ conf.setInt("hbase.regionserver.port", 0);
+ conf.setInt("hbase.regionserver.info.port", 0);
+
+ testUtil.startMiniCluster();
+ hBaseAdmin = testUtil.getHBaseAdmin();
+ txStateStorage = new InMemoryTransactionStateStorage();
+ txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
+ txManager.startAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ testUtil.shutdownMiniCluster();
+ hBaseAdmin.close();
+ }
+
+ @Before
+ public void setupBeforeTest() throws Exception {
+ hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
+ transactionAwareHTable = new TransactionAwareHTable(hTable);
+ transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
+ }
+
+ @After
+ public void shutdownAfterTest() throws IOException {
+ hBaseAdmin.disableTable(TestBytes.table);
+ hBaseAdmin.deleteTable(TestBytes.table);
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
+ return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
+ List<String> coprocessors) throws Exception {
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+ for (byte[] family : columnFamilies) {
+ HColumnDescriptor columnDesc = new HColumnDescriptor(family);
+ columnDesc.setMaxVersions(Integer.MAX_VALUE);
+ columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
+ desc.addFamily(columnDesc);
+ }
+ if (existingData) {
+ desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
+ }
+ // Divide individually to prevent any overflow
+ int priority = Coprocessor.PRIORITY_USER;
+ desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
+ // order in list is the same order that coprocessors will be invoked
+ for (String coprocessor : coprocessors) {
+ desc.addCoprocessor(coprocessor, null, ++priority, null);
+ }
+ hBaseAdmin.createTable(desc);
+ testUtil.waitTableAvailable(tableName, 5000);
+ return new HTable(testUtil.getConfiguration(), tableName);
+ }
+
+ /**
+ * Test transactional put and get requests.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Test aborted put requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(value, null);
+ }
+
+ /**
+ * Test transactional delete operations.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
+ new byte[][]{TestBytes.family, TestBytes.family2});
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+
+ // test column delete
+ // load 10 rows
+ txContext.start();
+ int rowCount = 10;
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
+ }
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertFalse(r.isEmpty());
+ for (int j = 0; j < 10; j++) {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
+ }
+ }
+ txContext.finish();
+
+ // delete odds columns from odd rows and even columns from even rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ if (i % 2 == j % 2) {
+ LOG.info("Deleting row={}, column={}", i, j);
+ d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
+ }
+ }
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted columns
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertEquals(5, r.size());
+ for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
+ int col = Bytes.toInt(entry.getKey());
+ LOG.info("Got row={}, col={}", i, col);
+ // each row should only have the opposite mod (odd=even, even=odd)
+ assertNotEquals(i % 2, col % 2);
+ assertArrayEquals(TestBytes.value, entry.getValue());
+ }
+ }
+ txContext.finish();
+
+ // test family delete
+ // load 10 rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("famrow" + i));
+ p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify all loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(2, r.size());
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ txContext.finish();
+
+ // delete family1 for even rows, family2 for odd rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("famrow" + i));
+ d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted families
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(1, r.size());
+ if (i % 2 == 0) {
+ assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ } else {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ }
+ txContext.finish();
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test that put and delete attributes are preserved
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAttributesPreserved() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, false,
+ Lists.newArrayList(TestRegionObserver.class.getName()));
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ // set an attribute on the put, TestRegionObserver will verify it still exists
+ put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete, TestRegionObserver will verify it still exists
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test aborted transactional delete requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionalDelete() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ transactionAwareHTable.delete(delete);
+ transactionContext.abort();
+
+ transactionContext.start();
+ result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
+ String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
+ HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ // Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ // Start a tx, delete a column family and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
+ txContext.abort();
+
+ // Above operations should have no effect on the row, since they were aborted
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testDeleteRollback() throws Exception {
+ testDeleteRollback(TxConstants.ConflictDetection.ROW);
+ testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
+ testDeleteRollback(TxConstants.ConflictDetection.NONE);
+ }
+
+ @Test
+ public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ txContext.start();
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ Result result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+
+ //Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ //Start a tx and scan all the col families to make sure none of them have delete markers
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testRowDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][]{TestBytes.family, TestBytes.family2});
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
+ try {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Test 1: full row delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // delete entire row
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.finish();
+
+ // verify row is now empty
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for explicit column retrieval
+ result = txTable.get(new Get(TestBytes.row)
+ .addColumn(TestBytes.family, TestBytes.qualifier)
+ .addFamily(TestBytes.family2));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for scan
+ ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
+ assertNull(scanner.next());
+ scanner.close();
+
+ // verify row is empty for scan with explicit column
+ scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
+ assertNull(scanner.next());
+ scanner.close();
+ txContext.finish();
+
+ // write swapped values to one column per family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
+ txContext.finish();
+
+ // verify new values appear
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertFalse(result.isEmpty());
+ assertEquals(2, result.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+
+ scanner = txTable.getScanner(new Scan(TestBytes.row));
+ Result result1 = scanner.next();
+ assertNotNull(result1);
+ assertFalse(result1.isEmpty());
+ assertEquals(2, result1.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ scanner.close();
+ txContext.finish();
+
+ // Test 2: delete of first column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row2)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
+ txContext.finish();
+
+ txContext.start();
+ Result fam1Result = txTable.get(new Get(TestBytes.row2));
+ assertFalse(fam1Result.isEmpty());
+ assertEquals(2, fam1Result.size());
+ assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 3: delete of second column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
+ txContext.finish();
+
+ txContext.start();
+ Result fam2Result = txTable.get(new Get(TestBytes.row3));
+ assertFalse(fam2Result.isEmpty());
+ assertEquals(2, fam2Result.size());
+ assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 4: delete specific rows in a range
+ txContext.start();
+ for (int i = 0; i < 10; i++) {
+ txTable.put(new Put(Bytes.toBytes("z" + i))
+ .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
+ .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ // delete odd rows
+ for (int i = 1; i < 10; i += 2) {
+ txTable.delete(new Delete(Bytes.toBytes("z" + i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ int cnt = 0;
+ ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
+ Result res;
+ while ((res = zScanner.next()) != null) {
+ assertFalse(res.isEmpty());
+ assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
+ cnt += 2;
+ }
+
+ // Test 5: delete prior writes in the same transaction
+ txContext.start();
+ txTable.put(new Put(TestBytes.row4)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txTable.delete(new Delete(TestBytes.row4));
+ txContext.finish();
+
+ txContext.start();
+ Result row4Result = txTable.get(new Get(TestBytes.row4));
+ assertTrue(row4Result.isEmpty());
+ txContext.finish();
+ } finally {
+ txTable.close();
+ }
+ }
+
+ /**
+ * Expect an exception since a transaction hasn't been started.
+ *
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testTransactionlessFailure() throws Exception {
+ transactionAwareHTable.get(new Get(TestBytes.row));
+ }
+
+ /**
+ * Tests that each transaction can see its own persisted writes, while not seeing writes from other
+ * in-progress transactions.
+ */
+ @Test
+ public void testReadYourWrites() throws Exception {
+ // In-progress tx1: started before our main transaction
+ HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
+ TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
+
+ // In-progress tx2: started while our main transaction is running
+ HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
+ TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
+
+ // create an in-progress write that should be ignored
+ byte[] col2 = Bytes.toBytes("col2");
+ inprogressTxContext1.start();
+ Put putCol2 = new Put(TestBytes.row);
+ byte[] valueCol2 = Bytes.toBytes("writing in progress");
+ putCol2.add(TestBytes.family, col2, valueCol2);
+ txHTable1.put(putCol2);
+
+ // start a tx and write a value to test reading in same tx
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] value = Bytes.toBytes("writing");
+ put.add(TestBytes.family, TestBytes.qualifier, value);
+ transactionAwareHTable.put(put);
+
+ // test that a write from a tx started after the first is not visible
+ inprogressTxContext2.start();
+ Put put2 = new Put(TestBytes.row);
+ byte[] value2 = Bytes.toBytes("writing2");
+ put2.add(TestBytes.family, TestBytes.qualifier, value2);
+ txHTable2.put(put2);
+
+ Get get = new Get(TestBytes.row);
+ Result row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
+ Assert.assertNotNull(col1Value);
+ Assert.assertArrayEquals(value, col1Value);
+ // write from in-progress transaction should not be visible
+ byte[] col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ // commit in-progress transaction, should still not be visible
+ inprogressTxContext1.finish();
+
+ get = new Get(TestBytes.row);
+ row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ transactionContext.finish();
+
+ inprogressTxContext2.abort();
+ }
+
+ @Test
+ public void testRowLevelConflictDetection() throws Exception {
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ byte[] row1 = Bytes.toBytes("row1");
+ byte[] row2 = Bytes.toBytes("row2");
+ byte[] col1 = Bytes.toBytes("c1");
+ byte[] col2 = Bytes.toBytes("c2");
+ byte[] val1 = Bytes.toBytes("val1");
+ byte[] val2 = Bytes.toBytes("val2");
+
+ // test that concurrent writing to different rows succeeds
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
+
+ // should be no conflicts
+ txContext1.finish();
+ txContext2.finish();
+
+ transactionContext.start();
+ Result res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // test that writing to different columns in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
+
+ txContext2.start();
+ txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row-level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ // should now be val2
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ // col2 should not be visible due to conflict
+ assertNull(cell);
+ transactionContext.finish();
+
+ // test that writing to the same column in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row and column level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ assertNotNull(cell);
+ // should now be val1
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // verify change set that is being reported only on rows
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ Collection<byte[]> changeSet = txTable1.getTxChanges();
+ assertNotNull(changeSet);
+ assertEquals(2, changeSet.size());
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
+ txContext1.finish();
+ }
+
+ @Test
+ public void testNoneLevelConflictDetection() throws Exception {
+ InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+
+ // overlapping writes to the same row + column should not conflict
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // changes should not be visible yet
+ txContext2.start();
+ Result row = txTable2.get(new Get(TestBytes.row));
+ assertTrue(row.isEmpty());
+
+ txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // both commits should succeed
+ txContext1.finish();
+ txContext2.finish();
+
+ txContext1.start();
+ row = txTable1.get(new Get(TestBytes.row));
+ assertFalse(row.isEmpty());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext1.finish();
+
+ // transaction abort should still rollback changes
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext1.abort();
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row2 = txTable2.get(new Get(TestBytes.row2));
+ assertTrue(row2.isEmpty());
+ txContext2.finish();
+
+ // transaction invalidate should still make changes invisible
+
+ txContext1.start();
+ Transaction tx1 = txContext1.getCurrentTransaction();
+ txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ assertNotNull(tx1);
+ txClient.invalidate(tx1.getWritePointer());
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row3 = txTable2.get(new Get(TestBytes.row3));
+ assertTrue(row3.isEmpty());
+ txContext2.finish();
+ }
+
+ @Test
+ public void testCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
+ assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
+ long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
+ assertEquals(1, checkpointPtrs.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
+ assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
+ long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
+ assertEquals(2, checkpointPtrs2.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
+ assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // by default, all rows should be visible with Read-Your-Writes
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+
+ // when disabling current write pointer, only the previous checkpoints should be visible
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ Get get = new Get(TestBytes.row);
+ verifyRow(transactionAwareHTable, get, TestBytes.value);
+ get = new Get(TestBytes.row2);
+ verifyRow(transactionAwareHTable, get, TestBytes.value2);
+ get = new Get(TestBytes.row3);
+ verifyRow(transactionAwareHTable, get, null);
+
+ // test scan results excluding current write pointer
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+
+ Result row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row2, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNull(row);
+ scanner.close();
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+
+ // commit transaction, verify writes are visible
+ transactionContext.finish();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testInProgressCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // check that writes are still not visible to other clients
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ txContext2.start();
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ txContext2.finish();
+ txTable2.close();
+
+ transactionContext.finish();
+
+ // verify writes are visible after commit
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointRollback() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, null);
+ verifyRow(transactionAwareHTable, TestBytes.row2, null);
+ verifyRow(transactionAwareHTable, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointInvalidate() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
+
+ // check that writes are not visible
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+ txContext2.start();
+ Transaction newTx = txContext2.getCurrentTransaction();
+
+ // all 3 writes pointers from the previous transaction should now be excluded
+ assertTrue(newTx.isExcluded(origTx.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
+
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ verifyRow(txTable2, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = txTable2.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ txContext2.finish();
+ }
+
+ @Test
+ public void testExistingData() throws Exception {
+ byte[] val11 = Bytes.toBytes("val11");
+ byte[] val12 = Bytes.toBytes("val12");
+ byte[] val21 = Bytes.toBytes("val21");
+ byte[] val22 = Bytes.toBytes("val22");
+ byte[] val31 = Bytes.toBytes("val31");
+ byte[] val111 = Bytes.toBytes("val111");
+
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
+ Collections.<String>emptyList()));
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Add some pre-existing, non-transactional data
+ HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
+ HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.flushCommits();
+
+ // Add transactional data
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
+ txContext.finish();
+
+ txContext.start();
+ // test get
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+
+ // test scan
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+
+ // test update and delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
+ txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+
+ txContext.start();
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+ txContext.finish();
+
+ // test scan
+ txContext.start();
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+ }
+
+ private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
+ verifyRow(table, new Get(rowkey), expectedValue);
+ }
+
+ private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
+ verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
+ }
+
+ private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
+ Result result = table.get(get);
+ if (expectedValues == null) {
+ assertTrue(result.isEmpty());
+ } else {
+ assertFalse(result.isEmpty());
+ byte[] family = TestBytes.family;
+ byte[] col = TestBytes.qualifier;
+ if (get.hasFamilies()) {
+ family = get.getFamilyMap().keySet().iterator().next();
+ col = get.getFamilyMap().get(family).first();
+ }
+ Iterator<Cell> it = result.getColumnCells(family, col).iterator();
+ for (byte[] expectedValue : expectedValues) {
+ Assert.assertTrue(it.hasNext());
+ assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
+ }
+ }
+ }
+
+ private Cell[] getRow(HTableInterface table, Get get) throws Exception {
+ Result result = table.get(get);
+ return result.rawCells();
+ }
+
+ private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
+ List<Cell> actualCells = new ArrayList<>();
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ Result[] results = scanner.next(expectedCells.size() + 1);
+ for (Result result : results) {
+ actualCells.addAll(Lists.newArrayList(result.rawCells()));
+ }
+ Assert.assertEquals(expectedCells, actualCells);
+ }
+ }
+
+ @Test
+ public void testVisibilityAll() throws Exception {
+ HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(nonTxTable,
+ TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // start a transaction and create a delete marker
+ txContext.start();
+ //noinspection ConstantConditions
+ long txWp0 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // start a new transaction and write some values
+ txContext.start();
+ @SuppressWarnings("ConstantConditions")
+ long txWp1 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+
+ // verify written data
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ TestBytes.value2);
+
+ // checkpoint and make changes to written data now
+ txContext.checkpoint();
+ long txWp2 = txContext.getCurrentTransaction().getWritePointer();
+ // delete a column
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
+ // no change to a column
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ // update a column
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
+ // delete column family
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
+
+ // verify changed values
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ null);
+
+ // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
+ // and the raw change by prior transaction
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ List<KeyValue> expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // verify a Get is also able to return all snapshot versions
+ Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
+ Cell[] cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(3, cells.length);
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+ Assert.assertTrue(CellUtil.isDeleteColumns(cells[2]));
+
+ verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ ImmutableList.of(TestBytes.value3, TestBytes.value));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+
+ // Verify VisibilityLevel.SNAPSHOT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+ txContext.finish();
+
+ // finally verify values once more after commit, this time we should get only committed raw values for
+ // all visibility levels
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ txContext.finish();
+
+ // Test with regular HBase deletes in pre-existing data
+ long now = System.currentTimeMillis();
+ Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteColumn);
+ Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteFamily);
+ nonTxTable.flushCommits();
+
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
+ new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
+ );
+ // test scan
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ verifyScan(txTable, scan, expected);
+ txContext.finish();
+ }
+
+ @Test
+ public void testFilters() throws Exception {
+ // Add some values to table
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] val1 = Bytes.toBytes(1L);
+ put.add(TestBytes.family, TestBytes.qualifier, val1);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row2);
+ byte[] val2 = Bytes.toBytes(2L);
+ put.add(TestBytes.family, TestBytes.qualifier, val2);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row3);
+ byte[] val3 = Bytes.toBytes(3L);
+ put.add(TestBytes.family, TestBytes.qualifier, val3);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row4);
+ byte[] val4 = Bytes.toBytes(4L);
+ put.add(TestBytes.family, TestBytes.qualifier, val4);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Delete cell with value 2
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row2);
+ delete.addColumn(TestBytes.family, TestBytes.qualifier);
+ transactionAwareHTable.delete(delete);
+ transactionContext.finish();
+
+ // Scan for values less than 4, should get only values 1 and 3
+ transactionContext.start();
+ Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(4)));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run a Get with a filter for less than 10 on row4, should get value 4
+ transactionContext.start();
+ Get get = new Get(TestBytes.row4);
+ get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
+ Result result = transactionAwareHTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
+ transactionContext.finish();
+
+ // Change value of row4 to 40
+ transactionContext.start();
+ put = new Put(TestBytes.row4);
+ byte[] val40 = Bytes.toBytes(40L);
+ put.add(TestBytes.family, TestBytes.qualifier, val40);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Scan for values less than 10, should get only values 1 and 3
+ transactionContext.start();
+ scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run the Get again with a filter for less than 10 on row4, this time should not get any results
+ transactionContext.start();
+ result = transactionAwareHTable.get(get);
+ assertTrue(result.isEmpty());
+ transactionContext.finish();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java
new file mode 100644
index 0000000..6e21b7d
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilterTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * HBase 1.0-cdh specific test for testing {@link CellSkipFilter}.
+ */
+public class CellSkipFilterTest {
+
+ private static final String ROW1KEY = "row1";
+ private static final String ROW2KEY = "row2";
+ private static final String FAM1KEY = "fam1";
+ private static final String COL1KEY = "col1";
+ private static final String FAM2KEY = "fam2";
+ private static final String COL2KEY = "col2";
+ private static final String VALUE = "value";
+
+ @Test
+ public void testSkipFiltering() throws Exception {
+ long timestamp = System.currentTimeMillis();
+ // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
+ Filter filter = new CellSkipFilter(new MyFilter(0));
+ assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give SKIP from MyFilter
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
+ timestamp - 4)));
+
+ // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
+ filter = new CellSkipFilter(new MyFilter(2));
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give NEXT_ROW from MyFilter
+ assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
+ timestamp - 4)));
+
+ // Next call with the new key should returned the SEEK_NEXT_USING_HINT
+ assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 5)));
+ }
+
+ private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
+ timestamp, Bytes.toBytes(value));
+ }
+
+ /**
+ * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
+ * start index in the list and start serving the return codes corresponding that that index. Every time the
+ * return code is served, index is incremented.
+ */
+ class MyFilter extends FilterBase {
+
+ private final List<ReturnCode> returnCodes;
+ private int counter;
+
+ public MyFilter(int startIndex) {
+ returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
+ ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
+ counter = startIndex;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ ReturnCode code = returnCodes.get(counter % returnCodes.size());
+ counter++;
+ return code;
+ }
+ }
+}
[33/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/HDFSUtil.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/HDFSUtil.java b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSUtil.java
new file mode 100644
index 0000000..4b9e646
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSUtil.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.persist;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.lang.reflect.Method;
+
+/**
+ * Utility for handling HDFS file lease recovery. This is a copy-n-paste fork of
+ * {@link org.apache.hadoop.hbase.util.FSHDFSUtils} from the latest HBase 0.94 version (as of 0.94.12),
+ * which contains some additional fixes not present in our current HBase dependency version --
+ * mainly checking the return value of the {@code DistributedFileSystem.recoverLease()} call to verify that
+ * recovery succeeded.
+ */
+public class HDFSUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(HDFSUtil.class);
+ /**
+ * Recover the lease from HDFS, retrying multiple times.
+ */
+ public void recoverFileLease(final FileSystem fs, final Path p,
+ Configuration conf)
+ throws IOException {
+ // lease recovery not needed for local file system case.
+ if (!(fs instanceof DistributedFileSystem)) {
+ return;
+ }
+ recoverDFSFileLease((DistributedFileSystem) fs, p, conf);
+ }
+
+ /*
+ * Run the dfs recover lease. recoverLease is asynchronous. It returns:
+ * -false when it starts the lease recovery (i.e. lease recovery not *yet* done)
+ * - true when the lease recovery has succeeded or the file is closed.
+ * But, we have to be careful. Each time we call recoverLease, it starts the recover lease
+ * process over from the beginning. We could put ourselves in a situation where we are
+ * doing nothing but starting a recovery, interrupting it to start again, and so on.
+ * The findings over in HBASE-8354 have it that the namenode will try to recover the lease
+ * on the file's primary node. If all is well, it should return near immediately. But,
+ * as is common, it is the very primary node that has crashed and so the namenode will be
+ * stuck waiting on a socket timeout before it will ask another datanode to start the
+ * recovery. It does not help if we call recoverLease in the meantime and in particular,
+ * subsequent to the socket timeout, a recoverLease invocation will cause us to start
+ * over from square one (possibly waiting on socket timeout against primary node). So,
+ * in the below, we do the following:
+ * 1. Call recoverLease.
+ * 2. If it returns true, break.
+ * 3. If it returns false, wait a few seconds and then call it again.
+ * 4. If it returns true, break.
+ * 5. If it returns false, wait for what we think the datanode socket timeout is
+ * (configurable) and then try again.
+ * 6. If it returns true, break.
+ * 7. If it returns false, repeat starting at step 5. above.
+ *
+ * If HDFS-4525 is available, call it every second and we might be able to exit early.
+ */
+ boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
+ final Configuration conf)
+ throws IOException {
+ LOG.info("Recovering lease on dfs file " + p);
+ long startWaiting = System.currentTimeMillis();
+ // Default is 15 minutes. It's huge, but the idea is that if we have a major issue, HDFS
+ // usually needs 10 minutes before marking the nodes as dead. So we're putting ourselves
+ // beyond that limit 'to be safe'.
+ long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
+ // This setting should be what the cluster dfs heartbeat is set to.
+ long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 3000);
+ // This should be set to how long it'll take for us to timeout against primary datanode if it
+ // is dead. We set it to 61 seconds, 1 second than the default READ_TIMEOUT in HDFS, the
+ // default value for DFS_CLIENT_SOCKET_TIMEOUT_KEY.
+ long subsequentPause = conf.getInt("hbase.lease.recovery.dfs.timeout", 61 * 1000);
+
+ Method isFileClosedMeth = null;
+ // whether we need to look for isFileClosed method
+ boolean findIsFileClosedMeth = true;
+ boolean recovered = false;
+ // We break the loop if we succeed the lease recovery, timeout, or we throw an exception.
+ for (int nbAttempt = 0; !recovered; nbAttempt++) {
+ recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
+ if (recovered || checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) {
+ break;
+ }
+ try {
+ // On the first time through wait the short 'firstPause'.
+ if (nbAttempt == 0) {
+ Thread.sleep(firstPause);
+ } else {
+ // Cycle here until subsequentPause elapses. While spinning, check isFileClosed if
+ // available (should be in hadoop 2.0.5... not in hadoop 1 though.
+ long localStartWaiting = System.currentTimeMillis();
+ while ((System.currentTimeMillis() - localStartWaiting) <
+ subsequentPause) {
+ Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
+ if (findIsFileClosedMeth) {
+ try {
+ isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
+ new Class[]{ Path.class });
+ } catch (NoSuchMethodException nsme) {
+ LOG.debug("isFileClosed not available");
+ } finally {
+ findIsFileClosedMeth = false;
+ }
+ }
+ if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
+ recovered = true;
+ break;
+ }
+ }
+ }
+ } catch (InterruptedException ie) {
+ InterruptedIOException iioe = new InterruptedIOException();
+ iioe.initCause(ie);
+ throw iioe;
+ }
+ }
+ return recovered;
+ }
+
+ boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
+ final int nbAttempt, final Path p, final long startWaiting) {
+ if (recoveryTimeout < System.currentTimeMillis()) {
+ LOG.warn("Cannot recoverLease after trying for " +
+ conf.getInt("hbase.lease.recovery.timeout", 900000) +
+ "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
+ getLogMessageDetail(nbAttempt, p, startWaiting));
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Try to recover the lease.
+ * @param dfs The filesystem instance.
+ * @param nbAttempt Count number of this attempt.
+ * @param p Path of the file to recover.
+ * @param startWaiting Timestamp of when we started attempting to recover the file lease.
+ * @return True if dfs#recoverLease came by true.
+ * @throws java.io.FileNotFoundException
+ */
+ boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
+ final long startWaiting)
+ throws FileNotFoundException {
+ boolean recovered = false;
+ try {
+ recovered = dfs.recoverLease(p);
+ LOG.info("recoverLease=" + recovered + ", " +
+ getLogMessageDetail(nbAttempt, p, startWaiting));
+ } catch (IOException e) {
+ if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
+ // This exception comes out instead of FNFE, fix it
+ throw new FileNotFoundException("The given file wasn't found at " + p);
+ } else if (e instanceof FileNotFoundException) {
+ throw (FileNotFoundException) e;
+ }
+ LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
+ }
+ return recovered;
+ }
+
+ /**
+ * @param nbAttempt Attempt number for the lease recovery.
+ * @param p Path of the file to recover.
+ * @param startWaiting Timestamp of when we started attempting to recover the file lease.
+ * @return Detail to append to any log message around lease recovering.
+ */
+ private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
+ return "attempt=" + nbAttempt + " on file=" + p + " after " +
+ (System.currentTimeMillis() - startWaiting) + "ms";
+ }
+
+ /**
+ * Call HDFS-4525 isFileClosed if it is available.
+ * @param dfs Filesystem instance to use.
+ * @param m Method instance to call.
+ * @param p Path of the file to check is closed.
+ * @return True if file is closed.
+ */
+ private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
+ try {
+ return (Boolean) m.invoke(dfs, p);
+ } catch (SecurityException e) {
+ LOG.warn("No access", e);
+ } catch (Exception e) {
+ LOG.warn("Failed invocation for " + p.toString(), e);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionLog.java b/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionLog.java
new file mode 100644
index 0000000..d81ba38
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionLog.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import org.apache.tephra.metrics.MetricsCollector;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/**
+ * Reads and writes transaction logs against files in the local filesystem.
+ */
+public class LocalFileTransactionLog extends AbstractTransactionLog {
+ private final File logFile;
+
+ /**
+ * Creates a new transaction log using the given file instance.
+ * @param logFile The log file to use.
+ */
+ public LocalFileTransactionLog(File logFile, long timestamp, MetricsCollector metricsCollector) {
+ super(timestamp, metricsCollector);
+ this.logFile = logFile;
+ }
+
+ @Override
+ public String getName() {
+ return logFile.getAbsolutePath();
+ }
+
+ @Override
+ protected TransactionLogWriter createWriter() throws IOException {
+ return new LogWriter(logFile);
+ }
+
+ @Override
+ public TransactionLogReader getReader() throws IOException {
+ return new LogReader(logFile);
+ }
+
+ private static final class LogWriter implements TransactionLogWriter {
+ private final FileOutputStream fos;
+ private final DataOutputStream out;
+
+ public LogWriter(File logFile) throws IOException {
+ this.fos = new FileOutputStream(logFile);
+ this.out = new DataOutputStream(new BufferedOutputStream(fos, LocalFileTransactionStateStorage.BUFFER_SIZE));
+ }
+
+ @Override
+ public void append(Entry entry) throws IOException {
+ entry.write(out);
+ }
+
+ @Override
+ public void commitMarker(int count) throws IOException {
+ // skip for local file
+ }
+
+ @Override
+ public void sync() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.flush();
+ out.close();
+ fos.close();
+ }
+ }
+
+ private static final class LogReader implements TransactionLogReader {
+ private final FileInputStream fin;
+ private final DataInputStream in;
+ private Entry reuseEntry = new Entry();
+
+ public LogReader(File logFile) throws IOException {
+ this.fin = new FileInputStream(logFile);
+ this.in = new DataInputStream(new BufferedInputStream(fin, LocalFileTransactionStateStorage.BUFFER_SIZE));
+ }
+
+ @Override
+ public TransactionEdit next() throws IOException {
+ Entry entry = new Entry();
+ try {
+ entry.readFields(in);
+ } catch (EOFException eofe) {
+ // signal end of file by returning null
+ return null;
+ }
+ return entry.getEdit();
+ }
+
+ @Override
+ public TransactionEdit next(TransactionEdit reuse) throws IOException {
+ try {
+ reuseEntry.getKey().readFields(in);
+ reuse.readFields(in);
+ } catch (EOFException eofe) {
+ // signal end of file by returning null
+ return null;
+ }
+ return reuse;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ fin.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionStateStorage.java b/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionStateStorage.java
new file mode 100644
index 0000000..beddbb2
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/LocalFileTransactionStateStorage.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.io.Closeables;
+import com.google.common.io.Files;
+import com.google.common.primitives.Longs;
+import com.google.inject.Inject;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * Persists transaction snapshots and write-ahead logs to files on the local filesystem.
+ */
+public class LocalFileTransactionStateStorage extends AbstractTransactionStateStorage {
+ private static final String TMP_SNAPSHOT_FILE_PREFIX = ".in-progress.";
+ private static final String SNAPSHOT_FILE_PREFIX = "snapshot.";
+ private static final String LOG_FILE_PREFIX = "txlog.";
+ private static final Logger LOG = LoggerFactory.getLogger(LocalFileTransactionStateStorage.class);
+ static final int BUFFER_SIZE = 16384;
+
+ private static final FilenameFilter SNAPSHOT_FILE_FILTER = new FilenameFilter() {
+ @Override
+ public boolean accept(File file, String s) {
+ return s.startsWith(SNAPSHOT_FILE_PREFIX);
+ }
+ };
+
+ private final String configuredSnapshotDir;
+ private final MetricsCollector metricsCollector;
+ private File snapshotDir;
+
+ @Inject
+ public LocalFileTransactionStateStorage(Configuration conf, SnapshotCodecProvider codecProvider,
+ MetricsCollector metricsCollector) {
+ super(codecProvider);
+ this.configuredSnapshotDir = conf.get(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR);
+ this.metricsCollector = metricsCollector;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ Preconditions.checkState(configuredSnapshotDir != null,
+ "Snapshot directory is not configured. Please set " + TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR +
+ " in configuration.");
+ snapshotDir = new File(configuredSnapshotDir);
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ // nothing to do
+ }
+
+ @Override
+ public String getLocation() {
+ return snapshotDir.getAbsolutePath();
+ }
+
+ @Override
+ public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
+ // save the snapshot to a temporary file
+ File snapshotTmpFile = new File(snapshotDir, TMP_SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
+ LOG.debug("Writing snapshot to temporary file {}", snapshotTmpFile);
+ OutputStream out = Files.newOutputStreamSupplier(snapshotTmpFile).getOutput();
+ boolean threw = true;
+ try {
+ codecProvider.encode(out, snapshot);
+ threw = false;
+ } finally {
+ Closeables.close(out, threw);
+ }
+
+ // move the temporary file into place with the correct filename
+ File finalFile = new File(snapshotDir, SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
+ if (!snapshotTmpFile.renameTo(finalFile)) {
+ throw new IOException("Failed renaming temporary snapshot file " + snapshotTmpFile.getName() + " to " +
+ finalFile.getName());
+ }
+
+ LOG.debug("Completed snapshot to file {}", finalFile);
+ }
+
+ @Override
+ public TransactionSnapshot getLatestSnapshot() throws IOException {
+ InputStream is = getLatestSnapshotInputStream();
+ if (is == null) {
+ return null;
+ }
+ try {
+ return readSnapshotFile(is);
+ } finally {
+ is.close();
+ }
+ }
+
+ @Override
+ public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
+ InputStream is = getLatestSnapshotInputStream();
+ if (is == null) {
+ return null;
+ }
+ try {
+ return codecProvider.decodeTransactionVisibilityState(is);
+ } finally {
+ is.close();
+ }
+ }
+
+ private InputStream getLatestSnapshotInputStream() throws IOException {
+ File[] snapshotFiles = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER);
+ TimestampedFilename mostRecent = null;
+ for (File file : snapshotFiles) {
+ TimestampedFilename tsFile = new TimestampedFilename(file);
+ if (mostRecent == null || tsFile.compareTo(mostRecent) > 0) {
+ mostRecent = tsFile;
+ }
+ }
+
+ if (mostRecent == null) {
+ LOG.info("No snapshot files found in {}", snapshotDir.getAbsolutePath());
+ return null;
+ }
+
+ return new FileInputStream(mostRecent.getFile());
+ }
+
+ private TransactionSnapshot readSnapshotFile(InputStream is) throws IOException {
+ return codecProvider.decode(is);
+ }
+
+ @Override
+ public long deleteOldSnapshots(int numberToKeep) throws IOException {
+ File[] snapshotFiles = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER);
+ if (snapshotFiles.length == 0) {
+ return -1;
+ }
+ TimestampedFilename[] snapshotFilenames = new TimestampedFilename[snapshotFiles.length];
+ for (int i = 0; i < snapshotFiles.length; i++) {
+ snapshotFilenames[i] = new TimestampedFilename(snapshotFiles[i]);
+ }
+ Arrays.sort(snapshotFilenames, Collections.reverseOrder());
+ if (snapshotFilenames.length <= numberToKeep) {
+ // nothing to delete, just return the oldest timestamp
+ return snapshotFilenames[snapshotFilenames.length - 1].getTimestamp();
+ }
+ int toRemoveCount = snapshotFilenames.length - numberToKeep;
+ TimestampedFilename[] toRemove = new TimestampedFilename[toRemoveCount];
+ System.arraycopy(snapshotFilenames, numberToKeep, toRemove, 0, toRemoveCount);
+ int removedCnt = 0;
+ for (int i = 0; i < toRemove.length; i++) {
+ File currentFile = toRemove[i].getFile();
+ LOG.debug("Removing old snapshot file {}", currentFile.getAbsolutePath());
+ if (!toRemove[i].getFile().delete()) {
+ LOG.error("Failed deleting snapshot file {}", currentFile.getAbsolutePath());
+ } else {
+ removedCnt++;
+ }
+ }
+ long oldestTimestamp = snapshotFilenames[numberToKeep - 1].getTimestamp();
+ LOG.info("Removed {} out of {} expected snapshot files older than {}", removedCnt, toRemoveCount, oldestTimestamp);
+ return oldestTimestamp;
+ }
+
+ @Override
+ public List<String> listSnapshots() throws IOException {
+ File[] snapshots = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER);
+ return Lists.transform(Arrays.asList(snapshots), new Function<File, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable File input) {
+ return input.getName();
+ }
+ });
+ }
+
+ @Override
+ public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
+ File[] logFiles = snapshotDir.listFiles(new LogFileFilter(timestamp, Long.MAX_VALUE));
+ TimestampedFilename[] timestampedFiles = new TimestampedFilename[logFiles.length];
+ for (int i = 0; i < logFiles.length; i++) {
+ timestampedFiles[i] = new TimestampedFilename(logFiles[i]);
+ }
+ // logs need to be processed in ascending order
+ Arrays.sort(timestampedFiles);
+ return Lists.transform(Arrays.asList(timestampedFiles), new Function<TimestampedFilename, TransactionLog>() {
+ @Nullable
+ @Override
+ public TransactionLog apply(@Nullable TimestampedFilename input) {
+ return new LocalFileTransactionLog(input.getFile(), input.getTimestamp(), metricsCollector);
+ }
+ });
+ }
+
+ @Override
+ public TransactionLog createLog(long timestamp) throws IOException {
+ File newLogFile = new File(snapshotDir, LOG_FILE_PREFIX + timestamp);
+ LOG.info("Creating new transaction log at {}", newLogFile.getAbsolutePath());
+ return new LocalFileTransactionLog(newLogFile, timestamp, metricsCollector);
+ }
+
+ @Override
+ public void deleteLogsOlderThan(long timestamp) throws IOException {
+ File[] logFiles = snapshotDir.listFiles(new LogFileFilter(0, timestamp));
+ int removedCnt = 0;
+ for (File file : logFiles) {
+ LOG.debug("Removing old transaction log {}", file.getPath());
+ if (file.delete()) {
+ removedCnt++;
+ } else {
+ LOG.warn("Failed to remove log file {}", file.getAbsolutePath());
+ }
+ }
+ LOG.debug("Removed {} transaction logs older than {}", removedCnt, timestamp);
+ }
+
+ @Override
+ public void setupStorage() throws IOException {
+ // create the directory if it doesn't exist
+ if (!snapshotDir.exists()) {
+ if (!snapshotDir.mkdirs()) {
+ throw new IOException("Failed to create directory " + configuredSnapshotDir +
+ " for transaction snapshot storage");
+ }
+ } else {
+ Preconditions.checkState(snapshotDir.isDirectory(),
+ "Configured snapshot directory " + configuredSnapshotDir + " is not a directory!");
+ Preconditions.checkState(snapshotDir.canWrite(), "Configured snapshot directory " +
+ configuredSnapshotDir + " exists but is not writable!");
+ }
+ }
+
+ @Override
+ public List<String> listLogs() throws IOException {
+ File[] logs = snapshotDir.listFiles(new LogFileFilter(0, Long.MAX_VALUE));
+ return Lists.transform(Arrays.asList(logs), new Function<File, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable File input) {
+ return input.getName();
+ }
+ });
+ }
+
+ private static class LogFileFilter implements FilenameFilter {
+ private final long startTime;
+ private final long endTime;
+
+ public LogFileFilter(long startTime, long endTime) {
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ @Override
+ public boolean accept(File file, String s) {
+ if (s.startsWith(LOG_FILE_PREFIX)) {
+ String[] parts = s.split("\\.");
+ if (parts.length == 2) {
+ try {
+ long fileTime = Long.parseLong(parts[1]);
+ return fileTime >= startTime && fileTime < endTime;
+ } catch (NumberFormatException ignored) {
+ LOG.warn("Filename {} did not match the expected pattern prefix.<timestamp>", s);
+ }
+ }
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Represents a filename composed of a prefix and a ".timestamp" suffix. This is useful for manipulating both
+ * snapshot and transaction log filenames.
+ */
+ private static class TimestampedFilename implements Comparable<TimestampedFilename> {
+ private File file;
+ private String prefix;
+ private long timestamp;
+
+ public TimestampedFilename(File file) {
+ this.file = file;
+ String[] parts = file.getName().split("\\.");
+ if (parts.length != 2) {
+ throw new IllegalArgumentException("Filename " + file.getName() +
+ " did not match the expected pattern prefix.timestamp");
+ }
+ prefix = parts[0];
+ timestamp = Long.parseLong(parts[1]);
+ }
+
+ public File getFile() {
+ return file;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public int compareTo(TimestampedFilename other) {
+ int res = prefix.compareTo(other.getPrefix());
+ if (res == 0) {
+ res = Longs.compare(timestamp, other.getTimestamp());
+ }
+ return res;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/NoOpTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/NoOpTransactionStateStorage.java b/tephra-core/src/main/java/org/apache/tephra/persist/NoOpTransactionStateStorage.java
new file mode 100644
index 0000000..12f2475
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/NoOpTransactionStateStorage.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.inject.Inject;
+import org.apache.tephra.snapshot.SnapshotCodec;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Minimal {@link TransactionStateStorage} implementation that does nothing, i.e. does not maintain any actual state.
+ */
+public class NoOpTransactionStateStorage extends AbstractIdleService implements TransactionStateStorage {
+
+ private final SnapshotCodec codec;
+
+ @Inject
+ public NoOpTransactionStateStorage(SnapshotCodecProvider codecProvider) {
+ codec = codecProvider;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ }
+
+ @Override
+ public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException {
+ codec.encode(out, snapshot);
+ }
+
+ @Override
+ public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
+ }
+
+ @Override
+ public TransactionSnapshot getLatestSnapshot() throws IOException {
+ return null;
+ }
+
+ @Override
+ public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
+ return null;
+ }
+
+ @Override
+ public long deleteOldSnapshots(int numberToKeep) throws IOException {
+ return 0;
+ }
+
+ @Override
+ public List<String> listSnapshots() throws IOException {
+ return new ArrayList<>(0);
+ }
+
+ @Override
+ public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
+ return new ArrayList<>(0);
+ }
+
+ @Override
+ public TransactionLog createLog(long timestamp) throws IOException {
+ return new NoOpTransactionLog();
+ }
+
+ @Override
+ public void deleteLogsOlderThan(long timestamp) throws IOException {
+ }
+
+ @Override
+ public void setupStorage() throws IOException {
+ }
+
+ @Override
+ public List<String> listLogs() throws IOException {
+ return new ArrayList<>(0);
+ }
+
+ @Override
+ public String getLocation() {
+ return "no-op";
+ }
+
+ private static class NoOpTransactionLog implements TransactionLog {
+ private long timestamp = System.currentTimeMillis();
+
+ @Override
+ public String getName() {
+ return "no-op";
+ }
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public void append(TransactionEdit edit) throws IOException {
+ }
+
+ @Override
+ public void append(List<TransactionEdit> edits) throws IOException {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public TransactionLogReader getReader() {
+ return new TransactionLogReader() {
+ @Override
+ public TransactionEdit next() {
+ return null;
+ }
+
+ @Override
+ public TransactionEdit next(TransactionEdit reuse) {
+ return null;
+ }
+
+ @Override
+ public void close() {
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEdit.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEdit.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEdit.java
new file mode 100644
index 0000000..1d07e72
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEdit.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.io.Writable;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * Represents a transaction state change in the {@link TransactionLog}.
+ */
+public class TransactionEdit implements Writable {
+
+ /**
+ * The possible state changes for a transaction.
+ */
+ public enum State {
+ INPROGRESS, COMMITTING, COMMITTED, INVALID, ABORTED, MOVE_WATERMARK, TRUNCATE_INVALID_TX, CHECKPOINT
+ }
+
+ private long writePointer;
+
+ /**
+ * stores the value of visibility upper bound
+ * (see {@link TransactionManager.InProgressTx#getVisibilityUpperBound()})
+ * for edit of {@link State#INPROGRESS} only
+ */
+ private long visibilityUpperBound;
+ private long commitPointer;
+ private long expirationDate;
+ private State state;
+ private Set<ChangeId> changes;
+ /** Whether or not the COMMITTED change should be fully committed. */
+ private boolean canCommit;
+ private TransactionType type;
+ private Set<Long> truncateInvalidTx;
+ private long truncateInvalidTxTime;
+ private long parentWritePointer;
+ private long[] checkpointPointers;
+
+ // for Writable
+ public TransactionEdit() {
+ this.changes = Sets.newHashSet();
+ this.truncateInvalidTx = Sets.newHashSet();
+ }
+
+ // package private for testing
+ TransactionEdit(long writePointer, long visibilityUpperBound, State state, long expirationDate,
+ Set<ChangeId> changes, long commitPointer, boolean canCommit, TransactionType type,
+ Set<Long> truncateInvalidTx, long truncateInvalidTxTime, long parentWritePointer,
+ long[] checkpointPointers) {
+ this.writePointer = writePointer;
+ this.visibilityUpperBound = visibilityUpperBound;
+ this.state = state;
+ this.expirationDate = expirationDate;
+ this.changes = changes != null ? changes : Collections.<ChangeId>emptySet();
+ this.commitPointer = commitPointer;
+ this.canCommit = canCommit;
+ this.type = type;
+ this.truncateInvalidTx = truncateInvalidTx != null ? truncateInvalidTx : Collections.<Long>emptySet();
+ this.truncateInvalidTxTime = truncateInvalidTxTime;
+ this.parentWritePointer = parentWritePointer;
+ this.checkpointPointers = checkpointPointers;
+ }
+
+ /**
+ * Returns the transaction write pointer assigned for the state change.
+ */
+ public long getWritePointer() {
+ return writePointer;
+ }
+
+ void setWritePointer(long writePointer) {
+ this.writePointer = writePointer;
+ }
+
+ public long getVisibilityUpperBound() {
+ return visibilityUpperBound;
+ }
+
+ void setVisibilityUpperBound(long visibilityUpperBound) {
+ this.visibilityUpperBound = visibilityUpperBound;
+ }
+
+ /**
+ * Returns the type of state change represented.
+ */
+ public State getState() {
+ return state;
+ }
+
+ void setState(State state) {
+ this.state = state;
+ }
+
+ /**
+ * Returns any expiration timestamp (in milliseconds) associated with the state change. This should only
+ * be populated for changes of type {@link State#INPROGRESS}.
+ */
+ public long getExpiration() {
+ return expirationDate;
+ }
+
+ void setExpiration(long expirationDate) {
+ this.expirationDate = expirationDate;
+ }
+
+ /**
+ * @return the set of changed row keys associated with the state change. This is only populated for edits
+ * of type {@link State#COMMITTING} or {@link State#COMMITTED}.
+ */
+ public Set<ChangeId> getChanges() {
+ return changes;
+ }
+
+ void setChanges(Set<ChangeId> changes) {
+ this.changes = changes;
+ }
+
+ /**
+ * Returns the write pointer used to commit the row key change set. This is only populated for edits of type
+ * {@link State#COMMITTED}.
+ */
+ public long getCommitPointer() {
+ return commitPointer;
+ }
+
+ void setCommitPointer(long commitPointer) {
+ this.commitPointer = commitPointer;
+ }
+
+ /**
+ * Returns whether or not the transaction should be moved to the committed set. This is only populated for edits
+ * of type {@link State#COMMITTED}.
+ */
+ public boolean getCanCommit() {
+ return canCommit;
+ }
+
+ void setCanCommit(boolean canCommit) {
+ this.canCommit = canCommit;
+ }
+
+ /**
+ * Returns the transaction type. This is only populated for edits of type {@link State#INPROGRESS} or
+ * {@link State#ABORTED}.
+ */
+ public TransactionType getType() {
+ return type;
+ }
+
+ void setType(TransactionType type) {
+ this.type = type;
+ }
+
+ /**
+ * Returns the transaction ids to be removed from invalid transaction list. This is only populated for
+ * edits of type {@link State#TRUNCATE_INVALID_TX}
+ */
+ public Set<Long> getTruncateInvalidTx() {
+ return truncateInvalidTx;
+ }
+
+ void setTruncateInvalidTx(Set<Long> truncateInvalidTx) {
+ this.truncateInvalidTx = truncateInvalidTx;
+ }
+
+ /**
+ * Returns the time until which the invalid transactions need to be truncated from invalid transaction list.
+ * This is only populated for edits of type {@link State#TRUNCATE_INVALID_TX}
+ */
+ public long getTruncateInvalidTxTime() {
+ return truncateInvalidTxTime;
+ }
+
+ void setTruncateInvalidTxTime(long truncateInvalidTxTime) {
+ this.truncateInvalidTxTime = truncateInvalidTxTime;
+ }
+
+ /**
+ * Returns the parent write pointer for a checkpoint operation. This is only populated for edits of type
+ * {@link State#CHECKPOINT}
+ */
+ public long getParentWritePointer() {
+ return parentWritePointer;
+ }
+
+ void setParentWritePointer(long parentWritePointer) {
+ this.parentWritePointer = parentWritePointer;
+ }
+
+ /**
+ * Returns the checkpoint write pointers for the edit. This is only populated for edits of type
+ * {@link State#ABORTED}.
+ */
+ public long[] getCheckpointPointers() {
+ return checkpointPointers;
+ }
+
+ void setCheckpointPointers(long[] checkpointPointers) {
+ this.checkpointPointers = checkpointPointers;
+ }
+
+ /**
+ * Creates a new instance in the {@link State#INPROGRESS} state.
+ */
+ public static TransactionEdit createStarted(long writePointer, long visibilityUpperBound,
+ long expirationDate, TransactionType type) {
+ return new TransactionEdit(writePointer, visibilityUpperBound, State.INPROGRESS,
+ expirationDate, null, 0L, false, type, null, 0L, 0L, null);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#COMMITTING} state.
+ */
+ public static TransactionEdit createCommitting(long writePointer, Set<ChangeId> changes) {
+ return new TransactionEdit(writePointer, 0L, State.COMMITTING, 0L, changes, 0L, false, null, null, 0L, 0L, null);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#COMMITTED} state.
+ */
+ public static TransactionEdit createCommitted(long writePointer, Set<ChangeId> changes, long nextWritePointer,
+ boolean canCommit) {
+ return new TransactionEdit(writePointer, 0L, State.COMMITTED, 0L, changes, nextWritePointer, canCommit, null,
+ null, 0L, 0L, null);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#ABORTED} state.
+ */
+ public static TransactionEdit createAborted(long writePointer, TransactionType type, long[] checkpointPointers) {
+ return new TransactionEdit(writePointer, 0L, State.ABORTED, 0L, null, 0L, false, type, null, 0L, 0L,
+ checkpointPointers);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#INVALID} state.
+ */
+ public static TransactionEdit createInvalid(long writePointer) {
+ return new TransactionEdit(writePointer, 0L, State.INVALID, 0L, null, 0L, false, null, null, 0L, 0L, null);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#MOVE_WATERMARK} state.
+ */
+ public static TransactionEdit createMoveWatermark(long writePointer) {
+ return new TransactionEdit(writePointer, 0L, State.MOVE_WATERMARK, 0L, null, 0L, false, null, null, 0L, 0L, null);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state.
+ */
+ public static TransactionEdit createTruncateInvalidTx(Set<Long> truncateInvalidTx) {
+ return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, truncateInvalidTx,
+ 0L, 0L, null);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state.
+ */
+ public static TransactionEdit createTruncateInvalidTxBefore(long truncateInvalidTxTime) {
+ return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, null,
+ truncateInvalidTxTime, 0L, null);
+ }
+
+ /**
+ * Creates a new instance in the {@link State#CHECKPOINT} state.
+ */
+ public static TransactionEdit createCheckpoint(long writePointer, long parentWritePointer) {
+ return new TransactionEdit(writePointer, 0L, State.CHECKPOINT, 0L, null, 0L, false, null, null, 0L,
+ parentWritePointer, null);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ TransactionEditCodecs.encode(this, out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ TransactionEditCodecs.decode(this, in);
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TransactionEdit)) {
+ return false;
+ }
+
+ TransactionEdit that = (TransactionEdit) o;
+
+ return Objects.equal(this.writePointer, that.writePointer) &&
+ Objects.equal(this.visibilityUpperBound, that.visibilityUpperBound) &&
+ Objects.equal(this.commitPointer, that.commitPointer) &&
+ Objects.equal(this.expirationDate, that.expirationDate) &&
+ Objects.equal(this.state, that.state) &&
+ Objects.equal(this.changes, that.changes) &&
+ Objects.equal(this.canCommit, that.canCommit) &&
+ Objects.equal(this.type, that.type) &&
+ Objects.equal(this.truncateInvalidTx, that.truncateInvalidTx) &&
+ Objects.equal(this.truncateInvalidTxTime, that.truncateInvalidTxTime) &&
+ Objects.equal(this.parentWritePointer, that.parentWritePointer) &&
+ Arrays.equals(this.checkpointPointers, that.checkpointPointers);
+ }
+
+ @Override
+ public final int hashCode() {
+ return Objects.hashCode(writePointer, visibilityUpperBound, commitPointer, expirationDate, state, changes,
+ canCommit, type, parentWritePointer, checkpointPointers);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("writePointer", writePointer)
+ .add("visibilityUpperBound", visibilityUpperBound)
+ .add("commitPointer", commitPointer)
+ .add("expiration", expirationDate)
+ .add("state", state)
+ .add("changesSize", changes != null ? changes.size() : 0)
+ .add("canCommit", canCommit)
+ .add("type", type)
+ .add("truncateInvalidTx", truncateInvalidTx)
+ .add("truncateInvalidTxTime", truncateInvalidTxTime)
+ .add("parentWritePointer", parentWritePointer)
+ .add("checkpointPointers", checkpointPointers)
+ .toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEditCodecs.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEditCodecs.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEditCodecs.java
new file mode 100644
index 0000000..387ad41
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionEditCodecs.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.TransactionType;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * Utilities to handle encoding and decoding of {@link TransactionEdit} entries, while maintaining compatibility
+ * with older versions of the serialized data.
+ */
+public class TransactionEditCodecs {
+
+ private static final TransactionEditCodec[] ALL_CODECS = {
+ new TransactionEditCodecV1(),
+ new TransactionEditCodecV2(),
+ new TransactionEditCodecV3(),
+ new TransactionEditCodecV4()
+ };
+
+ private static final SortedMap<Byte, TransactionEditCodec> CODECS = new TreeMap<>();
+ static {
+ for (TransactionEditCodec codec : ALL_CODECS) {
+ CODECS.put(codec.getVersion(), codec);
+ }
+ }
+
+ /**
+ * Deserializes the encoded data from the given input stream, setting the values as fields
+ * on the given {@code TransactionEdit} instances. This method expects first value in the
+ * {code DataInput} to be a byte representing the codec version used to serialize the instance.
+ *
+ * @param dest the transaction edit to populate with the deserialized values
+ * @param in the input stream containing the encoded data
+ * @throws IOException if an error occurs while deserializing from the input stream
+ */
+ public static void decode(TransactionEdit dest, DataInput in) throws IOException {
+ byte version = in.readByte();
+ TransactionEditCodec codec = CODECS.get(version);
+ if (codec == null) {
+ throw new IOException("TransactionEdit was serialized with an unknown codec version " + version +
+ ". Was it written with a newer version of Tephra?");
+ }
+ codec.decode(dest, in);
+ }
+
+ /**
+ * Serializes the given {@code TransactionEdit} instance with the latest available codec.
+ * This will first write out the version of the codec used to serialize the instance so that
+ * the correct codec can be used when calling {@link #decode(TransactionEdit, DataInput)}.
+ *
+ * @param src the transaction edit to serialize
+ * @param out the output stream to contain the data
+ * @throws IOException if an error occurs while serializing to the output stream
+ */
+ public static void encode(TransactionEdit src, DataOutput out) throws IOException {
+ TransactionEditCodec latestCodec = CODECS.get(CODECS.firstKey());
+ out.writeByte(latestCodec.getVersion());
+ latestCodec.encode(src, out);
+ }
+
+ /**
+ * Encodes the given transaction edit using a specific codec. Note that this is only exposed
+ * for use by tests.
+ */
+ @VisibleForTesting
+ static void encode(TransactionEdit src, DataOutput out, TransactionEditCodec codec) throws IOException {
+ out.writeByte(codec.getVersion());
+ codec.encode(src, out);
+ }
+
+ /**
+ * Defines the interface used for encoding and decoding {@link TransactionEdit} instances to and from
+ * binary representations.
+ */
+ interface TransactionEditCodec {
+ /**
+ * Reads the encoded values from the data input stream and sets the fields in the given {@code TransactionEdit}
+ * instance.
+ *
+ * @param dest the instance on which to set all the deserialized values
+ * @param in the input stream containing the serialized data
+ * @throws IOException if an error occurs while deserializing the data
+ */
+ void decode(TransactionEdit dest, DataInput in) throws IOException;
+
+ /**
+ * Writes all the field values from the {@code TransactionEdit} instance in serialized form to the data
+ * output stream.
+ *
+ * @param src the instance to serialize to the stream
+ * @param out the output stream to contain the data
+ * @throws IOException if an error occurs while serializing the data
+ */
+ void encode(TransactionEdit src, DataOutput out) throws IOException;
+
+ /**
+ * Returns the version number for this codec. Each codec should use a unique version number, with the newest
+ * codec having the lowest number.
+ */
+ byte getVersion();
+ }
+
+
+ // package-private for unit-test access
+ static class TransactionEditCodecV1 implements TransactionEditCodec {
+ @Override
+ public void decode(TransactionEdit dest, DataInput in) throws IOException {
+ dest.setWritePointer(in.readLong());
+ int stateIdx = in.readInt();
+ try {
+ dest.setState(TransactionEdit.State.values()[stateIdx]);
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new IOException("State enum ordinal value is out of range: " + stateIdx);
+ }
+ dest.setExpiration(in.readLong());
+ dest.setCommitPointer(in.readLong());
+ dest.setCanCommit(in.readBoolean());
+ int changeSize = in.readInt();
+ Set<ChangeId> changes = Sets.newHashSet();
+ for (int i = 0; i < changeSize; i++) {
+ int currentLength = in.readInt();
+ byte[] currentBytes = new byte[currentLength];
+ in.readFully(currentBytes);
+ changes.add(new ChangeId(currentBytes));
+ }
+ dest.setChanges(changes);
+ // 1st version did not store this info. It is safe to set firstInProgress to 0, it may decrease performance until
+ // this tx is finished, but correctness will be preserved.
+ dest.setVisibilityUpperBound(0);
+ }
+
+ /** @deprecated use {@link TransactionEditCodecs.TransactionEditCodecV4} instead, it is still here for
+ * unit-tests only */
+ @Override
+ @Deprecated
+ public void encode(TransactionEdit src, DataOutput out) throws IOException {
+ out.writeLong(src.getWritePointer());
+ // use ordinal for predictable size, though this does not support evolution
+ out.writeInt(src.getState().ordinal());
+ out.writeLong(src.getExpiration());
+ out.writeLong(src.getCommitPointer());
+ out.writeBoolean(src.getCanCommit());
+ Set<ChangeId> changes = src.getChanges();
+ if (changes == null) {
+ out.writeInt(0);
+ } else {
+ out.writeInt(changes.size());
+ for (ChangeId c : changes) {
+ byte[] cKey = c.getKey();
+ out.writeInt(cKey.length);
+ out.write(cKey);
+ }
+ }
+ // NOTE: we didn't have visibilityUpperBound in V1, it was added in V2
+ // we didn't have transaction type, truncateInvalidTx and truncateInvalidTxTime in V1 and V2,
+ // it was added in V3
+ }
+
+ @Override
+ public byte getVersion() {
+ return -1;
+ }
+ }
+
+ // package-private for unit-test access
+ static class TransactionEditCodecV2 extends TransactionEditCodecV1 implements TransactionEditCodec {
+ @Override
+ public void decode(TransactionEdit dest, DataInput in) throws IOException {
+ super.decode(dest, in);
+ dest.setVisibilityUpperBound(in.readLong());
+ }
+
+ /** @deprecated use {@link TransactionEditCodecs.TransactionEditCodecV4} instead, it is still here for
+ * unit-tests only */
+ @Override
+ public void encode(TransactionEdit src, DataOutput out) throws IOException {
+ super.encode(src, out);
+ out.writeLong(src.getVisibilityUpperBound());
+ // NOTE: we didn't have transaction type, truncateInvalidTx and truncateInvalidTxTime in V1 and V2,
+ // it was added in V3
+ }
+
+ @Override
+ public byte getVersion() {
+ return -2;
+ }
+ }
+
+ // TODO: refactor to avoid duplicate code among different version of codecs
+ // package-private for unit-test access
+ static class TransactionEditCodecV3 extends TransactionEditCodecV2 implements TransactionEditCodec {
+ @Override
+ public void decode(TransactionEdit dest, DataInput in) throws IOException {
+ super.decode(dest, in);
+ int typeIdx = in.readInt();
+ // null transaction type is represented as -1
+ if (typeIdx < 0) {
+ dest.setType(null);
+ } else {
+ try {
+ dest.setType(TransactionType.values()[typeIdx]);
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new IOException("Type enum ordinal value is out of range: " + typeIdx);
+ }
+ }
+
+ int truncateInvalidTxSize = in.readInt();
+ Set<Long> truncateInvalidTx = emptySet(dest.getTruncateInvalidTx());
+ for (int i = 0; i < truncateInvalidTxSize; i++) {
+ truncateInvalidTx.add(in.readLong());
+ }
+ dest.setTruncateInvalidTx(truncateInvalidTx);
+ dest.setTruncateInvalidTxTime(in.readLong());
+ }
+
+ private <T> Set<T> emptySet(Set<T> set) {
+ if (set == null) {
+ return Sets.newHashSet();
+ }
+ set.clear();
+ return set;
+ }
+
+ @Override
+ public void encode(TransactionEdit src, DataOutput out) throws IOException {
+ super.encode(src, out);
+ // null transaction type is represented as -1
+ if (src.getType() == null) {
+ out.writeInt(-1);
+ } else {
+ out.writeInt(src.getType().ordinal());
+ }
+
+ Set<Long> truncateInvalidTx = src.getTruncateInvalidTx();
+ if (truncateInvalidTx == null) {
+ out.writeInt(0);
+ } else {
+ out.writeInt(truncateInvalidTx.size());
+ for (long id : truncateInvalidTx) {
+ out.writeLong(id);
+ }
+ }
+ out.writeLong(src.getTruncateInvalidTxTime());
+ }
+
+ @Override
+ public byte getVersion() {
+ return -3;
+ }
+ }
+
+ static class TransactionEditCodecV4 extends TransactionEditCodecV3 {
+ @Override
+ public void decode(TransactionEdit dest, DataInput in) throws IOException {
+ super.decode(dest, in);
+ dest.setParentWritePointer(in.readLong());
+ int checkpointPointersLen = in.readInt();
+ if (checkpointPointersLen >= 0) {
+ long[] checkpointPointers = new long[checkpointPointersLen];
+ for (int i = 0; i < checkpointPointersLen; i++) {
+ checkpointPointers[i] = in.readLong();
+ }
+ dest.setCheckpointPointers(checkpointPointers);
+ }
+ }
+
+ @Override
+ public void encode(TransactionEdit src, DataOutput out) throws IOException {
+ super.encode(src, out);
+ out.writeLong(src.getParentWritePointer());
+ long[] checkpointPointers = src.getCheckpointPointers();
+ if (checkpointPointers == null) {
+ out.writeInt(-1);
+ } else {
+ out.writeInt(checkpointPointers.length);
+ for (int i = 0; i < checkpointPointers.length; i++) {
+ out.writeLong(checkpointPointers[i]);
+ }
+ }
+ }
+
+ @Override
+ public byte getVersion() {
+ return -4;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLog.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLog.java
new file mode 100644
index 0000000..e453523
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLog.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Represents a log of transaction state changes.
+ */
+public interface TransactionLog {
+
+ String getName();
+
+ long getTimestamp();
+
+ void append(TransactionEdit edit) throws IOException;
+
+ void append(List<TransactionEdit> edits) throws IOException;
+
+ void close() throws IOException;
+
+ TransactionLogReader getReader() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogReader.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogReader.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogReader.java
new file mode 100644
index 0000000..51fda0b
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogReader.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Represents a reader for {@link TransactionLog} instances.
+ */
+public interface TransactionLogReader extends Closeable {
+ /**
+ * Returns the next {@code TransactionEdit} from the log file, based on the current position, or {@code null}
+ * if the end of the file has been reached.
+ */
+ TransactionEdit next() throws IOException;
+
+ /**
+ * Populates {@code reuse} with the next {@code TransactionEdit}, based on the reader's current position in the
+ * log file.
+ * @param reuse The {@code TransactionEdit} instance to populate with the log entry data.
+ * @return The {@code TransactionEdit} instance, or {@code null} if the end of the file has been reached.
+ * @throws IOException If an error is encountered reading the log data.
+ */
+ TransactionEdit next(TransactionEdit reuse) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogWriter.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogWriter.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogWriter.java
new file mode 100644
index 0000000..14893ac
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionLogWriter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Common interface for transaction log writers used by classes extending {@link AbstractTransactionLog}.
+ */
+public interface TransactionLogWriter extends Closeable {
+ /**
+ * Adds a new transaction entry to the log. Note that this does not guarantee that the entry has been flushed
+ * to persistent storage until {@link #sync()} has been called.
+ *
+ * @param entry The transaction edit to append.
+ * @throws IOException If an error occurs while writing the edit to storage.
+ */
+ void append(AbstractTransactionLog.Entry entry) throws IOException;
+
+ /**
+ * Makes an entry of number of transaction entries that will follow in that log in a single sync.
+ *
+ * @param count Number of transaction entries.
+ * @throws IOException If an error occurs while writing the count to storage.
+ */
+ void commitMarker(int count) throws IOException;
+
+ /**
+ * Syncs any pending transaction edits added through {@link #append(AbstractTransactionLog.Entry)},
+ * but not yet flushed to durable storage.
+ *
+ * @throws IOException If an error occurs while flushing the outstanding edits.
+ */
+ void sync() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionSnapshot.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionSnapshot.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionSnapshot.java
new file mode 100644
index 0000000..ccf7374
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionSnapshot.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.TransactionManager;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Represents an in-memory snapshot of the full transaction state.
+ */
+public class TransactionSnapshot implements TransactionVisibilityState {
+ private long timestamp;
+ private long readPointer;
+ private long writePointer;
+ private Collection<Long> invalid;
+ private NavigableMap<Long, TransactionManager.InProgressTx> inProgress;
+ private Map<Long, Set<ChangeId>> committingChangeSets;
+ private Map<Long, Set<ChangeId>> committedChangeSets;
+
+ public TransactionSnapshot(long timestamp, long readPointer, long writePointer, Collection<Long> invalid,
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress,
+ Map<Long, Set<ChangeId>> committing, Map<Long, Set<ChangeId>> committed) {
+ this(timestamp, readPointer, writePointer, invalid, inProgress);
+ this.committingChangeSets = committing;
+ this.committedChangeSets = committed;
+ }
+
+ public TransactionSnapshot(long timestamp, long readPointer, long writePointer, Collection<Long> invalid,
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress) {
+ this.timestamp = timestamp;
+ this.readPointer = readPointer;
+ this.writePointer = writePointer;
+ this.invalid = invalid;
+ this.inProgress = inProgress;
+ this.committingChangeSets = Collections.emptyMap();
+ this.committedChangeSets = Collections.emptyMap();
+ }
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public long getReadPointer() {
+ return readPointer;
+ }
+
+ @Override
+ public long getWritePointer() {
+ return writePointer;
+ }
+
+ @Override
+ public Collection<Long> getInvalid() {
+ return invalid;
+ }
+
+ @Override
+ public NavigableMap<Long, TransactionManager.InProgressTx> getInProgress() {
+ return inProgress;
+ }
+
+ @Override
+ public long getVisibilityUpperBound() {
+ // the readPointer of the oldest in-progress tx is the oldest in use
+ // todo: potential problem with not moving visibility upper bound for the whole duration of long-running tx
+ Map.Entry<Long, TransactionManager.InProgressTx> firstInProgress = inProgress.firstEntry();
+ if (firstInProgress == null) {
+ // using readPointer as smallest visible when non txs are there
+ return readPointer;
+ }
+ return firstInProgress.getValue().getVisibilityUpperBound();
+ }
+
+ /**
+ * Returns a map of transaction write pointer to sets of changed row keys for transactions that had called
+ * {@code InMemoryTransactionManager.canCommit(Transaction, Collection)} but not yet called
+ * {@code InMemoryTransactionManager.commit(Transaction)} at the time of the snapshot.
+ *
+ * @return a map of transaction write pointer to set of changed row keys.
+ */
+ public Map<Long, Set<ChangeId>> getCommittingChangeSets() {
+ return committingChangeSets;
+ }
+
+ /**
+ * Returns a map of transaction write pointer to set of changed row keys for transaction that had successfully called
+ * {@code InMemoryTransactionManager.commit(Transaction)} at the time of the snapshot.
+ *
+ * @return a map of transaction write pointer to set of changed row keys.
+ */
+ public Map<Long, Set<ChangeId>> getCommittedChangeSets() {
+ return committedChangeSets;
+ }
+
+ /**
+ * Checks that this instance matches another {@code TransactionSnapshot} instance. Note that the equality check
+ * ignores the snapshot timestamp value, but includes all other properties.
+ *
+ * @param obj the other instance to check for equality.
+ * @return {@code true} if the instances are equal, {@code false} if not.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof TransactionSnapshot)) {
+ return false;
+ }
+ TransactionSnapshot other = (TransactionSnapshot) obj;
+ return readPointer == other.readPointer &&
+ writePointer == other.writePointer &&
+ invalid.equals(other.invalid) &&
+ inProgress.equals(other.inProgress) &&
+ committingChangeSets.equals(other.committingChangeSets) &&
+ committedChangeSets.equals(other.committedChangeSets);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("timestamp", timestamp)
+ .add("readPointer", readPointer)
+ .add("writePointer", writePointer)
+ .add("invalidSize", invalid.size())
+ .add("inProgressSize", inProgress.size())
+ .add("committingSize", committingChangeSets.size())
+ .add("committedSize", committedChangeSets.size())
+ .toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(readPointer, writePointer, invalid, inProgress, committingChangeSets, committedChangeSets);
+ }
+
+ /**
+ * Creates a new {@code TransactionSnapshot} instance with copies of all of the individual collections.
+ * @param readPointer current transaction read pointer
+ * @param writePointer current transaction write pointer
+ * @param invalid current list of invalid write pointers
+ * @param inProgress current map of in-progress write pointers to expiration timestamps
+ * @param committing current map of write pointers to change sets which have passed {@code canCommit()} but not
+ * yet committed
+ * @param committed current map of write pointers to change sets which have committed
+ * @return a new {@code TransactionSnapshot} instance
+ */
+ public static TransactionSnapshot copyFrom(long snapshotTime, long readPointer,
+ long writePointer, Collection<Long> invalid,
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress,
+ Map<Long, Set<ChangeId>> committing,
+ NavigableMap<Long, Set<ChangeId>> committed) {
+ // copy invalid IDs
+ Collection<Long> invalidCopy = Lists.newArrayList(invalid);
+ // copy in-progress IDs and expirations
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgressCopy = Maps.newTreeMap(inProgress);
+
+ // for committing and committed maps, we need to copy each individual Set as well to prevent modification
+ Map<Long, Set<ChangeId>> committingCopy = Maps.newHashMap();
+ for (Map.Entry<Long, Set<ChangeId>> entry : committing.entrySet()) {
+ committingCopy.put(entry.getKey(), new HashSet<>(entry.getValue()));
+ }
+
+ NavigableMap<Long, Set<ChangeId>> committedCopy = new TreeMap<>();
+ for (Map.Entry<Long, Set<ChangeId>> entry : committed.entrySet()) {
+ committedCopy.put(entry.getKey(), new HashSet<>(entry.getValue()));
+ }
+
+ return new TransactionSnapshot(snapshotTime, readPointer, writePointer,
+ invalidCopy, inProgressCopy, committingCopy, committedCopy);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionStateStorage.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionStateStorage.java
new file mode 100644
index 0000000..f7edc8b
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionStateStorage.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.util.concurrent.Service;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+
+/**
+ * Defines the common contract for persisting transaction state changes.
+ */
+public interface TransactionStateStorage extends Service {
+
+ /**
+ * Persists a snapshot of transaction state to an output stream.
+ */
+ public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException;
+
+ /**
+ * Persists a snapshot of transaction state.
+ */
+ public void writeSnapshot(TransactionSnapshot snapshot) throws IOException;
+
+ /**
+ * Returns the most recent snapshot that has been successfully written. Note that this may return {@code null}
+ * if no completed snapshot files are found.
+ */
+ public TransactionSnapshot getLatestSnapshot() throws IOException;
+
+ /**
+ * Returns the most recent transaction visibility state that has been successfully written.
+ * Note that this may return {@code null} if no completed snapshot files are found.
+ * @return {@link TransactionVisibilityState}
+ */
+ public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException;
+
+ /**
+ * Removes any snapshots prior to the {@code numberToKeep} most recent.
+ *
+ * @param numberToKeep The number of most recent snapshots to keep.
+ * @throws IOException If an error occurs while deleting old snapshots.
+ * @return The timestamp of the oldest snapshot kept.
+ */
+ public long deleteOldSnapshots(int numberToKeep) throws IOException;
+
+ /**
+ * Returns the (non-qualified) names of available snapshots.
+ */
+ public List<String> listSnapshots() throws IOException;
+
+ /**
+ * Returns all {@link TransactionLog}s with a timestamp greater than or equal to the given timestamp. Note that
+ * the returned list is guaranteed to be sorted in ascending timestamp order.
+ */
+ public List<TransactionLog> getLogsSince(long timestamp) throws IOException;
+
+ /**
+ * Creates a new {@link TransactionLog}.
+ */
+ public TransactionLog createLog(long timestamp) throws IOException;
+
+ /**
+ * Returns the (non-qualified) names of available logs.
+ */
+ public List<String> listLogs() throws IOException;
+
+ /**
+ * Removes any transaction logs with a timestamp older than the given value. Logs must be removed based on timestamp
+ * to ensure we can fully recover state based on a given snapshot.
+ * @param timestamp The timestamp to delete up to. Logs with a timestamp less than this value will be removed.
+ * @throws IOException If an error occurs while removing logs.
+ */
+ public void deleteLogsOlderThan(long timestamp) throws IOException;
+
+ /**
+ * Create the directories required for the transaction state stage.
+ * @throws IOException If an error occurred during the creation of required directories for transaction state storage.
+ */
+ public void setupStorage() throws IOException;
+
+ /**
+ * Returns a string representation of the location used for persistence.
+ */
+ public String getLocation();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/TransactionVisibilityState.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/TransactionVisibilityState.java b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionVisibilityState.java
new file mode 100644
index 0000000..cf845af
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/TransactionVisibilityState.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import org.apache.tephra.TransactionManager;
+
+import java.util.Collection;
+import java.util.NavigableMap;
+
+/**
+ * Transaction Visibility state contains information required by TransactionProcessor CoProcessor
+ * to determine cell visibility.
+ */
+public interface TransactionVisibilityState {
+
+ /**
+ * Returns the timestamp from when this snapshot was created.
+ */
+ long getTimestamp();
+
+ /**
+ * Returns the read pointer at the time of the snapshot.
+ */
+ long getReadPointer();
+
+ /**
+ * Returns the next write pointer at the time of the snapshot.
+ */
+ long getWritePointer();
+
+ /**
+ * Returns the list of invalid write pointers at the time of the snapshot.
+ */
+ Collection<Long> getInvalid();
+
+ /**
+ * Returns the map of write pointers to in-progress transactions at the time of the snapshot.
+ */
+ NavigableMap<Long, TransactionManager.InProgressTx> getInProgress();
+
+ /**
+ * @return transaction id {@code X} such that any of the transactions newer than {@code X} might be invisible to
+ * some of the currently in-progress transactions or to those that will be started <p>
+ * NOTE: the returned tx id can be invalid.
+ */
+ long getVisibilityUpperBound();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/package-info.java b/tephra-core/src/main/java/org/apache/tephra/persist/package-info.java
new file mode 100644
index 0000000..01decb0
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains interfaces and implementations for persisting transaction state.
+ */
+package org.apache.tephra.persist;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/rpc/RPCServiceHandler.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/rpc/RPCServiceHandler.java b/tephra-core/src/main/java/org/apache/tephra/rpc/RPCServiceHandler.java
new file mode 100644
index 0000000..242a6fe
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/rpc/RPCServiceHandler.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.rpc;
+
+/**
+ * Defines lifecycle interface for all rpc handlers.
+ */
+public interface RPCServiceHandler {
+
+ void init() throws Exception;
+
+ void destroy() throws Exception;
+}
[30/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/TransactionContextTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/TransactionContextTest.java b/tephra-core/src/test/java/co/cask/tephra/TransactionContextTest.java
deleted file mode 100644
index 419bffb..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/TransactionContextTest.java
+++ /dev/null
@@ -1,676 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.snapshot.SnapshotCodecV4;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Singleton;
-import com.google.inject.util.Modules;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Tests the transaction executor.
- */
-public class TransactionContextTest {
- private static DummyTxClient txClient;
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @BeforeClass
- public static void setup() throws IOException {
- final Configuration conf = new Configuration();
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new DiscoveryModules().getInMemoryModules(),
- Modules.override(
- new TransactionModules().getInMemoryModules()).with(new AbstractModule() {
- @Override
- protected void configure() {
- TransactionManager txManager = new TransactionManager(conf);
- txManager.startAndWait();
- bind(TransactionManager.class).toInstance(txManager);
- bind(TransactionSystemClient.class).to(DummyTxClient.class).in(Singleton.class);
- }
- }));
-
- txClient = (DummyTxClient) injector.getInstance(TransactionSystemClient.class);
- }
-
- final DummyTxAware ds1 = new DummyTxAware(), ds2 = new DummyTxAware();
-
- static final byte[] A = { 'a' };
- static final byte[] B = { 'b' };
-
- private static TransactionContext newTransactionContext(TransactionAware... txAwares) {
- return new TransactionContext(txClient, txAwares);
- }
-
- @Before
- public void resetTxAwares() {
- ds1.reset();
- ds2.reset();
- }
-
- @Test
- public void testSuccessful() throws TransactionFailureException, InterruptedException {
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction
- context.finish();
- // verify both are committed and post-committed
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertTrue(ds1.postCommitted);
- Assert.assertTrue(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
- }
-
- @Test
- public void testPostCommitFailure() throws TransactionFailureException, InterruptedException {
- ds1.failPostCommitTxOnce = InduceFailure.ThrowException;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail but without rollback as the failure happens post-commit
- try {
- context.finish();
- Assert.fail("post commit failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("post failure", e.getCause().getMessage());
- }
- // verify both are committed and post-committed
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertTrue(ds1.postCommitted);
- Assert.assertTrue(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
- }
-
- @Test
- public void testPersistFailure() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ThrowException;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("Persist should have failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("persist failure", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testPersistFalse() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("Persist should have failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testPersistAndRollbackFailure() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ThrowException;
- ds1.failRollbackTxOnce = InduceFailure.ThrowException;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("Persist should have failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("persist failure", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is invalidated
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
- }
-
- @Test
- public void testPersistAndRollbackFalse() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
- ds1.failRollbackTxOnce = InduceFailure.ReturnFalse;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("Persist should have failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
- }
- // verify both are rolled back and tx is invalidated
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
- }
-
- @Test
- public void testCommitFalse() throws TransactionFailureException, InterruptedException {
- txClient.failCommits = 1;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("commit failed - exception should be thrown");
- } catch (TransactionConflictException e) {
- Assert.assertNull(e.getCause());
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testCanCommitFalse() throws TransactionFailureException, InterruptedException {
- txClient.failCanCommitOnce = true;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("commit failed - exception should be thrown");
- } catch (TransactionConflictException e) {
- Assert.assertNull(e.getCause());
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertFalse(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testChangesAndRollbackFailure() throws TransactionFailureException, InterruptedException {
- ds1.failChangesTxOnce = InduceFailure.ThrowException;
- ds1.failRollbackTxOnce = InduceFailure.ThrowException;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- context.start();
- // add a change to ds1 and ds2
- ds1.addChange(A);
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("get changes failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("changes failure", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is invalidated
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertFalse(ds2.checked);
- Assert.assertFalse(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
- }
-
- @Test
- public void testStartAndRollbackFailure() throws TransactionFailureException, InterruptedException {
- ds1.failStartTxOnce = InduceFailure.ThrowException;
- TransactionContext context = newTransactionContext(ds1, ds2);
- // start transaction
- try {
- context.start();
- Assert.fail("start failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("start failure", e.getCause().getMessage());
- }
- // verify both are not rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertFalse(ds2.started);
- Assert.assertFalse(ds1.checked);
- Assert.assertFalse(ds2.checked);
- Assert.assertFalse(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testAddThenSuccess() throws TransactionFailureException, InterruptedException {
- TransactionContext context = newTransactionContext(ds1);
- // start transaction
- context.start();
- // add a change to ds1
- ds1.addChange(A);
- // add ds2 to the tx
- context.addTransactionAware(ds2);
- // add a change to ds2
- ds2.addChange(B);
- // commit transaction
- context.finish();
- // verify both are committed and post-committed
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertTrue(ds1.postCommitted);
- Assert.assertTrue(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
- }
-
- @Test
- public void testAddThenFailure() throws TransactionFailureException, InterruptedException {
- ds2.failCommitTxOnce = InduceFailure.ThrowException;
-
- TransactionContext context = newTransactionContext(ds1);
- // start transaction
- context.start();
- // add a change to ds1
- ds1.addChange(A);
- // add ds2 to the tx
- context.addTransactionAware(ds2);
- // add a change to ds2
- ds2.addChange(B);
- // commit transaction should fail and cause rollback
- try {
- context.finish();
- Assert.fail("Persist should have failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("persist failure", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testAddThenRemoveSuccess() throws TransactionFailureException {
- TransactionContext context = newTransactionContext();
-
- context.start();
- Assert.assertTrue(context.addTransactionAware(ds1));
- ds1.addChange(A);
-
- try {
- context.removeTransactionAware(ds1);
- Assert.fail("Removal of TransactionAware should fails when there is active transaction.");
- } catch (IllegalStateException e) {
- // Expected
- }
-
- context.finish();
-
- Assert.assertTrue(context.removeTransactionAware(ds1));
- // Removing a TransactionAware not added before should returns false
- Assert.assertFalse(context.removeTransactionAware(ds2));
-
- // Verify ds1 is committed and post-committed
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds1.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
-
- // Verify nothing happen to ds2
- Assert.assertFalse(ds2.started);
- Assert.assertFalse(ds2.checked);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertFalse(ds2.rolledBack);
-
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
- }
-
- @Test
- public void testAndThenRemoveOnFailure() throws TransactionFailureException {
- ds1.failCommitTxOnce = InduceFailure.ThrowException;
- TransactionContext context = newTransactionContext();
-
- context.start();
- Assert.assertTrue(context.addTransactionAware(ds1));
- ds1.addChange(A);
-
- try {
- context.finish();
- Assert.fail("Persist should have failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("persist failure", e.getCause().getMessage());
- }
-
- Assert.assertTrue(context.removeTransactionAware(ds1));
-
- // Verify ds1 is rolled back
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
-
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- enum InduceFailure { NoFailure, ReturnFalse, ThrowException }
-
- static class DummyTxAware implements TransactionAware {
-
- Transaction tx;
- boolean started = false;
- boolean committed = false;
- boolean checked = false;
- boolean rolledBack = false;
- boolean postCommitted = false;
- List<byte[]> changes = Lists.newArrayList();
-
- InduceFailure failStartTxOnce = InduceFailure.NoFailure;
- InduceFailure failChangesTxOnce = InduceFailure.NoFailure;
- InduceFailure failCommitTxOnce = InduceFailure.NoFailure;
- InduceFailure failPostCommitTxOnce = InduceFailure.NoFailure;
- InduceFailure failRollbackTxOnce = InduceFailure.NoFailure;
-
- void addChange(byte[] key) {
- changes.add(key);
- }
-
- void reset() {
- tx = null;
- started = false;
- checked = false;
- committed = false;
- rolledBack = false;
- postCommitted = false;
- changes.clear();
- }
-
- @Override
- public void startTx(Transaction tx) {
- reset();
- started = true;
- this.tx = tx;
- if (failStartTxOnce == InduceFailure.ThrowException) {
- failStartTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("start failure");
- }
- }
-
- @Override
- public void updateTx(Transaction tx) {
- this.tx = tx;
- }
-
- @Override
- public Collection<byte[]> getTxChanges() {
- checked = true;
- if (failChangesTxOnce == InduceFailure.ThrowException) {
- failChangesTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("changes failure");
- }
- return ImmutableList.copyOf(changes);
- }
-
- @Override
- public boolean commitTx() throws Exception {
- committed = true;
- if (failCommitTxOnce == InduceFailure.ThrowException) {
- failCommitTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("persist failure");
- }
- if (failCommitTxOnce == InduceFailure.ReturnFalse) {
- failCommitTxOnce = InduceFailure.NoFailure;
- return false;
- }
- return true;
- }
-
- @Override
- public void postTxCommit() {
- postCommitted = true;
- if (failPostCommitTxOnce == InduceFailure.ThrowException) {
- failPostCommitTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("post failure");
- }
- }
-
- @Override
- public boolean rollbackTx() throws Exception {
- rolledBack = true;
- if (failRollbackTxOnce == InduceFailure.ThrowException) {
- failRollbackTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("rollback failure");
- }
- if (failRollbackTxOnce == InduceFailure.ReturnFalse) {
- failRollbackTxOnce = InduceFailure.NoFailure;
- return false;
- }
- return true;
- }
-
- @Override
- public String getTransactionAwareName() {
- return "dummy";
- }
- }
-
- static class DummyTxClient extends InMemoryTxSystemClient {
-
- boolean failCanCommitOnce = false;
- int failCommits = 0;
- enum CommitState {
- Started, Committed, Aborted, Invalidated
- }
- CommitState state = CommitState.Started;
-
- @Inject
- DummyTxClient(TransactionManager txmgr) {
- super(txmgr);
- }
-
- @Override
- public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
- if (failCanCommitOnce) {
- failCanCommitOnce = false;
- return false;
- } else {
- return super.canCommit(tx, changeIds);
- }
- }
-
- @Override
- public boolean commit(Transaction tx) throws TransactionNotInProgressException {
- if (failCommits-- > 0) {
- return false;
- } else {
- state = CommitState.Committed;
- return super.commit(tx);
- }
- }
-
- @Override
- public Transaction startLong() {
- state = CommitState.Started;
- return super.startLong();
- }
-
- @Override
- public Transaction startShort() {
- state = CommitState.Started;
- return super.startShort();
- }
-
- @Override
- public Transaction startShort(int timeout) {
- state = CommitState.Started;
- return super.startShort(timeout);
- }
-
- @Override
- public void abort(Transaction tx) {
- state = CommitState.Aborted;
- super.abort(tx);
- }
-
- @Override
- public boolean invalidate(long tx) {
- state = CommitState.Invalidated;
- return super.invalidate(tx);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/TransactionExecutorTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/TransactionExecutorTest.java b/tephra-core/src/test/java/co/cask/tephra/TransactionExecutorTest.java
deleted file mode 100644
index 7e67f2c..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/TransactionExecutorTest.java
+++ /dev/null
@@ -1,590 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.snapshot.DefaultSnapshotCodec;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Singleton;
-import com.google.inject.util.Modules;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import javax.annotation.Nullable;
-
-/**
- * Tests the transaction executor.
- */
-public class TransactionExecutorTest {
- private static DummyTxClient txClient;
- private static TransactionExecutorFactory factory;
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @BeforeClass
- public static void setup() throws IOException {
- final Configuration conf = new Configuration();
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new DiscoveryModules().getInMemoryModules(),
- Modules.override(
- new TransactionModules().getInMemoryModules()).with(new AbstractModule() {
- @Override
- protected void configure() {
- TransactionManager txManager = new TransactionManager(conf);
- txManager.startAndWait();
- bind(TransactionManager.class).toInstance(txManager);
- bind(TransactionSystemClient.class).to(DummyTxClient.class).in(Singleton.class);
- }
- }));
-
- txClient = (DummyTxClient) injector.getInstance(TransactionSystemClient.class);
- factory = injector.getInstance(TransactionExecutorFactory.class);
- }
-
- final DummyTxAware ds1 = new DummyTxAware(), ds2 = new DummyTxAware();
- final Collection<TransactionAware> txAwares = ImmutableList.<TransactionAware>of(ds1, ds2);
-
- private TransactionExecutor getExecutor() {
- return factory.createExecutor(txAwares);
- }
-
- private TransactionExecutor getExecutorWithNoRetry() {
- return new DefaultTransactionExecutor(txClient, txAwares, RetryStrategies.noRetries());
- }
-
- static final byte[] A = { 'a' };
- static final byte[] B = { 'b' };
-
- final TransactionExecutor.Function<Integer, Integer> testFunction =
- new TransactionExecutor.Function<Integer, Integer>() {
- @Override
- public Integer apply(@Nullable Integer input) {
- ds1.addChange(A);
- ds2.addChange(B);
- if (input == null) {
- throw new RuntimeException("function failed");
- }
- return input * input;
- }
- };
-
- @Before
- public void resetTxAwares() {
- ds1.reset();
- ds2.reset();
- }
-
- @Test
- public void testSuccessful() throws TransactionFailureException, InterruptedException {
- // execute: add a change to ds1 and ds2
- Integer result = getExecutor().execute(testFunction, 10);
- // verify both are committed and post-committed
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertTrue(ds1.postCommitted);
- Assert.assertTrue(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertTrue(100 == result);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
- }
-
- @Test
- public void testPostCommitFailure() throws TransactionFailureException, InterruptedException {
- ds1.failPostCommitTxOnce = InduceFailure.ThrowException;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, 10);
- Assert.fail("post commit failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("post failure", e.getCause().getMessage());
- }
- // verify both are committed and post-committed
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertTrue(ds1.postCommitted);
- Assert.assertTrue(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
- }
-
- @Test
- public void testPersistFailure() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ThrowException;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, 10);
- Assert.fail("persist failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("persist failure", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testPersistFalse() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, 10);
- Assert.fail("persist failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testPersistAndRollbackFailure() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ThrowException;
- ds1.failRollbackTxOnce = InduceFailure.ThrowException;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, 10);
- Assert.fail("persist failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("persist failure", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is invalidated
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
- }
-
- @Test
- public void testPersistAndRollbackFalse() throws TransactionFailureException, InterruptedException {
- ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
- ds1.failRollbackTxOnce = InduceFailure.ReturnFalse;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, 10);
- Assert.fail("persist failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
- }
- // verify both are rolled back and tx is invalidated
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
- }
-
- @Test
- public void testNoIndefiniteRetryByDefault() throws TransactionFailureException, InterruptedException {
- // we want retry by default, so that engineers don't miss it
- txClient.failCommits = 1000;
- try {
- // execute: add a change to ds1 and ds2
- getExecutor().execute(testFunction, 10);
- Assert.fail("commit failed too many times to retry - exception should be thrown");
- } catch (TransactionConflictException e) {
- Assert.assertNull(e.getCause());
- }
-
- txClient.failCommits = 0;
-
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testRetryByDefault() throws TransactionFailureException, InterruptedException {
- // we want retry by default, so that engineers don't miss it
- txClient.failCommits = 2;
- // execute: add a change to ds1 and ds2
- getExecutor().execute(testFunction, 10);
- // should not fail, but continue
-
- // verify both are committed
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertTrue(ds1.postCommitted);
- Assert.assertTrue(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
- }
-
- @Test
- public void testCommitFalse() throws TransactionFailureException, InterruptedException {
- txClient.failCommits = 1;
- // execute: add a change to ds1 and ds2
- try {
- getExecutorWithNoRetry().execute(testFunction, 10);
- Assert.fail("commit failed - exception should be thrown");
- } catch (TransactionConflictException e) {
- Assert.assertNull(e.getCause());
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertTrue(ds1.committed);
- Assert.assertTrue(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testCanCommitFalse() throws TransactionFailureException, InterruptedException {
- txClient.failCanCommitOnce = true;
- // execute: add a change to ds1 and ds2
- try {
- getExecutorWithNoRetry().execute(testFunction, 10);
- Assert.fail("commit failed - exception should be thrown");
- } catch (TransactionConflictException e) {
- Assert.assertNull(e.getCause());
- }
- // verify both are rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertTrue(ds2.checked);
- Assert.assertFalse(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- @Test
- public void testChangesAndRollbackFailure() throws TransactionFailureException, InterruptedException {
- ds1.failChangesTxOnce = InduceFailure.ThrowException;
- ds1.failRollbackTxOnce = InduceFailure.ThrowException;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, 10);
- Assert.fail("get changes failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("changes failure", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is invalidated
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertTrue(ds1.checked);
- Assert.assertFalse(ds2.checked);
- Assert.assertFalse(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
- }
-
- @Test
- public void testFunctionAndRollbackFailure() throws TransactionFailureException, InterruptedException {
- ds1.failRollbackTxOnce = InduceFailure.ReturnFalse;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, null);
- Assert.fail("function failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("function failed", e.getCause().getMessage());
- }
- // verify both are rolled back and tx is invalidated
- Assert.assertTrue(ds1.started);
- Assert.assertTrue(ds2.started);
- Assert.assertFalse(ds1.checked);
- Assert.assertFalse(ds2.checked);
- Assert.assertFalse(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertTrue(ds1.rolledBack);
- Assert.assertTrue(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
- }
-
- @Test
- public void testStartAndRollbackFailure() throws TransactionFailureException, InterruptedException {
- ds1.failStartTxOnce = InduceFailure.ThrowException;
- // execute: add a change to ds1 and ds2
- try {
- getExecutor().execute(testFunction, 10);
- Assert.fail("start failed - exception should be thrown");
- } catch (TransactionFailureException e) {
- Assert.assertEquals("start failure", e.getCause().getMessage());
- }
- // verify both are not rolled back and tx is aborted
- Assert.assertTrue(ds1.started);
- Assert.assertFalse(ds2.started);
- Assert.assertFalse(ds1.checked);
- Assert.assertFalse(ds2.checked);
- Assert.assertFalse(ds1.committed);
- Assert.assertFalse(ds2.committed);
- Assert.assertFalse(ds1.postCommitted);
- Assert.assertFalse(ds2.postCommitted);
- Assert.assertFalse(ds1.rolledBack);
- Assert.assertFalse(ds2.rolledBack);
- Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
- }
-
- enum InduceFailure { NoFailure, ReturnFalse, ThrowException }
-
- static class DummyTxAware implements TransactionAware {
-
- Transaction tx;
- boolean started = false;
- boolean committed = false;
- boolean checked = false;
- boolean rolledBack = false;
- boolean postCommitted = false;
- List<byte[]> changes = Lists.newArrayList();
-
- InduceFailure failStartTxOnce = InduceFailure.NoFailure;
- InduceFailure failChangesTxOnce = InduceFailure.NoFailure;
- InduceFailure failCommitTxOnce = InduceFailure.NoFailure;
- InduceFailure failPostCommitTxOnce = InduceFailure.NoFailure;
- InduceFailure failRollbackTxOnce = InduceFailure.NoFailure;
-
- void addChange(byte[] key) {
- changes.add(key);
- }
-
- void reset() {
- tx = null;
- started = false;
- checked = false;
- committed = false;
- rolledBack = false;
- postCommitted = false;
- changes.clear();
- }
-
- @Override
- public void startTx(Transaction tx) {
- reset();
- started = true;
- this.tx = tx;
- if (failStartTxOnce == InduceFailure.ThrowException) {
- failStartTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("start failure");
- }
- }
-
- @Override
- public void updateTx(Transaction tx) {
- this.tx = tx;
- }
-
- @Override
- public Collection<byte[]> getTxChanges() {
- checked = true;
- if (failChangesTxOnce == InduceFailure.ThrowException) {
- failChangesTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("changes failure");
- }
- return ImmutableList.copyOf(changes);
- }
-
- @Override
- public boolean commitTx() throws Exception {
- committed = true;
- if (failCommitTxOnce == InduceFailure.ThrowException) {
- failCommitTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("persist failure");
- }
- if (failCommitTxOnce == InduceFailure.ReturnFalse) {
- failCommitTxOnce = InduceFailure.NoFailure;
- return false;
- }
- return true;
- }
-
- @Override
- public void postTxCommit() {
- postCommitted = true;
- if (failPostCommitTxOnce == InduceFailure.ThrowException) {
- failPostCommitTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("post failure");
- }
- }
-
- @Override
- public boolean rollbackTx() throws Exception {
- rolledBack = true;
- if (failRollbackTxOnce == InduceFailure.ThrowException) {
- failRollbackTxOnce = InduceFailure.NoFailure;
- throw new RuntimeException("rollback failure");
- }
- if (failRollbackTxOnce == InduceFailure.ReturnFalse) {
- failRollbackTxOnce = InduceFailure.NoFailure;
- return false;
- }
- return true;
- }
-
- @Override
- public String getTransactionAwareName() {
- return "dummy";
- }
- }
-
- static class DummyTxClient extends InMemoryTxSystemClient {
-
- boolean failCanCommitOnce = false;
- int failCommits = 0;
- enum CommitState {
- Started, Committed, Aborted, Invalidated
- }
- CommitState state = CommitState.Started;
-
- @Inject
- DummyTxClient(TransactionManager txmgr) {
- super(txmgr);
- }
-
- @Override
- public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
- if (failCanCommitOnce) {
- failCanCommitOnce = false;
- return false;
- } else {
- return super.canCommit(tx, changeIds);
- }
- }
-
- @Override
- public boolean commit(Transaction tx) throws TransactionNotInProgressException {
- if (failCommits-- > 0) {
- return false;
- } else {
- state = CommitState.Committed;
- return super.commit(tx);
- }
- }
-
- @Override
- public Transaction startLong() {
- state = CommitState.Started;
- return super.startLong();
- }
-
- @Override
- public Transaction startShort() {
- state = CommitState.Started;
- return super.startShort();
- }
-
- @Override
- public Transaction startShort(int timeout) {
- state = CommitState.Started;
- return super.startShort(timeout);
- }
-
- @Override
- public void abort(Transaction tx) {
- state = CommitState.Aborted;
- super.abort(tx);
- }
-
- @Override
- public boolean invalidate(long tx) {
- state = CommitState.Invalidated;
- return super.invalidate(tx);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/TransactionManagerTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/TransactionManagerTest.java b/tephra-core/src/test/java/co/cask/tephra/TransactionManagerTest.java
deleted file mode 100644
index ddd32db..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/TransactionManagerTest.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public class TransactionManagerTest extends TransactionSystemTest {
-
- static Configuration conf = new Configuration();
-
- TransactionManager txManager = null;
- TransactionStateStorage txStateStorage = null;
-
- @Override
- protected TransactionSystemClient getClient() {
- return new InMemoryTxSystemClient(txManager);
- }
-
- @Override
- protected TransactionStateStorage getStateStorage() {
- return txStateStorage;
- }
-
- @Before
- public void before() {
- conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
- // todo should create two sets of tests, one with LocalFileTxStateStorage and one with InMemoryTxStateStorage
- txStateStorage = new InMemoryTransactionStateStorage();
- txManager = new TransactionManager
- (conf, txStateStorage, new TxMetricsCollector());
- txManager.startAndWait();
- }
-
- @After
- public void after() {
- txManager.stopAndWait();
- }
-
- @Test
- public void testTransactionCleanup() throws Exception {
- conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 3);
- conf.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, 2);
- // using a new tx manager that cleans up
- TransactionManager txm = new TransactionManager
- (conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
- txm.startAndWait();
- try {
- Assert.assertEquals(0, txm.getInvalidSize());
- Assert.assertEquals(0, txm.getCommittedSize());
- // start a transaction and leave it open
- Transaction tx1 = txm.startShort();
- // start a long running transaction and leave it open
- Transaction tx2 = txm.startLong();
- Transaction tx3 = txm.startLong();
- // start and commit a bunch of transactions
- for (int i = 0; i < 10; i++) {
- Transaction tx = txm.startShort();
- Assert.assertTrue(txm.canCommit(tx, Collections.singleton(new byte[] { (byte) i })));
- Assert.assertTrue(txm.commit(tx));
- }
- // all of these should still be in the committed set
- Assert.assertEquals(0, txm.getInvalidSize());
- Assert.assertEquals(10, txm.getCommittedSize());
- // sleep longer than the cleanup interval
- TimeUnit.SECONDS.sleep(5);
- // transaction should now be invalid
- Assert.assertEquals(1, txm.getInvalidSize());
- // run another transaction
- Transaction txx = txm.startShort();
- // verify the exclude
- Assert.assertFalse(txx.isVisible(tx1.getTransactionId()));
- Assert.assertFalse(txx.isVisible(tx2.getTransactionId()));
- Assert.assertFalse(txx.isVisible(tx3.getTransactionId()));
- // try to commit the last transaction that was started
- Assert.assertTrue(txm.canCommit(txx, Collections.singleton(new byte[] { 0x0a })));
- Assert.assertTrue(txm.commit(txx));
-
- // now the committed change sets should be empty again
- Assert.assertEquals(0, txm.getCommittedSize());
- // cannot commit transaction as it was timed out
- try {
- txm.canCommit(tx1, Collections.singleton(new byte[] { 0x11 }));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
- txm.abort(tx1);
- // abort should have removed from invalid
- Assert.assertEquals(0, txm.getInvalidSize());
- // run another bunch of transactions
- for (int i = 0; i < 10; i++) {
- Transaction tx = txm.startShort();
- Assert.assertTrue(txm.canCommit(tx, Collections.singleton(new byte[] { (byte) i })));
- Assert.assertTrue(txm.commit(tx));
- }
- // none of these should still be in the committed set (tx2 is long-running).
- Assert.assertEquals(0, txm.getInvalidSize());
- Assert.assertEquals(0, txm.getCommittedSize());
- // commit tx2, abort tx3
- Assert.assertTrue(txm.commit(tx2));
- txm.abort(tx3);
- // none of these should still be in the committed set (tx2 is long-running).
- // Only tx3 is invalid list as it was aborted and is long-running. tx1 is short one and it rolled back its changes
- // so it should NOT be in invalid list
- Assert.assertEquals(1, txm.getInvalidSize());
- Assert.assertEquals(tx3.getTransactionId(), (long) txm.getCurrentState().getInvalid().iterator().next());
- Assert.assertEquals(1, txm.getExcludedListSize());
- } finally {
- txm.stopAndWait();
- }
- }
-
- @Test
- public void testLongTransactionCleanup() throws Exception {
- conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 3);
- conf.setInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT, 2);
- // using a new tx manager that cleans up
- TransactionManager txm = new TransactionManager
- (conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
- txm.startAndWait();
- try {
- Assert.assertEquals(0, txm.getInvalidSize());
- Assert.assertEquals(0, txm.getCommittedSize());
-
- // start a long running transaction
- Transaction tx1 = txm.startLong();
-
- Assert.assertEquals(0, txm.getInvalidSize());
- Assert.assertEquals(0, txm.getCommittedSize());
-
- // sleep longer than the cleanup interval
- TimeUnit.SECONDS.sleep(5);
-
- // transaction should now be invalid
- Assert.assertEquals(1, txm.getInvalidSize());
- Assert.assertEquals(0, txm.getCommittedSize());
-
- // cannot commit transaction as it was timed out
- try {
- txm.canCommit(tx1, Collections.singleton(new byte[] { 0x11 }));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
-
- txm.abort(tx1);
- // abort should not remove long running transaction from invalid list
- Assert.assertEquals(1, txm.getInvalidSize());
- } finally {
- txm.stopAndWait();
- }
- }
-
- @Test
- public void testTruncateInvalid() throws Exception {
- InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage();
- Configuration testConf = new Configuration(conf);
- // No snapshots
- testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1);
- TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector());
- txm1.startAndWait();
-
- TransactionManager txm2 = null;
- Transaction tx1;
- Transaction tx2;
- Transaction tx3;
- Transaction tx4;
- Transaction tx5;
- Transaction tx6;
- try {
- Assert.assertEquals(0, txm1.getInvalidSize());
-
- // start a few transactions
- tx1 = txm1.startLong();
- tx2 = txm1.startShort();
- tx3 = txm1.startLong();
- tx4 = txm1.startShort();
- tx5 = txm1.startLong();
- tx6 = txm1.startShort();
-
- // invalidate tx1, tx2, tx5 and tx6
- txm1.invalidate(tx1.getTransactionId());
- txm1.invalidate(tx2.getTransactionId());
- txm1.invalidate(tx5.getTransactionId());
- txm1.invalidate(tx6.getTransactionId());
-
- // tx1, tx2, tx5 and tx6 should be in invalid list
- Assert.assertEquals(
- ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(), tx5.getTransactionId(),
- tx6.getTransactionId()),
- txm1.getCurrentState().getInvalid()
- );
-
- // remove tx1 and tx6 from invalid list
- Assert.assertTrue(txm1.truncateInvalidTx(ImmutableSet.of(tx1.getTransactionId(), tx6.getTransactionId())));
-
- // only tx2 and tx5 should be in invalid list now
- Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
- txm1.getCurrentState().getInvalid());
-
- // removing in-progress transactions should not have any effect
- Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
- txm1.getCurrentState().getInProgress().keySet());
- Assert.assertFalse(txm1.truncateInvalidTx(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId())));
- // no change to in-progress
- Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
- txm1.getCurrentState().getInProgress().keySet());
- // no change to invalid list
- Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
- txm1.getCurrentState().getInvalid());
-
- // Test transaction edit logs replay
- // Start another transaction manager without stopping txm1 so that snapshot does not get written,
- // and all logs can be replayed.
- txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector());
- txm2.startAndWait();
- Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
- txm2.getCurrentState().getInvalid());
- Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
- txm2.getCurrentState().getInProgress().keySet());
- } finally {
- txm1.stopAndWait();
- if (txm2 != null) {
- txm2.stopAndWait();
- }
- }
- }
-
- @Test
- public void testTruncateInvalidBeforeTime() throws Exception {
- InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage();
- Configuration testConf = new Configuration(conf);
- // No snapshots
- testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1);
- TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector());
- txm1.startAndWait();
-
- TransactionManager txm2 = null;
- Transaction tx1;
- Transaction tx2;
- Transaction tx3;
- Transaction tx4;
- Transaction tx5;
- Transaction tx6;
- try {
- Assert.assertEquals(0, txm1.getInvalidSize());
-
- // start a few transactions
- tx1 = txm1.startLong();
- tx2 = txm1.startShort();
- // Sleep so that transaction ids get generated a millisecond apart for assertion
- // TEPHRA-63 should eliminate the need to sleep
- TimeUnit.MILLISECONDS.sleep(1);
- long timeBeforeTx3 = System.currentTimeMillis();
- tx3 = txm1.startLong();
- tx4 = txm1.startShort();
- TimeUnit.MILLISECONDS.sleep(1);
- long timeBeforeTx5 = System.currentTimeMillis();
- tx5 = txm1.startLong();
- tx6 = txm1.startShort();
-
- // invalidate tx1, tx2, tx5 and tx6
- txm1.invalidate(tx1.getTransactionId());
- txm1.invalidate(tx2.getTransactionId());
- txm1.invalidate(tx5.getTransactionId());
- txm1.invalidate(tx6.getTransactionId());
-
- // tx1, tx2, tx5 and tx6 should be in invalid list
- Assert.assertEquals(
- ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(), tx5.getTransactionId(),
- tx6.getTransactionId()),
- txm1.getCurrentState().getInvalid()
- );
-
- // remove transactions before tx3 from invalid list
- Assert.assertTrue(txm1.truncateInvalidTxBefore(timeBeforeTx3));
-
- // only tx5 and tx6 should be in invalid list now
- Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
- txm1.getCurrentState().getInvalid());
-
- // removing invalid transactions before tx5 should throw exception since tx3 and tx4 are in-progress
- Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
- txm1.getCurrentState().getInProgress().keySet());
- try {
- txm1.truncateInvalidTxBefore(timeBeforeTx5);
- Assert.fail("Expected InvalidTruncateTimeException exception");
- } catch (InvalidTruncateTimeException e) {
- // Expected exception
- }
- // no change to in-progress
- Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
- txm1.getCurrentState().getInProgress().keySet());
- // no change to invalid list
- Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
- txm1.getCurrentState().getInvalid());
-
- // Test transaction edit logs replay
- // Start another transaction manager without stopping txm1 so that snapshot does not get written,
- // and all logs can be replayed.
- txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector());
- txm2.startAndWait();
- Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
- txm2.getCurrentState().getInvalid());
- Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
- txm2.getCurrentState().getInProgress().keySet());
- } finally {
- txm1.stopAndWait();
- if (txm2 != null) {
- txm2.stopAndWait();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/TransactionServiceMainTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/TransactionServiceMainTest.java b/tephra-core/src/test/java/co/cask/tephra/TransactionServiceMainTest.java
deleted file mode 100644
index e54956d..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/TransactionServiceMainTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.distributed.TransactionServiceClient;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.internal.zookeeper.InMemoryZKServer;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.util.concurrent.CountDownLatch;
-
-/**
- * Test for verifying TransactionServiceMain works correctly.
- */
-public class TransactionServiceMainTest {
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Test
- public void testClientServer() throws Exception {
- // Simply start a transaction server and connect to it with the client.
- InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
- zkServer.startAndWait();
-
- try {
- Configuration conf = new Configuration();
- conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
-
- final TransactionServiceMain main = new TransactionServiceMain(conf);
- final CountDownLatch latch = new CountDownLatch(1);
- Thread t = new Thread() {
- @Override
- public void run() {
- try {
- main.start();
- latch.countDown();
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
- };
-
- try {
- t.start();
- // Wait for service to startup
- latch.await();
- TransactionServiceClient.doMain(true, conf);
- } finally {
- main.stop();
- t.join();
- }
- } finally {
- zkServer.stopAndWait();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/TransactionSystemTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/TransactionSystemTest.java b/tephra-core/src/test/java/co/cask/tephra/TransactionSystemTest.java
deleted file mode 100644
index cca6f5a..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/TransactionSystemTest.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionStateStorage;
-import com.google.common.collect.ImmutableSet;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public abstract class TransactionSystemTest {
-
- public static final byte[] C1 = new byte[] { 'c', '1' };
- public static final byte[] C2 = new byte[] { 'c', '2' };
- public static final byte[] C3 = new byte[] { 'c', '3' };
- public static final byte[] C4 = new byte[] { 'c', '4' };
-
- protected abstract TransactionSystemClient getClient() throws Exception;
-
- protected abstract TransactionStateStorage getStateStorage() throws Exception;
-
- @Test
- public void testCommitRaceHandling() throws Exception {
- TransactionSystemClient client1 = getClient();
- TransactionSystemClient client2 = getClient();
-
- Transaction tx1 = client1.startShort();
- Transaction tx2 = client2.startShort();
-
- Assert.assertTrue(client1.canCommit(tx1, asList(C1, C2)));
- // second one also can commit even thought there are conflicts with first since first one hasn't committed yet
- Assert.assertTrue(client2.canCommit(tx2, asList(C2, C3)));
-
- Assert.assertTrue(client1.commit(tx1));
-
- // now second one should not commit, since there are conflicts with tx1 that has been committed
- Assert.assertFalse(client2.commit(tx2));
- }
-
- @Test
- public void testMultipleCommitsAtSameTime() throws Exception {
- // We want to check that if two txs finish at same time (wrt tx manager) they do not overwrite changesets of each
- // other in tx manager used for conflicts detection (we had this bug)
- // NOTE: you don't have to use multiple clients for that
- TransactionSystemClient client1 = getClient();
- TransactionSystemClient client2 = getClient();
- TransactionSystemClient client3 = getClient();
- TransactionSystemClient client4 = getClient();
- TransactionSystemClient client5 = getClient();
-
- Transaction tx1 = client1.startShort();
- Transaction tx2 = client2.startShort();
- Transaction tx3 = client3.startShort();
- Transaction tx4 = client4.startShort();
- Transaction tx5 = client5.startShort();
-
- Assert.assertTrue(client1.canCommit(tx1, asList(C1)));
- Assert.assertTrue(client1.commit(tx1));
-
- Assert.assertTrue(client2.canCommit(tx2, asList(C2)));
- Assert.assertTrue(client2.commit(tx2));
-
- // verifying conflicts detection
- Assert.assertFalse(client3.canCommit(tx3, asList(C1)));
- Assert.assertFalse(client4.canCommit(tx4, asList(C2)));
- Assert.assertTrue(client5.canCommit(tx5, asList(C3)));
- }
-
- @Test
- public void testCommitTwice() throws Exception {
- TransactionSystemClient client = getClient();
- Transaction tx = client.startShort();
-
- Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
- Assert.assertTrue(client.commit(tx));
- // cannot commit twice same tx
- try {
- Assert.assertFalse(client.commit(tx));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
- }
-
- @Test
- public void testAbortTwice() throws Exception {
- TransactionSystemClient client = getClient();
- Transaction tx = client.startShort();
-
- Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
- client.abort(tx);
- // abort of not active tx has no affect
- client.abort(tx);
- }
-
- @Test
- public void testReuseTx() throws Exception {
- TransactionSystemClient client = getClient();
- Transaction tx = client.startShort();
-
- Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
- Assert.assertTrue(client.commit(tx));
- // can't re-use same tx again
- try {
- client.canCommit(tx, asList(C3, C4));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
- try {
- Assert.assertFalse(client.commit(tx));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
-
- // abort of not active tx has no affect
- client.abort(tx);
- }
-
- @Test
- public void testUseNotStarted() throws Exception {
- TransactionSystemClient client = getClient();
- Transaction tx1 = client.startShort();
- Assert.assertTrue(client.commit(tx1));
-
- // we know this is one is older than current writePointer and was not used
- Transaction txOld = new Transaction(tx1.getReadPointer(), tx1.getTransactionId() - 1,
- new long[] {}, new long[] {}, Transaction.NO_TX_IN_PROGRESS,
- TransactionType.SHORT);
- try {
- Assert.assertFalse(client.canCommit(txOld, asList(C3, C4)));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
- try {
- Assert.assertFalse(client.commit(txOld));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
- // abort of not active tx has no affect
- client.abort(txOld);
-
- // we know this is one is newer than current readPointer and was not used
- Transaction txNew = new Transaction(tx1.getReadPointer(), tx1.getTransactionId() + 1,
- new long[] {}, new long[] {}, Transaction.NO_TX_IN_PROGRESS,
- TransactionType.SHORT);
- try {
- Assert.assertFalse(client.canCommit(txNew, asList(C3, C4)));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
- try {
- Assert.assertFalse(client.commit(txNew));
- Assert.fail();
- } catch (TransactionNotInProgressException e) {
- // expected
- }
- // abort of not active tx has no affect
- client.abort(txNew);
- }
-
- @Test
- public void testAbortAfterCommit() throws Exception {
- TransactionSystemClient client = getClient();
- Transaction tx = client.startShort();
-
- Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
- Assert.assertTrue(client.commit(tx));
- // abort of not active tx has no affect
- client.abort(tx);
- }
-
- // todo add test invalidate method
- @Test
- public void testInvalidateTx() throws Exception {
- TransactionSystemClient client = getClient();
- // Invalidate an in-progress tx
- Transaction tx1 = client.startShort();
- client.canCommit(tx1, asList(C1, C2));
- Assert.assertTrue(client.invalidate(tx1.getTransactionId()));
- // Cannot invalidate a committed tx
- Transaction tx2 = client.startShort();
- client.canCommit(tx2, asList(C3, C4));
- client.commit(tx2);
- Assert.assertFalse(client.invalidate(tx2.getTransactionId()));
- }
-
- @Test
- public void testResetState() throws Exception {
- // have tx in progress, committing and committed then reset,
- // get the last snapshot and see that it is empty
- TransactionSystemClient client = getClient();
- TransactionStateStorage stateStorage = getStateStorage();
-
- Transaction tx1 = client.startShort();
- Transaction tx2 = client.startShort();
- client.canCommit(tx1, asList(C1, C2));
- client.commit(tx1);
- client.canCommit(tx2, asList(C3, C4));
-
- Transaction txPreReset = client.startShort();
- long currentTs = System.currentTimeMillis();
- client.resetState();
-
- TransactionSnapshot snapshot = stateStorage.getLatestSnapshot();
- Assert.assertTrue(snapshot.getTimestamp() >= currentTs);
- Assert.assertEquals(0, snapshot.getInvalid().size());
- Assert.assertEquals(0, snapshot.getInProgress().size());
- Assert.assertEquals(0, snapshot.getCommittingChangeSets().size());
- Assert.assertEquals(0, snapshot.getCommittedChangeSets().size());
-
- // confirm that transaction IDs are not reset
- Transaction txPostReset = client.startShort();
- Assert.assertTrue("New tx ID should be greater than last ID before reset",
- txPostReset.getTransactionId() > txPreReset.getTransactionId());
- }
-
- @Test
- public void testTruncateInvalidTx() throws Exception {
- // Start few transactions and invalidate all of them
- TransactionSystemClient client = getClient();
- Transaction tx1 = client.startLong();
- Transaction tx2 = client.startShort();
- Transaction tx3 = client.startLong();
-
- client.invalidate(tx1.getTransactionId());
- client.invalidate(tx2.getTransactionId());
- client.invalidate(tx3.getTransactionId());
-
- // Remove tx2 and tx3 from invalid list
- Assert.assertTrue(client.truncateInvalidTx(ImmutableSet.of(tx2.getTransactionId(), tx3.getTransactionId())));
-
- Transaction tx = client.startShort();
- // Only tx1 should be in invalid list now
- Assert.assertArrayEquals(new long[] {tx1.getTransactionId()}, tx.getInvalids());
- client.abort(tx);
- }
-
- @Test
- public void testTruncateInvalidTxBefore() throws Exception {
- // Start few transactions
- TransactionSystemClient client = getClient();
- Transaction tx1 = client.startLong();
- Transaction tx2 = client.startShort();
- // Sleep so that transaction ids get generated a millisecond apart for assertion
- // TEPHRA-63 should eliminate the need to sleep
- TimeUnit.MILLISECONDS.sleep(1);
- long beforeTx3 = System.currentTimeMillis();
- Transaction tx3 = client.startLong();
-
- try {
- // throws exception since tx1 and tx2 are still in-progress
- client.truncateInvalidTxBefore(beforeTx3);
- Assert.fail("Expected InvalidTruncateTimeException exception");
- } catch (InvalidTruncateTimeException e) {
- // Expected exception
- }
-
- // Invalidate all of them
- client.invalidate(tx1.getTransactionId());
- client.invalidate(tx2.getTransactionId());
- client.invalidate(tx3.getTransactionId());
-
- // Remove transactions before time beforeTx3
- Assert.assertTrue(client.truncateInvalidTxBefore(beforeTx3));
-
- Transaction tx = client.startShort();
- // Only tx3 should be in invalid list now
- Assert.assertArrayEquals(new long[] {tx3.getTransactionId()}, tx.getInvalids());
- client.abort(tx);
- }
-
- @Test
- public void testGetInvalidSize() throws Exception {
- // Start few transactions and invalidate all of them
- TransactionSystemClient client = getClient();
- Transaction tx1 = client.startLong();
- Transaction tx2 = client.startShort();
- Transaction tx3 = client.startLong();
-
- Assert.assertEquals(0, client.getInvalidSize());
-
- client.invalidate(tx1.getTransactionId());
- client.invalidate(tx2.getTransactionId());
- client.invalidate(tx3.getTransactionId());
-
- Assert.assertEquals(3, client.getInvalidSize());
- }
-
- private Collection<byte[]> asList(byte[]... val) {
- return Arrays.asList(val);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/TransactionTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/TransactionTest.java b/tephra-core/src/test/java/co/cask/tephra/TransactionTest.java
deleted file mode 100644
index b1c7e43..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/TransactionTest.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.primitives.Longs;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Set;
-
-/**
- *
- */
-public class TransactionTest {
- // Current transaction id
- private final long txId = 200L;
- // Read pointer for current transaction
- private final long readPointer = 100L;
- // Transactions committed before current transaction was started.
- private final Set<Long> priorCommitted = ImmutableSet.of(80L, 99L, 100L);
- // Transactions committed after current transaction was started.
- private final Set<Long> postCommitted = ImmutableSet.of(150L, 180L, 210L);
- // Invalid transactions before current transaction was started.
- private final Set<Long> priorInvalids = ImmutableSet.of(90L, 110L, 190L);
- // Invalid transactions after current transaction was started.
- private final Set<Long> postInvalids = ImmutableSet.of(201L, 221L, 231L);
- // Transactions in progress before current transaction was started.
- private final Set<Long> priorInProgress = ImmutableSet.of(95L, 120L, 150L);
- // Transactions in progress after current transaction was started.
- private final Set<Long> postInProgress = ImmutableSet.of(205L, 215L, 300L);
-
- @Test
- public void testSnapshotVisibility() throws Exception {
- Transaction.VisibilityLevel visibilityLevel = Transaction.VisibilityLevel.SNAPSHOT;
-
- Set<Long> checkPointers = ImmutableSet.of(220L, 250L);
- Transaction tx = new Transaction(readPointer, txId, 250L, toSortedArray(priorInvalids),
- toSortedArray(priorInProgress), 95L,
- TransactionType.SHORT, toSortedArray(checkPointers),
- visibilityLevel);
- Set<Long> visibleCurrent = ImmutableSet.of(200L, 220L, 250L);
- Set<Long> notVisibleCurrent = ImmutableSet.of();
- assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
- visibleCurrent, notVisibleCurrent, tx);
-
- checkPointers = ImmutableSet.of();
- tx = new Transaction(readPointer, txId, txId, toSortedArray(priorInvalids), toSortedArray(priorInProgress), 95L,
- TransactionType.SHORT, toSortedArray(checkPointers),
- visibilityLevel);
- visibleCurrent = ImmutableSet.of(txId);
- notVisibleCurrent = ImmutableSet.of();
- assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
- visibleCurrent, notVisibleCurrent, tx);
- }
-
- @Test
- public void testSnapshotExcludeVisibility() throws Exception {
- Transaction.VisibilityLevel visibilityLevel = Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT;
-
- Set<Long> checkPointers = ImmutableSet.of(220L, 250L);
- Transaction tx = new Transaction(readPointer, txId, 250L, toSortedArray(priorInvalids),
- toSortedArray(priorInProgress), 95L,
- TransactionType.SHORT, toSortedArray(checkPointers),
- visibilityLevel);
- Set<Long> visibleCurrent = ImmutableSet.of(200L, 220L);
- Set<Long> notVisibleCurrent = ImmutableSet.of(250L);
- assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
- visibleCurrent, notVisibleCurrent, tx);
-
- checkPointers = ImmutableSet.of();
- tx = new Transaction(readPointer, txId, txId, toSortedArray(priorInvalids), toSortedArray(priorInProgress), 95L,
- TransactionType.SHORT, toSortedArray(checkPointers),
- visibilityLevel);
- visibleCurrent = ImmutableSet.of();
- notVisibleCurrent = ImmutableSet.of(txId);
- assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
- visibleCurrent, notVisibleCurrent, tx);
- }
-
- @Test
- public void testSnapshotAllVisibility() throws Exception {
- Transaction.VisibilityLevel visibilityLevel = Transaction.VisibilityLevel.SNAPSHOT_ALL;
-
- Set<Long> checkPointers = ImmutableSet.of(220L, 250L);
- Transaction tx = new Transaction(readPointer, txId, 250L, toSortedArray(priorInvalids),
- toSortedArray(priorInProgress), 95L,
- TransactionType.SHORT, toSortedArray(checkPointers),
- visibilityLevel);
- Set<Long> visibleCurrent = ImmutableSet.of(200L, 220L, 250L);
- Set<Long> notVisibleCurrent = ImmutableSet.of();
- assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
- visibleCurrent, notVisibleCurrent, tx);
-
- checkPointers = ImmutableSet.of();
- tx = new Transaction(readPointer, txId, txId, toSortedArray(priorInvalids),
- toSortedArray(priorInProgress), 95L,
- TransactionType.SHORT, toSortedArray(checkPointers),
- visibilityLevel);
- visibleCurrent = ImmutableSet.of(txId);
- notVisibleCurrent = ImmutableSet.of();
- assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
- visibleCurrent, notVisibleCurrent, tx);
- }
-
- private void assertVisibility(Set<Long> priorCommitted, Set<Long> postCommitted, Set<Long> priorInvalids,
- Set<Long> postInvalids, Set<Long> priorInProgress, Set<Long> postInProgress,
- Set<Long> visibleCurrent, Set<Long> notVisibleCurrent,
- Transaction tx) {
- // Verify visible snapshots of tx are visible
- for (long t : visibleCurrent) {
- Assert.assertTrue("Assertion error for version = " + t, tx.isVisible(t));
- }
-
- // Verify not visible snapshots of tx are not visible
- for (long t : notVisibleCurrent) {
- Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t));
- }
-
- // Verify prior committed versions are visible
- for (long t : priorCommitted) {
- Assert.assertTrue("Assertion error for version = " + t, tx.isVisible(t));
- }
-
- // Verify versions committed after tx started, and not part of tx are not visible
- for (long t : postCommitted) {
- Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t));
- }
-
- // Verify invalid and in-progress versions are not visible
- for (long t : Iterables.concat(priorInvalids, postInvalids, priorInProgress, postInProgress)) {
- Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t));
- }
- }
-
- private long[] toSortedArray(Set<Long> set) {
- long[] array = Longs.toArray(set);
- Arrays.sort(array);
- return array;
- }
-}
[26/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java b/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java
new file mode 100644
index 0000000..628bced
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/AbstractTransactionStateStorageTest.java
@@ -0,0 +1,555 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.util.TransactionEditUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Commons tests to run against the {@link TransactionStateStorage} implementations.
+ */
+public abstract class AbstractTransactionStateStorageTest {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionStateStorageTest.class);
+ private static Random random = new Random();
+
+ protected abstract Configuration getConfiguration(String testName) throws IOException;
+
+ protected abstract AbstractTransactionStateStorage getStorage(Configuration conf);
+
+ @Test
+ public void testSnapshotPersistence() throws Exception {
+ Configuration conf = getConfiguration("testSnapshotPersistence");
+
+ TransactionSnapshot snapshot = createRandomSnapshot();
+ TransactionStateStorage storage = getStorage(conf);
+ try {
+ storage.startAndWait();
+ storage.writeSnapshot(snapshot);
+
+ TransactionSnapshot readSnapshot = storage.getLatestSnapshot();
+ assertNotNull(readSnapshot);
+ assertEquals(snapshot, readSnapshot);
+ } finally {
+ storage.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testLogWriteAndRead() throws Exception {
+ Configuration conf = getConfiguration("testLogWriteAndRead");
+
+ // create some random entries
+ List<TransactionEdit> edits = TransactionEditUtil.createRandomEdits(100);
+ TransactionStateStorage storage = getStorage(conf);
+ try {
+ long now = System.currentTimeMillis();
+ storage.startAndWait();
+ TransactionLog log = storage.createLog(now);
+ for (TransactionEdit edit : edits) {
+ log.append(edit);
+ }
+ log.close();
+
+ Collection<TransactionLog> logsToRead = storage.getLogsSince(now);
+ // should only be our one log
+ assertNotNull(logsToRead);
+ assertEquals(1, logsToRead.size());
+ TransactionLogReader logReader = logsToRead.iterator().next().getReader();
+ assertNotNull(logReader);
+
+ List<TransactionEdit> readEdits = Lists.newArrayListWithExpectedSize(edits.size());
+ TransactionEdit nextEdit;
+ while ((nextEdit = logReader.next()) != null) {
+ readEdits.add(nextEdit);
+ }
+ logReader.close();
+ assertEquals(edits.size(), readEdits.size());
+ for (int i = 0; i < edits.size(); i++) {
+ LOG.info("Checking edit " + i);
+ assertEquals(edits.get(i), readEdits.get(i));
+ }
+ } finally {
+ storage.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testTransactionManagerPersistence() throws Exception {
+ Configuration conf = getConfiguration("testTransactionManagerPersistence");
+ conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
+ // start snapshot thread, but with long enough interval so we only get snapshots on shutdown
+ conf.setInt(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 600);
+
+ TransactionStateStorage storage = null;
+ TransactionStateStorage storage2 = null;
+ TransactionStateStorage storage3 = null;
+ try {
+ storage = getStorage(conf);
+ TransactionManager txManager = new TransactionManager
+ (conf, storage, new TxMetricsCollector());
+ txManager.startAndWait();
+
+ // TODO: replace with new persistence tests
+ final byte[] a = { 'a' };
+ final byte[] b = { 'b' };
+ // start a tx1, add a change A and commit
+ Transaction tx1 = txManager.startShort();
+ Assert.assertTrue(txManager.canCommit(tx1, Collections.singleton(a)));
+ Assert.assertTrue(txManager.commit(tx1));
+ // start a tx2 and add a change B
+ Transaction tx2 = txManager.startShort();
+ Assert.assertTrue(txManager.canCommit(tx2, Collections.singleton(b)));
+ // start a tx3
+ Transaction tx3 = txManager.startShort();
+ // restart
+ txManager.stopAndWait();
+ TransactionSnapshot origState = txManager.getCurrentState();
+ LOG.info("Orig state: " + origState);
+
+ Thread.sleep(100);
+ // starts a new tx manager
+ storage2 = getStorage(conf);
+ txManager = new TransactionManager(conf, storage2, new TxMetricsCollector());
+ txManager.startAndWait();
+
+ // check that the reloaded state matches the old
+ TransactionSnapshot newState = txManager.getCurrentState();
+ LOG.info("New state: " + newState);
+ assertEquals(origState, newState);
+
+ // commit tx2
+ Assert.assertTrue(txManager.commit(tx2));
+ // start another transaction, must be greater than tx3
+ Transaction tx4 = txManager.startShort();
+ Assert.assertTrue(tx4.getTransactionId() > tx3.getTransactionId());
+ // tx1 must be visble from tx2, but tx3 and tx4 must not
+ Assert.assertTrue(tx2.isVisible(tx1.getTransactionId()));
+ Assert.assertFalse(tx2.isVisible(tx3.getTransactionId()));
+ Assert.assertFalse(tx2.isVisible(tx4.getTransactionId()));
+ // add same change for tx3
+ Assert.assertFalse(txManager.canCommit(tx3, Collections.singleton(b)));
+ // check visibility with new xaction
+ Transaction tx5 = txManager.startShort();
+ Assert.assertTrue(tx5.isVisible(tx1.getTransactionId()));
+ Assert.assertTrue(tx5.isVisible(tx2.getTransactionId()));
+ Assert.assertFalse(tx5.isVisible(tx3.getTransactionId()));
+ Assert.assertFalse(tx5.isVisible(tx4.getTransactionId()));
+ // can commit tx3?
+ txManager.abort(tx3);
+ txManager.abort(tx4);
+ txManager.abort(tx5);
+ // start new tx and verify its exclude list is empty
+ Transaction tx6 = txManager.startShort();
+ Assert.assertFalse(tx6.hasExcludes());
+ txManager.abort(tx6);
+
+ // now start 5 x claim size transactions
+ Transaction tx = txManager.startShort();
+ for (int i = 1; i < 50; i++) {
+ tx = txManager.startShort();
+ }
+ origState = txManager.getCurrentState();
+
+ Thread.sleep(100);
+ // simulate crash by starting a new tx manager without a stopAndWait
+ storage3 = getStorage(conf);
+ txManager = new TransactionManager(conf, storage3, new TxMetricsCollector());
+ txManager.startAndWait();
+
+ // verify state again matches (this time should include WAL replay)
+ newState = txManager.getCurrentState();
+ assertEquals(origState, newState);
+
+ // get a new transaction and verify it is greater
+ Transaction txAfter = txManager.startShort();
+ Assert.assertTrue(txAfter.getTransactionId() > tx.getTransactionId());
+ } finally {
+ if (storage != null) {
+ storage.stopAndWait();
+ }
+ if (storage2 != null) {
+ storage2.stopAndWait();
+ }
+ if (storage3 != null) {
+ storage3.stopAndWait();
+ }
+ }
+ }
+
+ /**
+ * Tests whether the committed set is advanced properly on WAL replay.
+ */
+ @Test
+ public void testCommittedSetClearing() throws Exception {
+ Configuration conf = getConfiguration("testCommittedSetClearing");
+ conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
+ conf.setInt(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 0); // no periodic snapshots
+
+ TransactionStateStorage storage1 = null;
+ TransactionStateStorage storage2 = null;
+ try {
+ storage1 = getStorage(conf);
+ TransactionManager txManager = new TransactionManager
+ (conf, storage1, new TxMetricsCollector());
+ txManager.startAndWait();
+
+ // TODO: replace with new persistence tests
+ final byte[] a = { 'a' };
+ final byte[] b = { 'b' };
+ // start a tx1, add a change A and commit
+ Transaction tx1 = txManager.startShort();
+ Assert.assertTrue(txManager.canCommit(tx1, Collections.singleton(a)));
+ Assert.assertTrue(txManager.commit(tx1));
+ // start a tx2 and add a change B
+ Transaction tx2 = txManager.startShort();
+ Assert.assertTrue(txManager.canCommit(tx2, Collections.singleton(b)));
+ // start a tx3
+ Transaction tx3 = txManager.startShort();
+ TransactionSnapshot origState = txManager.getCurrentState();
+ LOG.info("Orig state: " + origState);
+
+ // simulate a failure by starting a new tx manager without stopping first
+ storage2 = getStorage(conf);
+ txManager = new TransactionManager(conf, storage2, new TxMetricsCollector());
+ txManager.startAndWait();
+
+ // check that the reloaded state matches the old
+ TransactionSnapshot newState = txManager.getCurrentState();
+ LOG.info("New state: " + newState);
+ assertEquals(origState, newState);
+
+ } finally {
+ if (storage1 != null) {
+ storage1.stopAndWait();
+ }
+ if (storage2 != null) {
+ storage2.stopAndWait();
+ }
+ }
+ }
+
+ /**
+ * Tests removal of old snapshots and old transaction logs.
+ */
+ @Test
+ public void testOldFileRemoval() throws Exception {
+ Configuration conf = getConfiguration("testOldFileRemoval");
+ TransactionStateStorage storage = null;
+ try {
+ storage = getStorage(conf);
+ storage.startAndWait();
+ long now = System.currentTimeMillis();
+ long writePointer = 1;
+ Collection<Long> invalid = Lists.newArrayList();
+ NavigableMap<Long, TransactionManager.InProgressTx> inprogress = Maps.newTreeMap();
+ Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
+ Map<Long, Set<ChangeId>> committed = Maps.newHashMap();
+ TransactionSnapshot snapshot = new TransactionSnapshot(now, 0, writePointer++, invalid,
+ inprogress, committing, committed);
+ TransactionEdit dummyEdit = TransactionEdit.createStarted(1, 0, Long.MAX_VALUE, TransactionType.SHORT);
+
+ // write snapshot 1
+ storage.writeSnapshot(snapshot);
+ TransactionLog log = storage.createLog(now);
+ log.append(dummyEdit);
+ log.close();
+
+ snapshot = new TransactionSnapshot(now + 1, 0, writePointer++, invalid, inprogress, committing, committed);
+ // write snapshot 2
+ storage.writeSnapshot(snapshot);
+ log = storage.createLog(now + 1);
+ log.append(dummyEdit);
+ log.close();
+
+ snapshot = new TransactionSnapshot(now + 2, 0, writePointer++, invalid, inprogress, committing, committed);
+ // write snapshot 3
+ storage.writeSnapshot(snapshot);
+ log = storage.createLog(now + 2);
+ log.append(dummyEdit);
+ log.close();
+
+ snapshot = new TransactionSnapshot(now + 3, 0, writePointer++, invalid, inprogress, committing, committed);
+ // write snapshot 4
+ storage.writeSnapshot(snapshot);
+ log = storage.createLog(now + 3);
+ log.append(dummyEdit);
+ log.close();
+
+ snapshot = new TransactionSnapshot(now + 4, 0, writePointer++, invalid, inprogress, committing, committed);
+ // write snapshot 5
+ storage.writeSnapshot(snapshot);
+ log = storage.createLog(now + 4);
+ log.append(dummyEdit);
+ log.close();
+
+ snapshot = new TransactionSnapshot(now + 5, 0, writePointer++, invalid, inprogress, committing, committed);
+ // write snapshot 6
+ storage.writeSnapshot(snapshot);
+ log = storage.createLog(now + 5);
+ log.append(dummyEdit);
+ log.close();
+
+ List<String> allSnapshots = storage.listSnapshots();
+ LOG.info("All snapshots: " + allSnapshots);
+ assertEquals(6, allSnapshots.size());
+ List<String> allLogs = storage.listLogs();
+ LOG.info("All logs: " + allLogs);
+ assertEquals(6, allLogs.size());
+
+ long oldestKept = storage.deleteOldSnapshots(3);
+ assertEquals(now + 3, oldestKept);
+ allSnapshots = storage.listSnapshots();
+ LOG.info("All snapshots: " + allSnapshots);
+ assertEquals(3, allSnapshots.size());
+
+ storage.deleteLogsOlderThan(oldestKept);
+ allLogs = storage.listLogs();
+ LOG.info("All logs: " + allLogs);
+ assertEquals(3, allLogs.size());
+ } finally {
+ if (storage != null) {
+ storage.stopAndWait();
+ }
+ }
+ }
+
+ @Test
+ public void testLongTxnEditReplay() throws Exception {
+ Configuration conf = getConfiguration("testLongTxnEditReplay");
+ TransactionStateStorage storage = null;
+ try {
+ storage = getStorage(conf);
+ storage.startAndWait();
+
+ // Create long running txns. Abort one of them, invalidate another, invalidate and abort the last.
+ long time1 = System.currentTimeMillis();
+ long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit1 = TransactionEdit.createStarted(wp1, wp1 - 10, time1 + 100000, TransactionType.LONG);
+ TransactionEdit edit2 = TransactionEdit.createAborted(wp1, TransactionType.LONG, null);
+
+ long time2 = time1 + 100;
+ long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit3 = TransactionEdit.createStarted(wp2, wp2 - 10, time2 + 100000, TransactionType.LONG);
+ TransactionEdit edit4 = TransactionEdit.createInvalid(wp2);
+
+ long time3 = time1 + 200;
+ long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit5 = TransactionEdit.createStarted(wp3, wp3 - 10, time3 + 100000, TransactionType.LONG);
+ TransactionEdit edit6 = TransactionEdit.createInvalid(wp3);
+ TransactionEdit edit7 = TransactionEdit.createAborted(wp3, TransactionType.LONG, null);
+
+ // write transaction edits
+ TransactionLog log = storage.createLog(time1);
+ log.append(edit1);
+ log.append(edit2);
+ log.append(edit3);
+ log.append(edit4);
+ log.append(edit5);
+ log.append(edit6);
+ log.append(edit7);
+ log.close();
+
+ // Start transaction manager
+ TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
+ txm.startAndWait();
+ try {
+ // Verify that all txns are in invalid list.
+ TransactionSnapshot snapshot1 = txm.getCurrentState();
+ assertEquals(ImmutableList.of(wp1, wp2, wp3), snapshot1.getInvalid());
+ assertEquals(0, snapshot1.getInProgress().size());
+ assertEquals(0, snapshot1.getCommittedChangeSets().size());
+ assertEquals(0, snapshot1.getCommittedChangeSets().size());
+ } finally {
+ txm.stopAndWait();
+ }
+ } finally {
+ if (storage != null) {
+ storage.stopAndWait();
+ }
+ }
+ }
+
+ @Test
+ public void testTruncateInvalidTxEditReplay() throws Exception {
+ Configuration conf = getConfiguration("testTruncateInvalidTxEditReplay");
+ TransactionStateStorage storage = null;
+ try {
+ storage = getStorage(conf);
+ storage.startAndWait();
+
+ // Create some txns, and invalidate all of them.
+ long time1 = System.currentTimeMillis();
+ long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit1 = TransactionEdit.createStarted(wp1, wp1 - 10, time1 + 100000, TransactionType.LONG);
+ TransactionEdit edit2 = TransactionEdit.createInvalid(wp1);
+
+ long time2 = time1 + 100;
+ long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit3 = TransactionEdit.createStarted(wp2, wp2 - 10, time2 + 10000, TransactionType.SHORT);
+ TransactionEdit edit4 = TransactionEdit.createInvalid(wp2);
+
+ long time3 = time1 + 2000;
+ long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit5 = TransactionEdit.createStarted(wp3, wp3 - 10, time3 + 100000, TransactionType.LONG);
+ TransactionEdit edit6 = TransactionEdit.createInvalid(wp3);
+
+ long time4 = time1 + 2100;
+ long wp4 = time4 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit7 = TransactionEdit.createStarted(wp4, wp4 - 10, time4 + 10000, TransactionType.SHORT);
+ TransactionEdit edit8 = TransactionEdit.createInvalid(wp4);
+
+ // remove wp1 and wp3 from invalid list
+ TransactionEdit edit9 = TransactionEdit.createTruncateInvalidTx(ImmutableSet.of(wp1, wp3));
+ // truncate invalid transactions before time3
+ TransactionEdit edit10 = TransactionEdit.createTruncateInvalidTxBefore(time3);
+
+ // write transaction edits
+ TransactionLog log = storage.createLog(time1);
+ log.append(edit1);
+ log.append(edit2);
+ log.append(edit3);
+ log.append(edit4);
+ log.append(edit5);
+ log.append(edit6);
+ log.append(edit7);
+ log.append(edit8);
+ log.append(edit9);
+ log.append(edit10);
+ log.close();
+
+ // Start transaction manager
+ TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
+ txm.startAndWait();
+ try {
+ // Only wp4 should be in invalid list.
+ TransactionSnapshot snapshot = txm.getCurrentState();
+ assertEquals(ImmutableList.of(wp4), snapshot.getInvalid());
+ assertEquals(0, snapshot.getInProgress().size());
+ assertEquals(0, snapshot.getCommittedChangeSets().size());
+ assertEquals(0, snapshot.getCommittedChangeSets().size());
+ } finally {
+ txm.stopAndWait();
+ }
+ } finally {
+ if (storage != null) {
+ storage.stopAndWait();
+ }
+ }
+ }
+
+ /**
+ * Generates a new snapshot object with semi-randomly populated values. This does not necessarily accurately
+ * represent a typical snapshot's distribution of values, as we only set an upper bound on pointer values.
+ *
+ * We generate a new snapshot with the contents:
+ * <ul>
+ * <li>readPointer = 1M + (random % 1M)</li>
+ * <li>writePointer = readPointer + 1000</li>
+ * <li>waterMark = writePointer + 1000</li>
+ * <li>inProgress = one each for (writePointer - 500)..writePointer, ~ 5% "long" transaction</li>
+ * <li>invalid = 100 randomly distributed, 0..1M</li>
+ * <li>committing = one each, (readPointer + 1)..(readPointer + 100)</li>
+ * <li>committed = one each, (readPointer - 1000)..readPointer</li>
+ * </ul>
+ * @return a new snapshot of transaction state.
+ */
+ private TransactionSnapshot createRandomSnapshot() {
+ // limit readPointer to a reasonable range, but make it > 1M so we can assign enough keys below
+ long readPointer = (Math.abs(random.nextLong()) % 1000000L) + 1000000L;
+ long writePointer = readPointer + 1000L;
+
+ // generate in progress -- assume last 500 write pointer values
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
+ long startPointer = writePointer - 500L;
+ for (int i = 0; i < 500; i++) {
+ long currentTime = System.currentTimeMillis();
+ // make some "long" transactions
+ if (i % 20 == 0) {
+ inProgress.put(startPointer + i,
+ new TransactionManager.InProgressTx(startPointer - 1, currentTime + TimeUnit.DAYS.toSeconds(1),
+ TransactionType.LONG));
+ } else {
+ inProgress.put(startPointer + i,
+ new TransactionManager.InProgressTx(startPointer - 1, currentTime + 300000L,
+ TransactionType.SHORT));
+ }
+ }
+
+ // make 100 random invalid IDs
+ LongArrayList invalid = new LongArrayList();
+ for (int i = 0; i < 100; i++) {
+ invalid.add(Math.abs(random.nextLong()) % 1000000L);
+ }
+
+ // make 100 committing entries, 10 keys each
+ Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
+ for (int i = 0; i < 100; i++) {
+ committing.put(readPointer + i, generateChangeSet(10));
+ }
+
+ // make 1000 committed entries, 10 keys each
+ long startCommitted = readPointer - 1000L;
+ NavigableMap<Long, Set<ChangeId>> committed = Maps.newTreeMap();
+ for (int i = 0; i < 1000; i++) {
+ committed.put(startCommitted + i, generateChangeSet(10));
+ }
+
+ return new TransactionSnapshot(System.currentTimeMillis(), readPointer, writePointer,
+ invalid, inProgress, committing, committed);
+ }
+
+ private Set<ChangeId> generateChangeSet(int numEntries) {
+ Set<ChangeId> changes = Sets.newHashSet();
+ for (int i = 0; i < numEntries; i++) {
+ byte[] bytes = new byte[8];
+ random.nextBytes(bytes);
+ changes.add(new ChangeId(bytes));
+ }
+ return changes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/persist/CommitMarkerCodecTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/CommitMarkerCodecTest.java b/tephra-core/src/test/java/org/apache/tephra/persist/CommitMarkerCodecTest.java
new file mode 100644
index 0000000..22cdb59
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/CommitMarkerCodecTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.primitives.Ints;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.tephra.TxConstants;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Unit Test for {@link CommitMarkerCodec}.
+ */
+public class CommitMarkerCodecTest {
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ private static final String LOG_FILE = "txlog";
+ private static final Random RANDOM = new Random();
+
+ private static MiniDFSCluster dfsCluster;
+ private static Configuration conf;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration hConf = new Configuration();
+ hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
+
+ dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
+ conf = new Configuration(dfsCluster.getFileSystem().getConf());
+ fs = FileSystem.newInstance(FileSystem.getDefaultUri(conf), conf);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ @Test
+ public void testRandomCommitMarkers() throws Exception {
+ List<Integer> randomInts = new ArrayList<>();
+ Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE);
+
+ // Write a bunch of random commit markers
+ try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class,
+ LongWritable.class,
+ SequenceFile.CompressionType.NONE)) {
+ for (int i = 0; i < 1000; i++) {
+ int randomNum = RANDOM.nextInt(Integer.MAX_VALUE);
+ CommitMarkerCodec.writeMarker(writer, randomNum);
+ randomInts.add(randomNum);
+ }
+ writer.hflush();
+ writer.hsync();
+ }
+
+ // Read the commit markers back to verify the marker
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf);
+ CommitMarkerCodec markerCodec = new CommitMarkerCodec()) {
+ for (int num : randomInts) {
+ Assert.assertEquals(num, markerCodec.readMarker(reader));
+ }
+ }
+ }
+
+ private static class IncompleteValueBytes implements SequenceFile.ValueBytes {
+
+ @Override
+ public void writeUncompressedBytes(DataOutputStream outStream) throws IOException {
+ // don't write anything to simulate incomplete record
+ }
+
+ @Override
+ public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException {
+ throw new IllegalArgumentException("Not possible");
+ }
+
+ @Override
+ public int getSize() {
+ return Ints.BYTES;
+ }
+ }
+
+ @Test
+ public void testIncompleteCommitMarker() throws Exception {
+ Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE);
+ try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class,
+ LongWritable.class,
+ SequenceFile.CompressionType.NONE)) {
+ String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED;
+ SequenceFile.ValueBytes valueBytes = new IncompleteValueBytes();
+ writer.appendRaw(key.getBytes(), 0, key.length(), valueBytes);
+ writer.hflush();
+ writer.hsync();
+ }
+
+ // Read the incomplete commit marker
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf);
+ CommitMarkerCodec markerCodec = new CommitMarkerCodec()) {
+ try {
+ markerCodec.readMarker(reader);
+ Assert.fail("Expected EOF Exception to be thrown");
+ } catch (EOFException e) {
+ // expected since we didn't write the value bytes
+ }
+ }
+ }
+
+ @Test
+ public void testIncorrectCommitMarker() throws Exception {
+ Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE);
+
+ // Write an incorrect marker
+ try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class,
+ LongWritable.class,
+ SequenceFile.CompressionType.NONE)) {
+ String invalidKey = "IncorrectKey";
+ SequenceFile.ValueBytes valueBytes = new CommitMarkerCodec.CommitEntriesCount(100);
+ writer.appendRaw(invalidKey.getBytes(), 0, invalidKey.length(), valueBytes);
+ writer.hflush();
+ writer.hsync();
+ }
+
+ // Read the commit markers back to verify the marker
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf);
+ CommitMarkerCodec markerCodec = new CommitMarkerCodec()) {
+ try {
+ markerCodec.readMarker(reader);
+ Assert.fail("Expected an IOException to be thrown");
+ } catch (IOException e) {
+ // expected
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionLogTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionLogTest.java b/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionLogTest.java
new file mode 100644
index 0000000..7b9f06b
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionLogTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.io.Closeables;
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.util.TransactionEditUtil;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+
+/**
+ * Testing for complete and partial sycs of {@link TransactionEdit} to {@link HDFSTransactionLog}
+ */
+public class HDFSTransactionLogTest {
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ private static final String LOG_FILE_PREFIX = "txlog.";
+
+ private static MiniDFSCluster dfsCluster;
+ private static Configuration conf;
+ private static MetricsCollector metricsCollector;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration hConf = new Configuration();
+ hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
+
+ dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
+ conf = new Configuration(dfsCluster.getFileSystem().getConf());
+ metricsCollector = new TxMetricsCollector();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ private Configuration getConfiguration() throws IOException {
+ // tests should use the current user for HDFS
+ conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
+ return conf;
+ }
+
+ private HDFSTransactionLog getHDFSTransactionLog(Configuration conf,
+ FileSystem fs, long timeInMillis) throws Exception {
+ String snapshotDir = conf.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR);
+ Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timeInMillis);
+ return new HDFSTransactionLog(fs, conf, newLog, timeInMillis, metricsCollector);
+ }
+
+ private SequenceFile.Writer getSequenceFileWriter(Configuration configuration, FileSystem fs,
+ long timeInMillis, boolean withMarker) throws IOException {
+ String snapshotDir = configuration.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR);
+ Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timeInMillis);
+ SequenceFile.Metadata metadata = new SequenceFile.Metadata();
+ if (withMarker) {
+ metadata.set(new Text(TxConstants.TransactionLog.VERSION_KEY),
+ new Text(Byte.toString(TxConstants.TransactionLog.CURRENT_VERSION)));
+ }
+ return SequenceFile.createWriter(fs, configuration, newLog, LongWritable.class,
+ TransactionEdit.class, SequenceFile.CompressionType.NONE, null, null, metadata);
+ }
+
+ private void writeNumWrites(SequenceFile.Writer writer, final int size) throws Exception {
+ String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED;
+ CommitMarkerCodec.writeMarker(writer, size);
+ }
+
+ private void testTransactionLogSync(int totalCount, int batchSize, boolean withMarker, boolean isComplete)
+ throws Exception {
+ List<TransactionEdit> edits = TransactionEditUtil.createRandomEdits(totalCount);
+ long timestamp = System.currentTimeMillis();
+ Configuration configuration = getConfiguration();
+ FileSystem fs = FileSystem.newInstance(FileSystem.getDefaultUri(configuration), configuration);
+ SequenceFile.Writer writer = getSequenceFileWriter(configuration, fs, timestamp, withMarker);
+ AtomicLong logSequence = new AtomicLong();
+ HDFSTransactionLog transactionLog = getHDFSTransactionLog(configuration, fs, timestamp);
+ AbstractTransactionLog.Entry entry;
+
+ for (int i = 0; i < totalCount - batchSize; i += batchSize) {
+ if (withMarker) {
+ writeNumWrites(writer, batchSize);
+ }
+ for (int j = 0; j < batchSize; j++) {
+ entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(j));
+ writer.append(entry.getKey(), entry.getEdit());
+ }
+ writer.syncFs();
+ }
+
+ if (withMarker) {
+ writeNumWrites(writer, batchSize);
+ }
+
+ for (int i = totalCount - batchSize; i < totalCount - 1; i++) {
+ entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(i));
+ writer.append(entry.getKey(), entry.getEdit());
+ }
+
+ entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()),
+ edits.get(totalCount - 1));
+ if (isComplete) {
+ writer.append(entry.getKey(), entry.getEdit());
+ } else {
+ byte[] bytes = Longs.toByteArray(entry.getKey().get());
+ writer.appendRaw(bytes, 0, bytes.length, new SequenceFile.ValueBytes() {
+ @Override
+ public void writeUncompressedBytes(DataOutputStream outStream) throws IOException {
+ byte[] test = new byte[]{0x2};
+ outStream.write(test, 0, 1);
+ }
+
+ @Override
+ public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException {
+ // no-op
+ }
+
+ @Override
+ public int getSize() {
+ // mimic size longer than the actual byte array size written, so we would reach EOF
+ return 12;
+ }
+ });
+ }
+ writer.syncFs();
+ Closeables.closeQuietly(writer);
+
+ // now let's try to read this log
+ TransactionLogReader reader = transactionLog.getReader();
+ int syncedEdits = 0;
+ while (reader.next() != null) {
+ // testing reading the transaction edits
+ syncedEdits++;
+ }
+ if (isComplete) {
+ Assert.assertEquals(totalCount, syncedEdits);
+ } else {
+ Assert.assertEquals(totalCount - batchSize, syncedEdits);
+ }
+ }
+
+ @Test
+ public void testTransactionLogNewVersion() throws Exception {
+ // in-complete sync
+ testTransactionLogSync(1000, 1, true, false);
+ testTransactionLogSync(2000, 5, true, false);
+
+ // complete sync
+ testTransactionLogSync(1000, 1, true, true);
+ testTransactionLogSync(2000, 5, true, true);
+ }
+
+ @Test
+ public void testTransactionLogOldVersion() throws Exception {
+ // in-complete sync
+ testTransactionLogSync(1000, 1, false, false);
+
+ // complete sync
+ testTransactionLogSync(2000, 5, false, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionStateStorageTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionStateStorageTest.java b/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionStateStorageTest.java
new file mode 100644
index 0000000..f0eb9e5
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/HDFSTransactionStateStorageTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+
+
+/**
+ * Tests persistence of transaction snapshots and write-ahead logs to HDFS storage, using the
+ * {@link HDFSTransactionStateStorage} and {@link HDFSTransactionLog} implementations.
+ */
+public class HDFSTransactionStateStorageTest extends AbstractTransactionStateStorageTest {
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ private static MiniDFSCluster dfsCluster;
+ private static Configuration conf;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration hConf = new Configuration();
+ hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
+
+ dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
+ conf = new Configuration(dfsCluster.getFileSystem().getConf());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ @Override
+ protected Configuration getConfiguration(String testName) throws IOException {
+ // tests should use the current user for HDFS
+ conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
+ return conf;
+ }
+
+ @Override
+ protected AbstractTransactionStateStorage getStorage(Configuration conf) {
+ return new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/persist/InMemoryTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/InMemoryTransactionStateStorage.java b/tephra-core/src/test/java/org/apache/tephra/persist/InMemoryTransactionStateStorage.java
new file mode 100644
index 0000000..366542b
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/InMemoryTransactionStateStorage.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.AbstractIdleService;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+
+/**
+ * Stores the latest transaction snapshot and logs in memory.
+ */
+public class InMemoryTransactionStateStorage extends AbstractIdleService implements TransactionStateStorage {
+ // only keeps the most recent snapshot in memory
+ private TransactionSnapshot lastSnapshot;
+
+ private NavigableMap<Long, TransactionLog> logs = new TreeMap<>();
+
+ @Override
+ protected void startUp() throws Exception {
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ lastSnapshot = null;
+ logs = new TreeMap<>();
+ }
+
+ @Override
+ public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException {
+ // no codecs in in-memory mode
+ }
+
+ @Override
+ public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
+ lastSnapshot = snapshot;
+ }
+
+ @Override
+ public TransactionSnapshot getLatestSnapshot() throws IOException {
+ return lastSnapshot;
+ }
+
+ @Override
+ public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
+ return lastSnapshot;
+ }
+
+ @Override
+ public long deleteOldSnapshots(int numberToKeep) throws IOException {
+ // always only keep the last snapshot
+ return lastSnapshot.getTimestamp();
+ }
+
+ @Override
+ public List<String> listSnapshots() throws IOException {
+ List<String> snapshots = Lists.newArrayListWithCapacity(1);
+ if (lastSnapshot != null) {
+ snapshots.add(Long.toString(lastSnapshot.getTimestamp()));
+ }
+ return snapshots;
+ }
+
+ @Override
+ public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
+ return Lists.newArrayList(logs.tailMap(timestamp).values());
+ }
+
+ @Override
+ public TransactionLog createLog(long timestamp) throws IOException {
+ TransactionLog log = new InMemoryTransactionLog(timestamp);
+ logs.put(timestamp, log);
+ return log;
+ }
+
+ @Override
+ public void deleteLogsOlderThan(long timestamp) throws IOException {
+ Iterator<Map.Entry<Long, TransactionLog>> logIter = logs.entrySet().iterator();
+ while (logIter.hasNext()) {
+ Map.Entry<Long, TransactionLog> logEntry = logIter.next();
+ if (logEntry.getKey() < timestamp) {
+ logIter.remove();
+ }
+ }
+ }
+
+ @Override
+ public void setupStorage() throws IOException {
+ }
+
+ @Override
+ public List<String> listLogs() throws IOException {
+ return Lists.transform(Lists.newArrayList(logs.keySet()), new Function<Long, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable Long input) {
+ return input.toString();
+ }
+ });
+ }
+
+ @Override
+ public String getLocation() {
+ return "in-memory";
+ }
+
+ public static class InMemoryTransactionLog implements TransactionLog {
+ private long timestamp;
+ private List<TransactionEdit> edits = Lists.newArrayList();
+ boolean isClosed = false;
+ public InMemoryTransactionLog(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public String getName() {
+ return "in-memory@" + timestamp;
+ }
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public void append(TransactionEdit edit) throws IOException {
+ if (isClosed) {
+ throw new IOException("Log is closed");
+ }
+ edits.add(edit);
+ }
+
+ @Override
+ public void append(List<TransactionEdit> edits) throws IOException {
+ if (isClosed) {
+ throw new IOException("Log is closed");
+ }
+ edits.addAll(edits);
+ }
+
+ @Override
+ public void close() {
+ isClosed = true;
+ }
+
+ @Override
+ public TransactionLogReader getReader() throws IOException {
+ return new InMemoryLogReader(edits.iterator());
+ }
+ }
+
+ public static class InMemoryLogReader implements TransactionLogReader {
+ private final Iterator<TransactionEdit> editIterator;
+
+ public InMemoryLogReader(Iterator<TransactionEdit> editIterator) {
+ this.editIterator = editIterator;
+ }
+
+ @Override
+ public TransactionEdit next() throws IOException {
+ if (editIterator.hasNext()) {
+ return editIterator.next();
+ }
+ return null;
+ }
+
+ @Override
+ public TransactionEdit next(TransactionEdit reuse) throws IOException {
+ return next();
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/persist/LocalTransactionStateStorageTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/LocalTransactionStateStorageTest.java b/tephra-core/src/test/java/org/apache/tephra/persist/LocalTransactionStateStorageTest.java
new file mode 100644
index 0000000..9535102
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/LocalTransactionStateStorageTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.snapshot.DefaultSnapshotCodec;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.snapshot.SnapshotCodecV4;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Runs transaction persistence tests against the {@link LocalFileTransactionStateStorage} and
+ * {@link LocalFileTransactionLog} implementations.
+ */
+public class LocalTransactionStateStorageTest extends AbstractTransactionStateStorageTest {
+ @ClassRule
+ public static TemporaryFolder tmpDir = new TemporaryFolder();
+
+ @Override
+ protected Configuration getConfiguration(String testName) throws IOException {
+ File testDir = tmpDir.newFolder(testName);
+ Configuration conf = new Configuration();
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
+ return conf;
+ }
+
+ @Override
+ protected AbstractTransactionStateStorage getStorage(Configuration conf) {
+ return new LocalFileTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
+ }
+
+ // v2 TransactionEdit
+ @SuppressWarnings("deprecation")
+ private class TransactionEditV2 extends TransactionEdit {
+ public TransactionEditV2(long writePointer, long visibilityUpperBound, State state, long expirationDate,
+ Set<ChangeId> changes, long commitPointer, boolean canCommit, TransactionType type) {
+ super(writePointer, visibilityUpperBound, state, expirationDate, changes, commitPointer, canCommit, type,
+ null, 0L, 0L, null);
+ }
+ @Override
+ public void write(DataOutput out) throws IOException {
+ TransactionEditCodecs.encode(this, out, new TransactionEditCodecs.TransactionEditCodecV2());
+ }
+ }
+
+ // Note: this test cannot run in AbstractTransactionStateStorageTest, since SequenceFile throws exception saying
+ // TransactionEditV2 is not TransactionEdit. Since the code path this test is verifying is the same path between
+ // HDFS and Local Storage, having this only over here is fine.
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testLongTxnBackwardsCompatibility() throws Exception {
+ Configuration conf = getConfiguration("testLongTxnBackwardsCompatibility");
+
+ // Use SnapshotCodec version 1
+ String latestSnapshotCodec = conf.get(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+
+ TransactionStateStorage storage = null;
+ try {
+ storage = getStorage(conf);
+ storage.startAndWait();
+
+ // Create transaction snapshot and transaction edits with version when long running txns had -1 expiration.
+ Collection<Long> invalid = Lists.newArrayList();
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
+ long time1 = System.currentTimeMillis();
+ long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
+ inProgress.put(wp1, new TransactionManager.InProgressTx(wp1 - 5, -1L));
+ long time2 = time1 + 100;
+ long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
+ inProgress.put(wp2, new TransactionManager.InProgressTx(wp2 - 50, time2 + 1000));
+ Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
+ Map<Long, Set<ChangeId>> committed = Maps.newHashMap();
+ TransactionSnapshot snapshot = new TransactionSnapshot(time2, 0, wp2, invalid,
+ inProgress, committing, committed);
+ long time3 = time1 + 200;
+ long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit1 = new TransactionEditV2(wp3, wp3 - 10, TransactionEdit.State.INPROGRESS, -1L,
+ null, 0L, false, null);
+ long time4 = time1 + 300;
+ long wp4 = time4 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit2 = new TransactionEditV2(wp4, wp4 - 10, TransactionEdit.State.INPROGRESS, time4 + 1000,
+ null, 0L, false, null);
+
+ // write snapshot and transaction edit
+ storage.writeSnapshot(snapshot);
+ TransactionLog log = storage.createLog(time2);
+ log.append(edit1);
+ log.append(edit2);
+ log.close();
+
+ // Start transaction manager
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, latestSnapshotCodec);
+ long longTimeout = TimeUnit.SECONDS.toMillis(conf.getLong(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
+ TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT));
+ TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
+ txm.startAndWait();
+ try {
+ // Verify that the txns in old format were read correctly.
+ // There should be four in-progress transactions, and no invalid transactions
+ TransactionSnapshot snapshot1 = txm.getCurrentState();
+ Assert.assertEquals(ImmutableSortedSet.of(wp1, wp2, wp3, wp4), snapshot1.getInProgress().keySet());
+ verifyInProgress(snapshot1.getInProgress().get(wp1), TransactionType.LONG, time1 + longTimeout);
+ verifyInProgress(snapshot1.getInProgress().get(wp2), TransactionType.SHORT, time2 + 1000);
+ verifyInProgress(snapshot1.getInProgress().get(wp3), TransactionType.LONG, time3 + longTimeout);
+ verifyInProgress(snapshot1.getInProgress().get(wp4), TransactionType.SHORT, time4 + 1000);
+ Assert.assertEquals(0, snapshot1.getInvalid().size());
+ } finally {
+ txm.stopAndWait();
+ }
+ } finally {
+ if (storage != null) {
+ storage.stopAndWait();
+ }
+ }
+ }
+
+ // Note: this test cannot run in AbstractTransactionStateStorageTest, since SequenceFile throws exception saying
+ // TransactionEditV2 is not TransactionEdit. Since the code path this test is verifying is the same path between
+ // HDFS and Local Storage, having this only over here is fine.
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testAbortEditBackwardsCompatibility() throws Exception {
+ Configuration conf = getConfiguration("testAbortEditBackwardsCompatibility");
+
+ TransactionStateStorage storage = null;
+ try {
+ storage = getStorage(conf);
+ storage.startAndWait();
+
+ // Create edits for transaction type addition to abort
+ long time1 = System.currentTimeMillis();
+ long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit1 = new TransactionEditV2(wp1, wp1 - 10, TransactionEdit.State.INPROGRESS, -1L,
+ null, 0L, false, null);
+ TransactionEdit edit2 = new TransactionEditV2(wp1, 0L, TransactionEdit.State.ABORTED, 0L,
+ null, 0L, false, null);
+
+ long time2 = time1 + 400;
+ long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
+ TransactionEdit edit3 = new TransactionEditV2(wp2, wp2 - 10, TransactionEdit.State.INPROGRESS, time2 + 10000,
+ null, 0L, false, null);
+ TransactionEdit edit4 = new TransactionEditV2(wp2, 0L, TransactionEdit.State.INVALID, 0L, null, 0L, false, null);
+ // Simulate case where we cannot determine txn state during abort
+ TransactionEdit edit5 = new TransactionEditV2(wp2, 0L, TransactionEdit.State.ABORTED, 0L, null, 0L, false, null);
+
+ // write snapshot and transaction edit
+ TransactionLog log = storage.createLog(time1);
+ log.append(edit1);
+ log.append(edit2);
+ log.append(edit3);
+ log.append(edit4);
+ log.append(edit5);
+ log.close();
+
+ // Start transaction manager
+ TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
+ txm.startAndWait();
+ try {
+ // Verify that the txns in old format were read correctly.
+ // Both transactions should be in invalid state
+ TransactionSnapshot snapshot1 = txm.getCurrentState();
+ Assert.assertEquals(ImmutableList.of(wp1, wp2), snapshot1.getInvalid());
+ Assert.assertEquals(0, snapshot1.getInProgress().size());
+ Assert.assertEquals(0, snapshot1.getCommittedChangeSets().size());
+ Assert.assertEquals(0, snapshot1.getCommittingChangeSets().size());
+ } finally {
+ txm.stopAndWait();
+ }
+ } finally {
+ if (storage != null) {
+ storage.stopAndWait();
+ }
+ }
+ }
+
+ private void verifyInProgress(TransactionManager.InProgressTx inProgressTx, TransactionType type,
+ long expiration) throws Exception {
+ Assert.assertEquals(type, inProgressTx.getType());
+ Assert.assertTrue(inProgressTx.getExpiration() == expiration);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/persist/TransactionEditTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/persist/TransactionEditTest.java b/tephra-core/src/test/java/org/apache/tephra/persist/TransactionEditTest.java
new file mode 100644
index 0000000..46cb81c
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/persist/TransactionEditTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.TransactionType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.DataInput;
+import java.io.IOException;
+
+/**
+ * test for {@link TransactionEdit}
+ */
+public class TransactionEditTest {
+ private static final byte[] COL = new byte[] {'c'};
+
+ @Test
+ public void testV1SerdeCompat() throws Exception {
+ TransactionEditCodecs.TransactionEditCodec olderCodec = new TransactionEditCodecs.TransactionEditCodecV1();
+ // start tx edit and committed tx edit cover all fields of tx edit
+ // NOTE: set visibilityUpperBound to 0 and transaction type to null as this is expected default
+ // for decoding older versions that doesn't store it
+ verifyDecodingSupportsOlderVersion(TransactionEdit.createStarted(2L, 0L, 1000L, null), olderCodec);
+ verifyDecodingSupportsOlderVersion(
+ TransactionEdit.createCommitted(2L, Sets.newHashSet(new ChangeId(COL)), 3L, true), olderCodec);
+ }
+
+ @Test
+ public void testV2SerdeCompat() throws Exception {
+ TransactionEditCodecs.TransactionEditCodec olderCodec = new TransactionEditCodecs.TransactionEditCodecV2();
+ // start tx edit and committed tx edit cover all fields of tx edit
+ // NOTE: transaction type to null as this is expected default for decoding older versions that doesn't store it
+ verifyDecodingSupportsOlderVersion(TransactionEdit.createStarted(2L, 100L, 1000L, null), olderCodec);
+ verifyDecodingSupportsOlderVersion(
+ TransactionEdit.createCommitted(2L, Sets.newHashSet(new ChangeId(COL)), 3L, true), olderCodec);
+ }
+
+ @SuppressWarnings("deprecation")
+ private void verifyDecodingSupportsOlderVersion(TransactionEdit edit,
+ TransactionEditCodecs.TransactionEditCodec olderCodec)
+ throws IOException {
+ // encoding with older version of codec
+ ByteArrayDataOutput out = ByteStreams.newDataOutput();
+ TransactionEditCodecs.encode(edit, out, olderCodec);
+
+ // decoding
+ TransactionEdit decodedEdit = new TransactionEdit();
+ DataInput in = ByteStreams.newDataInput(out.toByteArray());
+ decodedEdit.readFields(in);
+
+ Assert.assertEquals(edit, decodedEdit);
+ }
+
+ @Test
+ public void testSerialization() throws Exception {
+ assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.SHORT, new long[0]));
+ assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.SHORT, new long[]{ 2L, 3L }));
+ assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.LONG, new long[0]));
+ assertSerializedEdit(TransactionEdit.createAborted(1L, TransactionType.LONG, new long[]{ 2L, 3L }));
+
+ assertSerializedEdit(TransactionEdit.createCheckpoint(2L, 1L));
+
+ assertSerializedEdit(TransactionEdit.createCommitted(1L, Sets.<ChangeId>newHashSet(), 2L, false));
+ assertSerializedEdit(TransactionEdit.createCommitted(1L, Sets.<ChangeId>newHashSet(), 2L, true));
+ assertSerializedEdit(TransactionEdit.createCommitted(1L,
+ Sets.newHashSet(new ChangeId(new byte[]{'a', 'b', 'c'})), 2L, false));
+ assertSerializedEdit(TransactionEdit.createCommitted(1L,
+ Sets.newHashSet(new ChangeId(new byte[]{ 'a', 'b', 'c' }), new ChangeId(new byte[]{ 'd', 'e', 'f' })),
+ 2L, true));
+
+ assertSerializedEdit(TransactionEdit.createCommitting(1L, Sets.<ChangeId>newHashSet()));
+ assertSerializedEdit(TransactionEdit.createCommitting(1L,
+ Sets.newHashSet(new ChangeId(new byte[]{'a', 'b', 'c'}))));
+ assertSerializedEdit(TransactionEdit.createCommitting(1L,
+ Sets.newHashSet(new ChangeId(new byte[]{'a', 'b', 'c'}), new ChangeId(new byte[]{'d', 'e', 'f'}))));
+
+ assertSerializedEdit(TransactionEdit.createInvalid(1L));
+
+ assertSerializedEdit(TransactionEdit.createMoveWatermark(10L));
+
+ assertSerializedEdit(TransactionEdit.createStarted(2L, 1L, System.currentTimeMillis() + 1000,
+ TransactionType.SHORT));
+ assertSerializedEdit(TransactionEdit.createStarted(2L, 1L, System.currentTimeMillis() + 10000,
+ TransactionType.LONG));
+
+ assertSerializedEdit(TransactionEdit.createTruncateInvalidTx(Sets.newHashSet(new Long(1))));
+ assertSerializedEdit(TransactionEdit.createTruncateInvalidTx(
+ Sets.newHashSet(new Long(1), new Long(2), new Long(3))));
+
+ assertSerializedEdit(TransactionEdit.createTruncateInvalidTxBefore(System.currentTimeMillis()));
+ }
+
+ private void assertSerializedEdit(TransactionEdit originalEdit) throws IOException {
+ ByteArrayDataOutput out = ByteStreams.newDataOutput();
+ originalEdit.write(out);
+
+ TransactionEdit decodedEdit = new TransactionEdit();
+ DataInput in = ByteStreams.newDataInput(out.toByteArray());
+ decodedEdit.readFields(in);
+
+ Assert.assertEquals(originalEdit, decodedEdit);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/snapshot/SnapshotCodecTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/snapshot/SnapshotCodecTest.java b/tephra-core/src/test/java/org/apache/tephra/snapshot/SnapshotCodecTest.java
new file mode 100644
index 0000000..afdff5c
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/snapshot/SnapshotCodecTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionNotInProgressException;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionModules;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests related to {@link SnapshotCodec} implementations.
+ */
+public class SnapshotCodecTest {
+ @ClassRule
+ public static TemporaryFolder tmpDir = new TemporaryFolder();
+
+ @Test
+ public void testMinimalDeserilization() throws Exception {
+ long now = System.currentTimeMillis();
+ long nowWritePointer = now * TxConstants.MAX_TX_PER_MS;
+ /*
+ * Snapshot consisting of transactions at:
+ */
+ long tInvalid = nowWritePointer - 5; // t1 - invalid
+ long readPtr = nowWritePointer - 4; // t2 - here and earlier committed
+ long tLong = nowWritePointer - 3; // t3 - in-progress LONG
+ long tCommitted = nowWritePointer - 2; // t4 - committed, changeset (r1, r2)
+ long tShort = nowWritePointer - 1; // t5 - in-progress SHORT, canCommit called, changeset (r3, r4)
+
+ TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap(ImmutableSortedMap.of(
+ tLong, new TransactionManager.InProgressTx(readPtr,
+ TransactionManager.getTxExpirationFromWritePointer(
+ tLong, TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT),
+ TransactionType.LONG),
+ tShort, new TransactionManager.InProgressTx(readPtr, now + 1000, TransactionType.SHORT)));
+
+ TransactionSnapshot snapshot = new TransactionSnapshot(now, readPtr, nowWritePointer,
+ Lists.newArrayList(tInvalid), // invalid
+ inProgress, ImmutableMap.<Long, Set<ChangeId>>of(
+ tShort, Sets.<ChangeId>newHashSet()),
+ ImmutableMap.<Long, Set<ChangeId>>of(
+ tCommitted, Sets.<ChangeId>newHashSet()));
+
+ Configuration conf1 = new Configuration();
+ conf1.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
+ SnapshotCodecProvider provider1 = new SnapshotCodecProvider(conf1);
+
+ byte[] byteArray;
+ try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+ provider1.encode(out, snapshot);
+ byteArray = out.toByteArray();
+ }
+
+ // TransactionSnapshot and TransactionVisibilityState decode should pass now
+ TransactionSnapshot txSnapshot = provider1.decode(new ByteArrayInputStream(byteArray));
+ TransactionVisibilityState txVisibilityState =
+ provider1.decodeTransactionVisibilityState(new ByteArrayInputStream(byteArray));
+ assertTransactionVisibilityStateEquals(txSnapshot, txVisibilityState);
+
+ // Corrupt the serialization byte array so that full deserialization will fail
+ byteArray[byteArray.length - 1] = 'a';
+
+ // TransactionVisibilityState decoding should pass since it doesn't decode the committing and committed changesets.
+ TransactionVisibilityState txVisibilityState2 = provider1.decodeTransactionVisibilityState(
+ new ByteArrayInputStream(byteArray));
+ Assert.assertNotNull(txVisibilityState2);
+ Assert.assertEquals(txVisibilityState, txVisibilityState2);
+ Assert.assertEquals(readPtr, txVisibilityState2.getReadPointer());
+ try {
+ provider1.decode(new ByteArrayInputStream(byteArray));
+ Assert.fail();
+ } catch (RuntimeException e) {
+ // expected since we modified the serialization bytes
+ }
+ }
+
+ /**
+ * In-progress LONG transactions written with DefaultSnapshotCodec will not have the type serialized as part of
+ * the data. Since these transactions also contain a non-negative expiration, we need to ensure we reset the type
+ * correctly when the snapshot is loaded.
+ */
+ @Test
+ public void testDefaultToV3Compatibility() throws Exception {
+ long now = System.currentTimeMillis();
+ long nowWritePointer = now * TxConstants.MAX_TX_PER_MS;
+ /*
+ * Snapshot consisting of transactions at:
+ */
+ long tInvalid = nowWritePointer - 5; // t1 - invalid
+ long readPtr = nowWritePointer - 4; // t2 - here and earlier committed
+ long tLong = nowWritePointer - 3; // t3 - in-progress LONG
+ long tCommitted = nowWritePointer - 2; // t4 - committed, changeset (r1, r2)
+ long tShort = nowWritePointer - 1; // t5 - in-progress SHORT, canCommit called, changeset (r3, r4)
+
+ TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap(ImmutableSortedMap.of(
+ tLong, new TransactionManager.InProgressTx(readPtr,
+ TransactionManager.getTxExpirationFromWritePointer(tLong, TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT),
+ TransactionType.LONG),
+ tShort, new TransactionManager.InProgressTx(readPtr, now + 1000, TransactionType.SHORT)));
+
+ TransactionSnapshot snapshot = new TransactionSnapshot(now, readPtr, nowWritePointer,
+ Lists.newArrayList(tInvalid), // invalid
+ inProgress,
+ ImmutableMap.<Long, Set<ChangeId>>of(
+ tShort, Sets.newHashSet(new ChangeId(new byte[]{'r', '3'}), new ChangeId(new byte[]{'r', '4'}))),
+ ImmutableMap.<Long, Set<ChangeId>>of(
+ tCommitted, Sets.newHashSet(new ChangeId(new byte[]{'r', '1'}), new ChangeId(new byte[]{'r', '2'}))));
+
+ Configuration conf1 = new Configuration();
+ conf1.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+ SnapshotCodecProvider provider1 = new SnapshotCodecProvider(conf1);
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try {
+ provider1.encode(out, snapshot);
+ } finally {
+ out.close();
+ }
+
+ TransactionSnapshot snapshot2 = provider1.decode(new ByteArrayInputStream(out.toByteArray()));
+ TransactionVisibilityState minTxSnapshot = provider1.decodeTransactionVisibilityState(
+ new ByteArrayInputStream(out.toByteArray()));
+ assertTransactionVisibilityStateEquals(snapshot2, minTxSnapshot);
+
+ assertEquals(snapshot.getReadPointer(), snapshot2.getReadPointer());
+ assertEquals(snapshot.getWritePointer(), snapshot2.getWritePointer());
+ assertEquals(snapshot.getInvalid(), snapshot2.getInvalid());
+ // in-progress transactions will have missing types
+ assertNotEquals(snapshot.getInProgress(), snapshot2.getInProgress());
+ assertEquals(snapshot.getCommittingChangeSets(), snapshot2.getCommittingChangeSets());
+ assertEquals(snapshot.getCommittedChangeSets(), snapshot2.getCommittedChangeSets());
+
+ // after fixing in-progress, full snapshot should match
+ Map<Long, TransactionManager.InProgressTx> fixedInProgress = TransactionManager.txnBackwardsCompatCheck(
+ TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT, 10000L, snapshot2.getInProgress());
+ assertEquals(snapshot.getInProgress(), fixedInProgress);
+ assertEquals(snapshot, snapshot2);
+ }
+
+ /**
+ * Test full stack serialization for a TransactionManager migrating from DefaultSnapshotCodec to SnapshotCodecV3.
+ */
+ @Test
+ public void testDefaultToV3Migration() throws Exception {
+ File testDir = tmpDir.newFolder("testDefaultToV3Migration");
+ Configuration conf = new Configuration();
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
+
+ Injector injector = Guice.createInjector(new ConfigModule(conf),
+ new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());
+
+ TransactionManager txManager = injector.getInstance(TransactionManager.class);
+ txManager.startAndWait();
+
+ txManager.startLong();
+
+ // shutdown to force a snapshot
+ txManager.stopAndWait();
+
+ TransactionStateStorage txStorage = injector.getInstance(TransactionStateStorage.class);
+ txStorage.startAndWait();
+
+ // confirm that the in-progress entry is missing a type
+ TransactionSnapshot snapshot = txStorage.getLatestSnapshot();
+ TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState();
+ assertTransactionVisibilityStateEquals(snapshot, txVisibilityState);
+ assertNotNull(snapshot);
+ assertEquals(1, snapshot.getInProgress().size());
+ Map.Entry<Long, TransactionManager.InProgressTx> entry =
+ snapshot.getInProgress().entrySet().iterator().next();
+ assertNull(entry.getValue().getType());
+ txStorage.stopAndWait();
+
+
+ // start a new Tx manager to test fixup
+ Configuration conf2 = new Configuration();
+ conf2.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
+ conf2.setStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES,
+ DefaultSnapshotCodec.class.getName(), SnapshotCodecV3.class.getName());
+ Injector injector2 = Guice.createInjector(new ConfigModule(conf2),
+ new DiscoveryModules().getSingleNodeModules(), new TransactionModules().getSingleNodeModules());
+
+ TransactionManager txManager2 = injector2.getInstance(TransactionManager.class);
+ txManager2.startAndWait();
+
+ // state should be recovered
+ TransactionSnapshot snapshot2 = txManager2.getCurrentState();
+ assertEquals(1, snapshot2.getInProgress().size());
+ Map.Entry<Long, TransactionManager.InProgressTx> inProgressTx =
+ snapshot2.getInProgress().entrySet().iterator().next();
+ assertEquals(TransactionType.LONG, inProgressTx.getValue().getType());
+
+ // save a new snapshot
+ txManager2.stopAndWait();
+
+ TransactionStateStorage txStorage2 = injector2.getInstance(TransactionStateStorage.class);
+ txStorage2.startAndWait();
+
+ TransactionSnapshot snapshot3 = txStorage2.getLatestSnapshot();
+ // full snapshot should have deserialized correctly without any fixups
+ assertEquals(snapshot2.getInProgress(), snapshot3.getInProgress());
+ assertEquals(snapshot2, snapshot3);
+ txStorage2.stopAndWait();
+ }
+
+ @Test
+ public void testSnapshotCodecProviderConfiguration() throws Exception {
+ Configuration conf = new Configuration(false);
+ StringBuilder buf = new StringBuilder();
+ for (Class c : TxConstants.Persist.DEFAULT_TX_SNAPHOT_CODEC_CLASSES) {
+ if (buf.length() > 0) {
+ buf.append(",\n ");
+ }
+ buf.append(c.getName());
+ }
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, buf.toString());
+
+ SnapshotCodecProvider codecProvider = new SnapshotCodecProvider(conf);
+ SnapshotCodec v1codec = codecProvider.getCodecForVersion(new DefaultSnapshotCodec().getVersion());
+ assertNotNull(v1codec);
+ assertTrue(v1codec instanceof DefaultSnapshotCodec);
+
+ SnapshotCodec v2codec = codecProvider.getCodecForVersion(new SnapshotCodecV2().getVersion());
+ assertNotNull(v2codec);
+ assertTrue(v2codec instanceof SnapshotCodecV2);
+
+ SnapshotCodec v3codec = codecProvider.getCodecForVersion(new SnapshotCodecV3().getVersion());
+ assertNotNull(v3codec);
+ assertTrue(v3codec instanceof SnapshotCodecV3);
+
+ SnapshotCodec v4codec = codecProvider.getCodecForVersion(new SnapshotCodecV4().getVersion());
+ assertNotNull(v4codec);
+ assertTrue(v4codec instanceof SnapshotCodecV4);
+ }
+
+ @Test
+ public void testSnapshotCodecV4() throws IOException, TransactionNotInProgressException {
+ File testDir = tmpDir.newFolder("testSnapshotCodecV4");
+ Configuration conf = new Configuration();
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
+
+ Injector injector = Guice.createInjector(new ConfigModule(conf),
+ new DiscoveryModules().getSingleNodeModules(),
+ new TransactionModules().getSingleNodeModules());
+
+ TransactionManager txManager = injector.getInstance(TransactionManager.class);
+ txManager.startAndWait();
+
+ // Create a transaction and a checkpoint transaction
+ Transaction transaction = txManager.startLong();
+ Transaction checkpointTx = txManager.checkpoint(transaction);
+
+ // shutdown to force a snapshot
+ txManager.stopAndWait();
+
+ // Validate the snapshot on disk
+ TransactionStateStorage txStorage = injector.getInstance(TransactionStateStorage.class);
+ txStorage.startAndWait();
+
+ TransactionSnapshot snapshot = txStorage.getLatestSnapshot();
+ TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState();
+ assertTransactionVisibilityStateEquals(snapshot, txVisibilityState);
+
+ Map<Long, TransactionManager.InProgressTx> inProgress = snapshot.getInProgress();
+ Assert.assertEquals(1, inProgress.size());
+
+ TransactionManager.InProgressTx inProgressTx = inProgress.get(transaction.getTransactionId());
+ Assert.assertNotNull(inProgressTx);
+ Assert.assertArrayEquals(checkpointTx.getCheckpointWritePointers(),
+ inProgressTx.getCheckpointWritePointers().toLongArray());
+
+ txStorage.stopAndWait();
+
+ // start a new Tx manager to see if the transaction is restored correctly.
+ Injector injector2 = Guice.createInjector(new ConfigModule(conf),
+ new DiscoveryModules().getSingleNodeModules(),
+ new TransactionModules().getSingleNodeModules());
+
+ txManager = injector2.getInstance(TransactionManager.class);
+ txManager.startAndWait();
+
+ // state should be recovered
+ snapshot = txManager.getCurrentState();
+ inProgress = snapshot.getInProgress();
+ Assert.assertEquals(1, inProgress.size());
+
+ inProgressTx = inProgress.get(transaction.getTransactionId());
+ Assert.assertNotNull(inProgressTx);
+ Assert.assertArrayEquals(checkpointTx.getCheckpointWritePointers(),
+ inProgressTx.getCheckpointWritePointers().toLongArray());
+
+ // Should be able to commit the transaction
+ Assert.assertTrue(txManager.canCommit(checkpointTx, Collections.<byte[]>emptyList()));
+ Assert.assertTrue(txManager.commit(checkpointTx));
+
+ // save a new snapshot
+ txManager.stopAndWait();
+
+ TransactionStateStorage txStorage2 = injector2.getInstance(TransactionStateStorage.class);
+ txStorage2.startAndWait();
+
+ snapshot = txStorage2.getLatestSnapshot();
+ Assert.assertTrue(snapshot.getInProgress().isEmpty());
+ txStorage2.stopAndWait();
+ }
+
+ private void assertTransactionVisibilityStateEquals(TransactionVisibilityState expected,
+ TransactionVisibilityState input) {
+ Assert.assertEquals(expected.getTimestamp(), input.getTimestamp());
+ Assert.assertEquals(expected.getReadPointer(), input.getReadPointer());
+ Assert.assertEquals(expected.getWritePointer(), input.getWritePointer());
+ Assert.assertEquals(expected.getInProgress(), input.getInProgress());
+ Assert.assertEquals(expected.getInvalid(), input.getInvalid());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/util/AbstractConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/util/AbstractConfigurationProviderTest.java b/tephra-core/src/test/java/org/apache/tephra/util/AbstractConfigurationProviderTest.java
new file mode 100644
index 0000000..8526b75
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/util/AbstractConfigurationProviderTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ *
+ */
+public abstract class AbstractConfigurationProviderTest {
+ @Test
+ public void testVersionFactory() {
+ HBaseVersion.Version foundVersion = HBaseVersion.get();
+ assertEquals(getExpectedVersion(), foundVersion);
+ }
+
+ protected abstract HBaseVersion.Version getExpectedVersion();
+
+ @Test
+ public void testConfigurationProvider() {
+ Configuration conf = new Configuration();
+ conf.set("foo", "bar");
+ Configuration newConf = new ConfigurationFactory().get(conf);
+ assertNotNull(newConf);
+ assertEquals("bar", newConf.get("foo"));
+ }
+}
[25/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/util/HBaseVersionTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/util/HBaseVersionTest.java b/tephra-core/src/test/java/org/apache/tephra/util/HBaseVersionTest.java
new file mode 100644
index 0000000..a2344ba
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/util/HBaseVersionTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import org.junit.Test;
+
+import java.text.ParseException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests for HBase version parsing.
+ */
+public class HBaseVersionTest {
+ @Test
+ public void testVersionNumber() throws Exception {
+ HBaseVersion.VersionNumber ver = HBaseVersion.VersionNumber.create("1");
+ assertVersionNumber(ver, 1, null, null, null, false);
+
+ ver = HBaseVersion.VersionNumber.create("1-SNAPSHOT");
+ assertVersionNumber(ver, 1, null, null, null, true);
+
+ ver = HBaseVersion.VersionNumber.create("1-foo");
+ assertVersionNumber(ver, 1, null, null, "foo", false);
+
+ ver = HBaseVersion.VersionNumber.create("1-foo-SNAPSHOT");
+ assertVersionNumber(ver, 1, null, null, "foo", true);
+
+ ver = HBaseVersion.VersionNumber.create("10.0");
+ assertVersionNumber(ver, 10, 0, null, null, false);
+
+ ver = HBaseVersion.VersionNumber.create("10.0-bar");
+ assertVersionNumber(ver, 10, 0, null, "bar", false);
+
+ ver = HBaseVersion.VersionNumber.create("3.2.1");
+ assertVersionNumber(ver, 3, 2, 1, null, false);
+
+ ver = HBaseVersion.VersionNumber.create("3.2.1-SNAPSHOT");
+ assertVersionNumber(ver, 3, 2, 1, null, true);
+
+ ver = HBaseVersion.VersionNumber.create("3.2.1-baz");
+ assertVersionNumber(ver, 3, 2, 1, "baz", false);
+
+ ver = HBaseVersion.VersionNumber.create("3.2.1-baz1.2.3");
+ assertVersionNumber(ver, 3, 2, 1, "baz1.2.3", false);
+
+ ver = HBaseVersion.VersionNumber.create("3.2.1-baz1.2.3-SNAPSHOT");
+ assertVersionNumber(ver, 3, 2, 1, "baz1.2.3", true);
+
+ try {
+ ver = HBaseVersion.VersionNumber.create("abc");
+ fail("Invalid verison number 'abc' should have thrown a ParseException");
+ } catch (ParseException pe) {
+ // expected
+ }
+
+ try {
+ ver = HBaseVersion.VersionNumber.create("1.a.b");
+ fail("Invalid verison number '1.a.b' should have thrown a ParseException");
+ } catch (ParseException pe) {
+ // expected
+ }
+
+ ver = HBaseVersion.VersionNumber.create("1.2.0-CDH5.7.0");
+ assertVersionNumber(ver, 1, 2, 0, "CDH5.7.0", false);
+ }
+
+ private void assertVersionNumber(HBaseVersion.VersionNumber version, Integer expectedMajor, Integer expectedMinor,
+ Integer expectedPatch, String expectedClassifier, boolean snapshot) {
+ if (expectedMajor == null) {
+ assertNull(version.getMajor());
+ } else {
+ assertEquals(expectedMajor, version.getMajor());
+ }
+ if (expectedMinor == null) {
+ assertNull(version.getMinor());
+ } else {
+ assertEquals(expectedMinor, version.getMinor());
+ }
+ if (expectedPatch == null) {
+ assertNull(version.getPatch());
+ } else {
+ assertEquals(expectedPatch, version.getPatch());
+ }
+ if (expectedClassifier == null) {
+ assertNull(version.getClassifier());
+ } else {
+ assertEquals(expectedClassifier, version.getClassifier());
+ }
+ assertEquals(snapshot, version.isSnapshot());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/util/TransactionEditUtil.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/util/TransactionEditUtil.java b/tephra-core/src/test/java/org/apache/tephra/util/TransactionEditUtil.java
new file mode 100644
index 0000000..854ccdd
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/util/TransactionEditUtil.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.persist.TransactionEdit;
+
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+/**
+ * Util class for {@link TransactionEdit} related tests.
+ */
+public final class TransactionEditUtil {
+ private static Random random = new Random();
+
+ /**
+ * Generates a number of semi-random {@link TransactionEdit} instances.
+ * These are just randomly selected from the possible states, so would not necessarily reflect a real-world
+ * distribution.
+ *
+ * @param numEntries how many entries to generate in the returned list.
+ * @return a list of randomly generated transaction log edits.
+ */
+ public static List<TransactionEdit> createRandomEdits(int numEntries) {
+ List<TransactionEdit> edits = Lists.newArrayListWithCapacity(numEntries);
+ for (int i = 0; i < numEntries; i++) {
+ TransactionEdit.State nextType = TransactionEdit.State.values()[random.nextInt(6)];
+ long writePointer = Math.abs(random.nextLong());
+ switch (nextType) {
+ case INPROGRESS:
+ edits.add(
+ TransactionEdit.createStarted(writePointer, writePointer - 1,
+ System.currentTimeMillis() + 300000L, TransactionType.SHORT));
+ break;
+ case COMMITTING:
+ edits.add(TransactionEdit.createCommitting(writePointer, generateChangeSet(10)));
+ break;
+ case COMMITTED:
+ edits.add(TransactionEdit.createCommitted(writePointer, generateChangeSet(10), writePointer + 1,
+ random.nextBoolean()));
+ break;
+ case INVALID:
+ edits.add(TransactionEdit.createInvalid(writePointer));
+ break;
+ case ABORTED:
+ edits.add(TransactionEdit.createAborted(writePointer, TransactionType.SHORT, null));
+ break;
+ case MOVE_WATERMARK:
+ edits.add(TransactionEdit.createMoveWatermark(writePointer));
+ break;
+ }
+ }
+ return edits;
+ }
+
+ private static Set<ChangeId> generateChangeSet(int numEntries) {
+ Set<ChangeId> changes = Sets.newHashSet();
+ for (int i = 0; i < numEntries; i++) {
+ byte[] bytes = new byte[8];
+ random.nextBytes(bytes);
+ changes.add(new ChangeId(bytes));
+ }
+ return changes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java b/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java
new file mode 100644
index 0000000..7743105
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import org.apache.tephra.Transaction;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test cases for {@link TxUtils} utility methods.
+ */
+public class TxUtilsTest {
+ @Test
+ public void testMaxVisibleTimestamp() {
+ // make sure we don't overflow with MAX_VALUE write pointer
+ assertEquals(Long.MAX_VALUE, TxUtils.getMaxVisibleTimestamp(Transaction.ALL_VISIBLE_LATEST));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/visibility/VisibilityFenceTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/visibility/VisibilityFenceTest.java b/tephra-core/src/test/java/org/apache/tephra/visibility/VisibilityFenceTest.java
new file mode 100644
index 0000000..1effcfd
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/visibility/VisibilityFenceTest.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.visibility;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionAware;
+import org.apache.tephra.TransactionConflictException;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * The following are all the possible cases when using {@link VisibilityFence}.
+ *
+ * In the below table,
+ * "Read Txn" refers to the transaction that contains the read fence
+ * "Before Write", "During Write" and "After Write" refer to the write transaction time
+ * "Before Write Fence", "During Write Fence", "After Write Fence" refer to the write fence transaction time
+ *
+ * Timeline is: Before Write < During Write < After Write < Before Write Fence < During Write Fence <
+ * After Write Fence
+ *
+ * +------+----------------------+----------------------+--------------------+--------------------+
+ * | Case | Read Txn Start | Read Txn Commit | Conflict on Commit | Conflict on Commit |
+ * | | | | of Read Txn | of Write Fence |
+ * +------+----------------------+----------------------+--------------------+--------------------+
+ * | 1 | Before Write | Before Write | No | No |
+ * | 2 | Before Write | During Write | No | No |
+ * | 3 | Before Write | After Write | No | No |
+ * | 4 | Before Write | Before Write Fence | No | No |
+ * | 5 | Before Write | During Write Fence | No | Yes |
+ * | 6 | Before Write | After Write Fence | Yes | No |
+ * | | | | | |
+ * | 7 | During Write | During Write | No | No |
+ * | 8 | During Write | After Write | No | No |
+ * | 9 | During Write | Before Write Fence | No | No |
+ * | 10 | During Write | During Write Fence | No | Yes |
+ * | 11 | During Write | After Write Fence | Yes | No |
+ * | | | | | |
+ * | 12 | After Write | After Write | No | No |
+ * | 13 | After Write | Before Write Fence | No | No |
+ * | 14 | After Write | During Write Fence | No | Yes # |
+ * | 15 | After Write | After Write Fence | Yes # | No |
+ * | | | | | |
+ * | 16 | Before Write Fence | Before Write Fence | No | No |
+ * | 17 | Before Write Fence | During Write Fence | No | Yes # |
+ * | 18 | Before Write Fence | After Write Fence | Yes # | No |
+ * | | | | | |
+ * | 19 | During Write Fence | During Write Fence | No | No |
+ * | 20 | During Write Fence | After Write Fence | No | No |
+ * | | | | | |
+ * | 21 | After Write Fence | After Write Fence | No | No |
+ * +------+----------------------+----------------------+--------------------+--------------------+
+ *
+ * Note: Cases marked with '#' in conflict column should not conflict, however current implementation causes
+ * them to conflict. The remaining conflicts are a result of the fence.
+ *
+ * In the current implementation of VisibilityFence, read txns that start "Before Write", "During Write",
+ * and "After Write" can be represented by read txns that start "Before Write Fence".
+ * Verifying cases 16, 17, 18, 20 and 21 will effectively cover all other cases.
+ */
+public class VisibilityFenceTest {
+ private static Configuration conf = new Configuration();
+
+ private static TransactionManager txManager = null;
+
+ @BeforeClass
+ public static void before() {
+ txManager = new TransactionManager(conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
+ txManager.startAndWait();
+ }
+
+ @AfterClass
+ public static void after() {
+ txManager.stopAndWait();
+ }
+
+ @Test
+ public void testFence1() throws Exception {
+ byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
+
+ // Writer updates data here in a separate transaction (code not shown)
+ // start tx
+ // update
+ // commit tx
+
+ // Readers use fence to indicate that they are interested in changes to specific data
+ TransactionAware readFenceCase16 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContextCase16 =
+ new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase16);
+ readTxContextCase16.start();
+ readTxContextCase16.finish();
+
+ TransactionAware readFenceCase17 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContextCase17 =
+ new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase17);
+ readTxContextCase17.start();
+
+ TransactionAware readFenceCase18 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContextCase18 =
+ new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase18);
+ readTxContextCase18.start();
+
+ // Now writer needs to wait for in-progress readers to see the change, it uses write fence to do so
+ // Start write fence txn
+ TransactionAware writeFence = new WriteFence(fenceId);
+ TransactionContext writeTxContext = new TransactionContext(new InMemoryTxSystemClient(txManager), writeFence);
+ writeTxContext.start();
+
+ TransactionAware readFenceCase20 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContextCase20 =
+ new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase20);
+ readTxContextCase20.start();
+
+ readTxContextCase17.finish();
+
+ assertTxnConflict(writeTxContext);
+ writeTxContext.start();
+
+ // Commit write fence txn can commit without conflicts at this point
+ writeTxContext.finish();
+
+ TransactionAware readFenceCase21 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContextCase21 =
+ new TransactionContext(new InMemoryTxSystemClient(txManager), readFenceCase21);
+ readTxContextCase21.start();
+
+ assertTxnConflict(readTxContextCase18);
+ readTxContextCase20.finish();
+ readTxContextCase21.finish();
+ }
+
+ private void assertTxnConflict(TransactionContext txContext) throws Exception {
+ try {
+ txContext.finish();
+ Assert.fail("Expected transaction to fail");
+ } catch (TransactionConflictException e) {
+ // Expected
+ txContext.abort();
+ }
+ }
+
+ @Test
+ public void testFence2() throws Exception {
+ byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
+
+ // Readers use fence to indicate that they are interested in changes to specific data
+ // Reader 1
+ TransactionAware readFence1 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence1);
+ readTxContext1.start();
+
+ // Reader 2
+ TransactionAware readFence2 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence2);
+ readTxContext2.start();
+
+ // Reader 3
+ TransactionAware readFence3 = VisibilityFence.create(fenceId);
+ TransactionContext readTxContext3 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence3);
+ readTxContext3.start();
+
+ // Writer updates data here in a separate transaction (code not shown)
+ // start tx
+ // update
+ // commit tx
+
+ // Now writer needs to wait for readers 1, 2, and 3 to see the change, it uses write fence to do so
+ TransactionAware writeFence = new WriteFence(fenceId);
+ TransactionContext writeTxContext = new TransactionContext(new InMemoryTxSystemClient(txManager), writeFence);
+ writeTxContext.start();
+
+ // Reader 1 commits before writeFence is committed
+ readTxContext1.finish();
+
+ try {
+ // writeFence will throw exception since Reader 1 committed without seeing changes
+ writeTxContext.finish();
+ Assert.fail("Expected transaction to fail");
+ } catch (TransactionConflictException e) {
+ // Expected
+ writeTxContext.abort();
+ }
+
+ // Start over writeFence again
+ writeTxContext.start();
+
+ // Now, Reader 3 commits before writeFence
+ // Note that Reader 3 does not conflict with Reader 1
+ readTxContext3.finish();
+
+ try {
+ // writeFence will throw exception again since Reader 3 committed without seeing changes
+ writeTxContext.finish();
+ Assert.fail("Expected transaction to fail");
+ } catch (TransactionConflictException e) {
+ // Expected
+ writeTxContext.abort();
+ }
+
+ // Start over writeFence again
+ writeTxContext.start();
+ // This time writeFence commits before the other readers
+ writeTxContext.finish();
+
+ // After this point all readers will see the change
+
+ try {
+ // Reader 2 commits after writeFence, hence this commit with throw exception
+ readTxContext2.finish();
+ Assert.fail("Expected transaction to fail");
+ } catch (TransactionConflictException e) {
+ // Expected
+ readTxContext2.abort();
+ }
+
+ // Reader 2 has to abort and start over again. It will see the changes now.
+ readTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), readFence2);
+ readTxContext2.start();
+ readTxContext2.finish();
+ }
+
+ @Test
+ public void testFenceAwait() throws Exception {
+ byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
+
+ final TransactionContext fence1 = new TransactionContext(new InMemoryTxSystemClient(txManager),
+ VisibilityFence.create(fenceId));
+ fence1.start();
+ final TransactionContext fence2 = new TransactionContext(new InMemoryTxSystemClient(txManager),
+ VisibilityFence.create(fenceId));
+ fence2.start();
+ TransactionContext fence3 = new TransactionContext(new InMemoryTxSystemClient(txManager),
+ VisibilityFence.create(fenceId));
+ fence3.start();
+
+ final AtomicInteger attempts = new AtomicInteger();
+ TransactionSystemClient customTxClient = new InMemoryTxSystemClient(txManager) {
+ @Override
+ public Transaction startShort() {
+ Transaction transaction = super.startShort();
+ try {
+ switch (attempts.getAndIncrement()) {
+ case 0:
+ fence1.finish();
+ break;
+ case 1:
+ fence2.finish();
+ break;
+ case 2:
+ break;
+ default:
+ throw new IllegalStateException("Unexpected state");
+ }
+ } catch (TransactionFailureException e) {
+ Throwables.propagate(e);
+ }
+ return transaction;
+ }
+ };
+
+ FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, customTxClient);
+ fenceWait.await(1000, TimeUnit.MILLISECONDS);
+ Assert.assertEquals(3, attempts.get());
+
+ try {
+ fence3.finish();
+ Assert.fail("Expected transaction to fail");
+ } catch (TransactionConflictException e) {
+ // Expected exception
+ fence3.abort();
+ }
+
+ fence3.start();
+ fence3.finish();
+ }
+
+ @Test
+ public void testFenceTimeout() throws Exception {
+ byte[] fenceId = "test_table".getBytes(Charsets.UTF_8);
+
+ final TransactionContext fence1 = new TransactionContext(new InMemoryTxSystemClient(txManager),
+ VisibilityFence.create(fenceId));
+ fence1.start();
+
+ final long timeout = 100;
+ final TimeUnit timeUnit = TimeUnit.MILLISECONDS;
+ final AtomicInteger attempts = new AtomicInteger();
+ TransactionSystemClient customTxClient = new InMemoryTxSystemClient(txManager) {
+ @Override
+ public Transaction startShort() {
+ Transaction transaction = super.startShort();
+ try {
+ switch (attempts.getAndIncrement()) {
+ case 0:
+ fence1.finish();
+ break;
+ }
+ timeUnit.sleep(timeout + 1);
+ } catch (InterruptedException | TransactionFailureException e) {
+ Throwables.propagate(e);
+ }
+ return transaction;
+ }
+ };
+
+ try {
+ FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, customTxClient);
+ fenceWait.await(timeout, timeUnit);
+ Assert.fail("Expected await to fail");
+ } catch (TimeoutException e) {
+ // Expected exception
+ }
+ Assert.assertEquals(1, attempts.get());
+
+ FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, customTxClient);
+ fenceWait.await(timeout, timeUnit);
+ Assert.assertEquals(2, attempts.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-examples/src/main/java/co/cask/tephra/examples/BalanceBooks.java
----------------------------------------------------------------------
diff --git a/tephra-examples/src/main/java/co/cask/tephra/examples/BalanceBooks.java b/tephra-examples/src/main/java/co/cask/tephra/examples/BalanceBooks.java
deleted file mode 100644
index 7f82209..0000000
--- a/tephra-examples/src/main/java/co/cask/tephra/examples/BalanceBooks.java
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.examples;
-
-import co.cask.tephra.TransactionConflictException;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import co.cask.tephra.hbase10cdh.TransactionAwareHTable;
-import co.cask.tephra.hbase10cdh.coprocessor.TransactionProcessor;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import co.cask.tephra.util.ConfigurationFactory;
-import com.google.common.io.Closeables;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-/**
- * Simple example application that launches a number of concurrent clients, one per "account". Each client attempts to
- * make withdrawals from other clients, and deposit the same amount to its own account in a single transaction.
- * Since this means the client will be updating both its own row and the withdrawee's row, this will naturally lead to
- * transaction conflicts. All clients will run for a specified number of iterations. When the processing is complete,
- * the total sum of all rows should be zero, if transactional integrity was maintained.
- *
- * <p>
- * You can run the BalanceBooks application with the following command:
- * <pre>
- * ./bin/tephra run co.cask.tephra.examples.BalanceBooks [num clients] [num iterations]
- * </pre>
- * where <code>[num clients]</code> is the number of concurrent client threads to use, and
- * <code>[num iterations]</code> is the number of "transfer" operations to perform per client thread.
- * </p>
- */
-public class BalanceBooks implements Closeable {
- private static final Logger LOG = LoggerFactory.getLogger(BalanceBooks.class);
-
- private static final int MAX_AMOUNT = 100;
- private static final byte[] TABLE = Bytes.toBytes("testbalances");
- private static final byte[] FAMILY = Bytes.toBytes("f");
- private static final byte[] COL = Bytes.toBytes("b");
-
- private final int totalClients;
- private final int iterations;
-
- private Configuration conf;
- private ZKClientService zkClient;
- private TransactionServiceClient txClient;
- private HConnection conn;
-
- public BalanceBooks(int totalClients, int iterations) {
- this(totalClients, iterations, new ConfigurationFactory().get());
- }
-
- public BalanceBooks(int totalClients, int iterations, Configuration conf) {
- this.totalClients = totalClients;
- this.iterations = iterations;
- this.conf = conf;
- }
-
- /**
- * Sets up common resources required by all clients.
- */
- public void init() throws IOException {
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- new TransactionModules().getDistributedModules(),
- new TransactionClientModule()
- );
-
- zkClient = injector.getInstance(ZKClientService.class);
- zkClient.startAndWait();
- txClient = injector.getInstance(TransactionServiceClient.class);
-
- createTableIfNotExists(conf, TABLE, new byte[][]{ FAMILY });
- conn = HConnectionManager.createConnection(conf);
- }
-
- /**
- * Runs all clients and waits for them to complete.
- */
- public void run() throws IOException, InterruptedException {
- List<Client> clients = new ArrayList<>(totalClients);
- for (int i = 0; i < totalClients; i++) {
- Client c = new Client(i, totalClients, iterations);
- c.init(txClient, conn.getTable(TABLE));
- c.start();
- clients.add(c);
- }
-
- for (Client c : clients) {
- c.join();
- Closeables.closeQuietly(c);
- }
- }
-
- /**
- * Validates the current state of the data stored at the end of the test. Each update by a client consists of two
- * parts: a withdrawal of a random amount from a randomly select other account, and a corresponding to deposit to
- * the client's own account. So, if all the updates were performed consistently (no partial updates or partial
- * rollbacks), then the total sum of all balances at the end should be 0.
- */
- public boolean verify() {
- boolean success = false;
- try {
- TransactionAwareHTable table = new TransactionAwareHTable(conn.getTable(TABLE));
- TransactionContext context = new TransactionContext(txClient, table);
-
- LOG.info("VERIFYING BALANCES");
- context.start();
- long totalBalance = 0;
- ResultScanner scanner = table.getScanner(new Scan());
- try {
- for (Result r : scanner) {
- if (!r.isEmpty()) {
- int rowId = Bytes.toInt(r.getRow());
- long balance = Bytes.toLong(r.getValue(FAMILY, COL));
- totalBalance += balance;
- LOG.info("Client #{}: balance = ${}", rowId, balance);
- }
- }
- } finally {
- if (scanner != null) {
- Closeables.closeQuietly(scanner);
- }
- }
- if (totalBalance == 0) {
- LOG.info("PASSED!");
- success = true;
- } else {
- LOG.info("FAILED! Total balance should be 0 but was {}", totalBalance);
- }
- context.finish();
- } catch (Exception e) {
- LOG.error("Failed verification check", e);
- }
- return success;
- }
-
- /**
- * Frees up the underlying resources common to all clients.
- */
- public void close() {
- try {
- if (conn != null) {
- conn.close();
- }
- } catch (IOException ignored) { }
-
- if (zkClient != null) {
- zkClient.stopAndWait();
- }
- }
-
- protected void createTableIfNotExists(Configuration conf, byte[] tableName, byte[][] columnFamilies)
- throws IOException {
- HBaseAdmin admin = new HBaseAdmin(conf);
- try {
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
- for (byte[] family : columnFamilies) {
- HColumnDescriptor columnDesc = new HColumnDescriptor(family);
- columnDesc.setMaxVersions(Integer.MAX_VALUE);
- desc.addFamily(columnDesc);
- }
- desc.addCoprocessor(TransactionProcessor.class.getName());
- admin.createTable(desc);
- } finally {
- if (admin != null) {
- try {
- admin.close();
- } catch (IOException ioe) {
- LOG.warn("Error closing HBaseAdmin", ioe);
- }
- }
- }
- }
-
- public static void main(String[] args) {
- if (args.length != 2) {
- System.err.println("Usage: java " + BalanceBooks.class.getName() + " <num clients> <iterations>");
- System.err.println("\twhere <num clients> >= 2");
- System.exit(1);
- }
-
- BalanceBooks bb = new BalanceBooks(Integer.parseInt(args[0]), Integer.parseInt(args[1]));
- try {
- bb.init();
- bb.run();
- bb.verify();
- } catch (Exception e) {
- LOG.error("Failed during BalanceBooks run", e);
- } finally {
- bb.close();
- }
- }
-
- /**
- * Represents a single client actor in the test. Each client runs as a separate thread.
- *
- * For the given number of iterations, the client will:
- * <ol>
- * <li>select a random other client from which to withdraw</li>
- * <li>select a random amount from 0 to MAX_AMOUNT</li>
- * <li>start a new transaction and: deduct the amount from the other client's acccount, and deposit
- * the same amount to its own account.</li>
- * </ol>
- *
- * Since multiple clients operate concurrently and contend over a set of constrained resources
- * (the client accounts), it is expected that a portion of the attempted transactions will encounter
- * conflicts, due to a simultaneous deduction from or deposit to one the same accounts which has successfully
- * committed first. In this case, the updates from the transaction encountering the conflict should be completely
- * rolled back, leaving the data in a consistent state.
- */
- private static class Client extends Thread implements Closeable {
- private final int id;
- private final int totalClients;
- private final int iterations;
-
- private final Random random = new Random();
-
- private TransactionContext txContext;
- private TransactionAwareHTable txTable;
-
-
- public Client(int id, int totalClients, int iterations) {
- this.id = id;
- this.totalClients = totalClients;
- this.iterations = iterations;
- }
-
- /**
- * Sets up any resources needed by the individual client.
- *
- * @param txClient the transaction client to use in accessing the transaciton service
- * @param table the HBase table instance to use for accessing storage
- */
- public void init(TransactionSystemClient txClient, HTableInterface table) {
- txTable = new TransactionAwareHTable(table);
- txContext = new TransactionContext(txClient, txTable);
- }
-
- public void run() {
- try {
- for (int i = 0; i < iterations; i++) {
- runOnce();
- }
- } catch (TransactionFailureException e) {
- LOG.error("Client #{}: Failed on exception", id, e);
- }
- }
-
- /**
- * Runs a single iteration of the client logic.
- */
- private void runOnce() throws TransactionFailureException {
- int withdrawee = getNextWithdrawee();
- int amount = getAmount();
-
- try {
- txContext.start();
- long withdraweeBalance = getCurrentBalance(withdrawee);
- long ownBalance = getCurrentBalance(id);
- long withdraweeNew = withdraweeBalance - amount;
- long ownNew = ownBalance + amount;
-
- setBalance(withdrawee, withdraweeNew);
- setBalance(id, ownNew);
- LOG.info("Client #{}: Withdrew ${} from #{}; withdrawee old={}, new={}; own old={}, new={}",
- id, amount, withdrawee, withdraweeBalance, withdraweeNew, ownBalance, ownNew);
- txContext.finish();
-
- } catch (IOException ioe) {
- LOG.error("Client #{}: Unhandled client failure", id, ioe);
- txContext.abort();
- } catch (TransactionConflictException tce) {
- LOG.info("CONFLICT: client #{} attempting to withdraw from #{}", id, withdrawee);
- txContext.abort(tce);
- } catch (TransactionFailureException tfe) {
- LOG.error("Client #{}: Unhandled transaction failure", id, tfe);
- txContext.abort(tfe);
- }
- }
-
- private long getCurrentBalance(int id) throws IOException {
- Result r = txTable.get(new Get(Bytes.toBytes(id)));
- byte[] balanceBytes = r.getValue(FAMILY, COL);
- if (balanceBytes == null) {
- return 0;
- }
- return Bytes.toLong(balanceBytes);
- }
-
- private void setBalance(int id, long balance) throws IOException {
- txTable.put(new Put(Bytes.toBytes(id)).add(FAMILY, COL, Bytes.toBytes(balance)));
- }
-
- private int getNextWithdrawee() {
- int next;
- do {
- next = random.nextInt(totalClients);
- } while (next == id);
- return next;
- }
-
- private int getAmount() {
- return random.nextInt(MAX_AMOUNT);
- }
-
- public void close() throws IOException {
- txTable.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-examples/src/main/java/co/cask/tephra/examples/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-examples/src/main/java/co/cask/tephra/examples/package-info.java b/tephra-examples/src/main/java/co/cask/tephra/examples/package-info.java
deleted file mode 100644
index 3848e6a..0000000
--- a/tephra-examples/src/main/java/co/cask/tephra/examples/package-info.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains example applications for Tephra designed to illustrate sample Tephra usage
- * and provide out-of-the-box sample applications which can be run to test cluster functionality.
- *
- * <p>Currently the following applications are provided:
- *
- * <ul>
- * <li><strong>BalanceBooks</strong> - this application runs a specified number of concurrent clients in separate
- * threads, which perform transactions to make withdrawals from each other's accounts and deposits to their own
- * accounts. At the end of the test, the total value of all account balances is verified to be equal to zero,
- * which confirms that transactional integrity was not violated.
- * </li>
- * </ul>
- * </p>
- *
- * <p>
- * Note that, for simplicity, the examples package is currently hardcoded to compile against a specific HBase
- * version (currently 1.0-cdh). In the future, we should provide Maven profiles to allow compiling the examples
- * against each of the supported HBase versions.
- * </p>
- */
-package co.cask.tephra.examples;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-examples/src/main/java/org/apache/tephra/examples/BalanceBooks.java
----------------------------------------------------------------------
diff --git a/tephra-examples/src/main/java/org/apache/tephra/examples/BalanceBooks.java b/tephra-examples/src/main/java/org/apache/tephra/examples/BalanceBooks.java
new file mode 100644
index 0000000..369b11e
--- /dev/null
+++ b/tephra-examples/src/main/java/org/apache/tephra/examples/BalanceBooks.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.examples;
+
+import com.google.common.io.Closeables;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.TransactionConflictException;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.distributed.TransactionServiceClient;
+import org.apache.tephra.hbase10cdh.TransactionAwareHTable;
+import org.apache.tephra.hbase10cdh.coprocessor.TransactionProcessor;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.tephra.util.ConfigurationFactory;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Simple example application that launches a number of concurrent clients, one per "account". Each client attempts to
+ * make withdrawals from other clients, and deposit the same amount to its own account in a single transaction.
+ * Since this means the client will be updating both its own row and the withdrawee's row, this will naturally lead to
+ * transaction conflicts. All clients will run for a specified number of iterations. When the processing is complete,
+ * the total sum of all rows should be zero, if transactional integrity was maintained.
+ *
+ * <p>
+ * You can run the BalanceBooks application with the following command:
+ * <pre>
+ * ./bin/tephra run BalanceBooks [num clients] [num iterations]
+ * </pre>
+ * where <code>[num clients]</code> is the number of concurrent client threads to use, and
+ * <code>[num iterations]</code> is the number of "transfer" operations to perform per client thread.
+ * </p>
+ */
+public class BalanceBooks implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(BalanceBooks.class);
+
+ private static final int MAX_AMOUNT = 100;
+ private static final byte[] TABLE = Bytes.toBytes("testbalances");
+ private static final byte[] FAMILY = Bytes.toBytes("f");
+ private static final byte[] COL = Bytes.toBytes("b");
+
+ private final int totalClients;
+ private final int iterations;
+
+ private Configuration conf;
+ private ZKClientService zkClient;
+ private TransactionServiceClient txClient;
+ private HConnection conn;
+
+ public BalanceBooks(int totalClients, int iterations) {
+ this(totalClients, iterations, new ConfigurationFactory().get());
+ }
+
+ public BalanceBooks(int totalClients, int iterations, Configuration conf) {
+ this.totalClients = totalClients;
+ this.iterations = iterations;
+ this.conf = conf;
+ }
+
+ /**
+ * Sets up common resources required by all clients.
+ */
+ public void init() throws IOException {
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ new TransactionModules().getDistributedModules(),
+ new TransactionClientModule()
+ );
+
+ zkClient = injector.getInstance(ZKClientService.class);
+ zkClient.startAndWait();
+ txClient = injector.getInstance(TransactionServiceClient.class);
+
+ createTableIfNotExists(conf, TABLE, new byte[][]{ FAMILY });
+ conn = HConnectionManager.createConnection(conf);
+ }
+
+ /**
+ * Runs all clients and waits for them to complete.
+ */
+ public void run() throws IOException, InterruptedException {
+ List<Client> clients = new ArrayList<>(totalClients);
+ for (int i = 0; i < totalClients; i++) {
+ Client c = new Client(i, totalClients, iterations);
+ c.init(txClient, conn.getTable(TABLE));
+ c.start();
+ clients.add(c);
+ }
+
+ for (Client c : clients) {
+ c.join();
+ Closeables.closeQuietly(c);
+ }
+ }
+
+ /**
+ * Validates the current state of the data stored at the end of the test. Each update by a client consists of two
+ * parts: a withdrawal of a random amount from a randomly select other account, and a corresponding to deposit to
+ * the client's own account. So, if all the updates were performed consistently (no partial updates or partial
+ * rollbacks), then the total sum of all balances at the end should be 0.
+ */
+ public boolean verify() {
+ boolean success = false;
+ try {
+ TransactionAwareHTable table = new TransactionAwareHTable(conn.getTable(TABLE));
+ TransactionContext context = new TransactionContext(txClient, table);
+
+ LOG.info("VERIFYING BALANCES");
+ context.start();
+ long totalBalance = 0;
+ ResultScanner scanner = table.getScanner(new Scan());
+ try {
+ for (Result r : scanner) {
+ if (!r.isEmpty()) {
+ int rowId = Bytes.toInt(r.getRow());
+ long balance = Bytes.toLong(r.getValue(FAMILY, COL));
+ totalBalance += balance;
+ LOG.info("Client #{}: balance = ${}", rowId, balance);
+ }
+ }
+ } finally {
+ if (scanner != null) {
+ Closeables.closeQuietly(scanner);
+ }
+ }
+ if (totalBalance == 0) {
+ LOG.info("PASSED!");
+ success = true;
+ } else {
+ LOG.info("FAILED! Total balance should be 0 but was {}", totalBalance);
+ }
+ context.finish();
+ } catch (Exception e) {
+ LOG.error("Failed verification check", e);
+ }
+ return success;
+ }
+
+ /**
+ * Frees up the underlying resources common to all clients.
+ */
+ public void close() {
+ try {
+ if (conn != null) {
+ conn.close();
+ }
+ } catch (IOException ignored) { }
+
+ if (zkClient != null) {
+ zkClient.stopAndWait();
+ }
+ }
+
+ protected void createTableIfNotExists(Configuration conf, byte[] tableName, byte[][] columnFamilies)
+ throws IOException {
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ try {
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+ for (byte[] family : columnFamilies) {
+ HColumnDescriptor columnDesc = new HColumnDescriptor(family);
+ columnDesc.setMaxVersions(Integer.MAX_VALUE);
+ desc.addFamily(columnDesc);
+ }
+ desc.addCoprocessor(TransactionProcessor.class.getName());
+ admin.createTable(desc);
+ } finally {
+ if (admin != null) {
+ try {
+ admin.close();
+ } catch (IOException ioe) {
+ LOG.warn("Error closing HBaseAdmin", ioe);
+ }
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ if (args.length != 2) {
+ System.err.println("Usage: java " + BalanceBooks.class.getName() + " <num clients> <iterations>");
+ System.err.println("\twhere <num clients> >= 2");
+ System.exit(1);
+ }
+
+ BalanceBooks bb = new BalanceBooks(Integer.parseInt(args[0]), Integer.parseInt(args[1]));
+ try {
+ bb.init();
+ bb.run();
+ bb.verify();
+ } catch (Exception e) {
+ LOG.error("Failed during BalanceBooks run", e);
+ } finally {
+ bb.close();
+ }
+ }
+
+ /**
+ * Represents a single client actor in the test. Each client runs as a separate thread.
+ *
+ * For the given number of iterations, the client will:
+ * <ol>
+ * <li>select a random other client from which to withdraw</li>
+ * <li>select a random amount from 0 to MAX_AMOUNT</li>
+ * <li>start a new transaction and: deduct the amount from the other client's acccount, and deposit
+ * the same amount to its own account.</li>
+ * </ol>
+ *
+ * Since multiple clients operate concurrently and contend over a set of constrained resources
+ * (the client accounts), it is expected that a portion of the attempted transactions will encounter
+ * conflicts, due to a simultaneous deduction from or deposit to one the same accounts which has successfully
+ * committed first. In this case, the updates from the transaction encountering the conflict should be completely
+ * rolled back, leaving the data in a consistent state.
+ */
+ private static class Client extends Thread implements Closeable {
+ private final int id;
+ private final int totalClients;
+ private final int iterations;
+
+ private final Random random = new Random();
+
+ private TransactionContext txContext;
+ private TransactionAwareHTable txTable;
+
+
+ public Client(int id, int totalClients, int iterations) {
+ this.id = id;
+ this.totalClients = totalClients;
+ this.iterations = iterations;
+ }
+
+ /**
+ * Sets up any resources needed by the individual client.
+ *
+ * @param txClient the transaction client to use in accessing the transaciton service
+ * @param table the HBase table instance to use for accessing storage
+ */
+ public void init(TransactionSystemClient txClient, HTableInterface table) {
+ txTable = new TransactionAwareHTable(table);
+ txContext = new TransactionContext(txClient, txTable);
+ }
+
+ public void run() {
+ try {
+ for (int i = 0; i < iterations; i++) {
+ runOnce();
+ }
+ } catch (TransactionFailureException e) {
+ LOG.error("Client #{}: Failed on exception", id, e);
+ }
+ }
+
+ /**
+ * Runs a single iteration of the client logic.
+ */
+ private void runOnce() throws TransactionFailureException {
+ int withdrawee = getNextWithdrawee();
+ int amount = getAmount();
+
+ try {
+ txContext.start();
+ long withdraweeBalance = getCurrentBalance(withdrawee);
+ long ownBalance = getCurrentBalance(id);
+ long withdraweeNew = withdraweeBalance - amount;
+ long ownNew = ownBalance + amount;
+
+ setBalance(withdrawee, withdraweeNew);
+ setBalance(id, ownNew);
+ LOG.info("Client #{}: Withdrew ${} from #{}; withdrawee old={}, new={}; own old={}, new={}",
+ id, amount, withdrawee, withdraweeBalance, withdraweeNew, ownBalance, ownNew);
+ txContext.finish();
+
+ } catch (IOException ioe) {
+ LOG.error("Client #{}: Unhandled client failure", id, ioe);
+ txContext.abort();
+ } catch (TransactionConflictException tce) {
+ LOG.info("CONFLICT: client #{} attempting to withdraw from #{}", id, withdrawee);
+ txContext.abort(tce);
+ } catch (TransactionFailureException tfe) {
+ LOG.error("Client #{}: Unhandled transaction failure", id, tfe);
+ txContext.abort(tfe);
+ }
+ }
+
+ private long getCurrentBalance(int id) throws IOException {
+ Result r = txTable.get(new Get(Bytes.toBytes(id)));
+ byte[] balanceBytes = r.getValue(FAMILY, COL);
+ if (balanceBytes == null) {
+ return 0;
+ }
+ return Bytes.toLong(balanceBytes);
+ }
+
+ private void setBalance(int id, long balance) throws IOException {
+ txTable.put(new Put(Bytes.toBytes(id)).add(FAMILY, COL, Bytes.toBytes(balance)));
+ }
+
+ private int getNextWithdrawee() {
+ int next;
+ do {
+ next = random.nextInt(totalClients);
+ } while (next == id);
+ return next;
+ }
+
+ private int getAmount() {
+ return random.nextInt(MAX_AMOUNT);
+ }
+
+ public void close() throws IOException {
+ txTable.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-examples/src/main/java/org/apache/tephra/examples/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-examples/src/main/java/org/apache/tephra/examples/package-info.java b/tephra-examples/src/main/java/org/apache/tephra/examples/package-info.java
new file mode 100644
index 0000000..172ad07
--- /dev/null
+++ b/tephra-examples/src/main/java/org/apache/tephra/examples/package-info.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains example applications for Tephra designed to illustrate sample Tephra usage
+ * and provide out-of-the-box sample applications which can be run to test cluster functionality.
+ *
+ * <p>Currently the following applications are provided:
+ *
+ * <ul>
+ * <li><strong>BalanceBooks</strong> - this application runs a specified number of concurrent clients in separate
+ * threads, which perform transactions to make withdrawals from each other's accounts and deposits to their own
+ * accounts. At the end of the test, the total value of all account balances is verified to be equal to zero,
+ * which confirms that transactional integrity was not violated.
+ * </li>
+ * </ul>
+ * </p>
+ *
+ * <p>
+ * Note that, for simplicity, the examples package is currently hardcoded to compile against a specific HBase
+ * version (currently 1.0-cdh). In the future, we should provide Maven profiles to allow compiling the examples
+ * against each of the supported HBase versions.
+ * </p>
+ */
+package org.apache.tephra.examples;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-examples/src/test/java/co/cask/tephra/examples/BalanceBooksTest.java
----------------------------------------------------------------------
diff --git a/tephra-examples/src/test/java/co/cask/tephra/examples/BalanceBooksTest.java b/tephra-examples/src/test/java/co/cask/tephra/examples/BalanceBooksTest.java
deleted file mode 100644
index a5a9036..0000000
--- a/tephra-examples/src/test/java/co/cask/tephra/examples/BalanceBooksTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.examples;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.distributed.TransactionService;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Scopes;
-import com.google.inject.util.Modules;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests the {@link BalanceBooks} program.
- */
-public class BalanceBooksTest {
- private static final Logger LOG = LoggerFactory.getLogger(BalanceBooksTest.class);
- private static HBaseTestingUtility testUtil;
- private static TransactionService txService;
- private static ZKClientService zkClientService;
-
- @BeforeClass
- public static void setup() throws Exception {
- testUtil = new HBaseTestingUtility();
- Configuration conf = testUtil.getConfiguration();
- conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, "/tx.snapshot");
-
- // Tune down the connection thread pool size
- conf.setInt("hbase.hconnection.threads.core", 5);
- conf.setInt("hbase.hconnection.threads.max", 10);
- // Tunn down handler threads in regionserver
- conf.setInt("hbase.regionserver.handler.count", 10);
-
- // Set to random port
- conf.setInt("hbase.master.port", 0);
- conf.setInt("hbase.master.info.port", 0);
- conf.setInt("hbase.regionserver.port", 0);
- conf.setInt("hbase.regionserver.info.port", 0);
-
- testUtil.startMiniCluster();
-
- String zkClusterKey = testUtil.getClusterKey(); // hostname:clientPort:parentZnode
- String zkQuorum = zkClusterKey.substring(0, zkClusterKey.lastIndexOf(':'));
- LOG.info("Zookeeper Quorum is running at {}", zkQuorum);
- conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkQuorum);
-
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- Modules.override(new TransactionModules().getDistributedModules())
- .with(new AbstractModule() {
- @Override
- protected void configure() {
- bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class).in(Scopes.SINGLETON);
- }
- }),
- new TransactionClientModule()
- );
-
- zkClientService = injector.getInstance(ZKClientService.class);
- zkClientService.startAndWait();
-
- // start a tx server
- txService = injector.getInstance(TransactionService.class);
- try {
- LOG.info("Starting transaction service");
- txService.startAndWait();
- } catch (Exception e) {
- LOG.error("Failed to start service: ", e);
- }
-
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- if (txService != null) {
- txService.stopAndWait();
- }
- if (zkClientService != null) {
- zkClientService.stopAndWait();
- }
- testUtil.shutdownMiniCluster();
- }
-
- @Test
- public void testBalanceBooks() throws Exception {
- BalanceBooks bb = new BalanceBooks(5, 100, testUtil.getConfiguration());
- try {
- bb.init();
- bb.run();
- assertTrue(bb.verify());
- } finally {
- bb.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-examples/src/test/java/org/apache/tephra/examples/BalanceBooksTest.java
----------------------------------------------------------------------
diff --git a/tephra-examples/src/test/java/org/apache/tephra/examples/BalanceBooksTest.java b/tephra-examples/src/test/java/org/apache/tephra/examples/BalanceBooksTest.java
new file mode 100644
index 0000000..d74a133
--- /dev/null
+++ b/tephra-examples/src/test/java/org/apache/tephra/examples/BalanceBooksTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.examples;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.util.Modules;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.distributed.TransactionService;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the {@link BalanceBooks} program.
+ */
+public class BalanceBooksTest {
+ private static final Logger LOG = LoggerFactory.getLogger(BalanceBooksTest.class);
+ private static HBaseTestingUtility testUtil;
+ private static TransactionService txService;
+ private static ZKClientService zkClientService;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ Configuration conf = testUtil.getConfiguration();
+ conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, "/tx.snapshot");
+
+ // Tune down the connection thread pool size
+ conf.setInt("hbase.hconnection.threads.core", 5);
+ conf.setInt("hbase.hconnection.threads.max", 10);
+ // Tunn down handler threads in regionserver
+ conf.setInt("hbase.regionserver.handler.count", 10);
+
+ // Set to random port
+ conf.setInt("hbase.master.port", 0);
+ conf.setInt("hbase.master.info.port", 0);
+ conf.setInt("hbase.regionserver.port", 0);
+ conf.setInt("hbase.regionserver.info.port", 0);
+
+ testUtil.startMiniCluster();
+
+ String zkClusterKey = testUtil.getClusterKey(); // hostname:clientPort:parentZnode
+ String zkQuorum = zkClusterKey.substring(0, zkClusterKey.lastIndexOf(':'));
+ LOG.info("Zookeeper Quorum is running at {}", zkQuorum);
+ conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkQuorum);
+
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ Modules.override(new TransactionModules().getDistributedModules())
+ .with(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class).in(Scopes.SINGLETON);
+ }
+ }),
+ new TransactionClientModule()
+ );
+
+ zkClientService = injector.getInstance(ZKClientService.class);
+ zkClientService.startAndWait();
+
+ // start a tx server
+ txService = injector.getInstance(TransactionService.class);
+ try {
+ LOG.info("Starting transaction service");
+ txService.startAndWait();
+ } catch (Exception e) {
+ LOG.error("Failed to start service: ", e);
+ }
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (txService != null) {
+ txService.stopAndWait();
+ }
+ if (zkClientService != null) {
+ zkClientService.stopAndWait();
+ }
+ testUtil.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testBalanceBooks() throws Exception {
+ BalanceBooks bb = new BalanceBooks(5, 100, testUtil.getConfiguration());
+ try {
+ bb.init();
+ bb.run();
+ assertTrue(bb.verify());
+ } finally {
+ bb.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/HBase96ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/HBase96ConfigurationProvider.java b/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/HBase96ConfigurationProvider.java
deleted file mode 100644
index 834ed59..0000000
--- a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/HBase96ConfigurationProvider.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96;
-
-import co.cask.tephra.util.ConfigurationProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-/**
- * HBase 0.96 version of {@link co.cask.tephra.util.ConfigurationProvider}.
- */
-public class HBase96ConfigurationProvider extends ConfigurationProvider {
- @Override
- public Configuration get() {
- return HBaseConfiguration.create();
- }
-
- @Override
- public Configuration get(Configuration baseConf) {
- return HBaseConfiguration.create(baseConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/SecondaryIndexTable.java b/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/SecondaryIndexTable.java
deleted file mode 100644
index eaa5564..0000000
--- a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/SecondaryIndexTable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase96;
-
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * A Transactional SecondaryIndexTable.
- */
-public class SecondaryIndexTable {
- private byte[] secondaryIndex;
- private TransactionAwareHTable transactionAwareHTable;
- private TransactionAwareHTable secondaryIndexTable;
- private TransactionContext transactionContext;
- private final TableName secondaryIndexTableName;
- private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
- private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
- private static final byte[] DELIMITER = new byte[] {0};
-
- public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
- byte[] secondaryIndex) {
- secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
- HTable secondaryIndexHTable = null;
- HBaseAdmin hBaseAdmin = null;
- try {
- hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
- if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
- hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
- }
- secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
- } catch (Exception e) {
- Throwables.propagate(e);
- } finally {
- try {
- hBaseAdmin.close();
- } catch (Exception e) {
- Throwables.propagate(e);
- }
- }
-
- this.secondaryIndex = secondaryIndex;
- this.transactionAwareHTable = new TransactionAwareHTable(hTable);
- this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
- this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
- secondaryIndexTable);
- }
-
- public Result get(Get get) throws IOException {
- return get(Collections.singletonList(get))[0];
- }
-
- public Result[] get(List<Get> gets) throws IOException {
- try {
- transactionContext.start();
- Result[] result = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return result;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public Result[] getByIndex(byte[] value) throws IOException {
- try {
- transactionContext.start();
- Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
- scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
- ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
-
- ArrayList<Get> gets = new ArrayList<Get>();
- for (Result result : indexScanner) {
- for (Cell cell : result.listCells()) {
- gets.add(new Get(cell.getValue()));
- }
- }
- Result[] results = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return results;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public void put(Put put) throws IOException {
- put(Collections.singletonList(put));
- }
-
-
- public void put(List<Put> puts) throws IOException {
- try {
- transactionContext.start();
- ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
- for (Put put : puts) {
- List<Put> indexPuts = new ArrayList<Put>();
- Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
- for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
- for (KeyValue value : family.getValue()) {
- if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
- secondaryIndex, 0, secondaryIndex.length)) {
- byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
- Bytes.add(value.getValue(), DELIMITER,
- value.getRow()));
- Put indexPut = new Put(secondaryRow);
- indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
- indexPuts.add(indexPut);
- }
- }
- }
- secondaryIndexPuts.addAll(indexPuts);
- }
- transactionAwareHTable.put(puts);
- secondaryIndexTable.put(secondaryIndexPuts);
- transactionContext.finish();
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- }
-}
[31/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersionSpecificFactory.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersionSpecificFactory.java b/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersionSpecificFactory.java
new file mode 100644
index 0000000..ebbbd18
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersionSpecificFactory.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import com.google.inject.Provider;
+import com.google.inject.ProvisionException;
+import org.apache.twill.internal.utils.Instances;
+
+/**
+ * Common class factory behavior for classes which need specific implementations depending on HBase versions.
+ * Specific factories can subclass this class and simply plug in the class names for their implementations.
+ *
+ * @param <T> Version specific class provided by this factory.
+ */
+public abstract class HBaseVersionSpecificFactory<T> implements Provider<T> {
+ @Override
+ public T get() {
+ T instance = null;
+ try {
+ switch (HBaseVersion.get()) {
+ case HBASE_94:
+ throw new ProvisionException("HBase 0.94 is no longer supported. Please upgrade to HBase 0.96 or newer.");
+ case HBASE_96:
+ instance = createInstance(getHBase96Classname());
+ break;
+ case HBASE_98:
+ instance = createInstance(getHBase98Classname());
+ break;
+ case HBASE_10:
+ instance = createInstance(getHBase10Classname());
+ break;
+ case HBASE_10_CDH:
+ instance = createInstance(getHBase10CDHClassname());
+ break;
+ case HBASE_11:
+ case HBASE_12_CDH:
+ instance = createInstance(getHBase11Classname());
+ break;
+ case UNKNOWN:
+ throw new ProvisionException("Unknown HBase version: " + HBaseVersion.getVersionString());
+ }
+ } catch (ClassNotFoundException cnfe) {
+ throw new ProvisionException(cnfe.getMessage(), cnfe);
+ }
+ return instance;
+ }
+
+ protected T createInstance(String className) throws ClassNotFoundException {
+ Class clz = Class.forName(className);
+ return (T) Instances.newInstance(clz);
+ }
+
+ protected abstract String getHBase96Classname();
+ protected abstract String getHBase98Classname();
+ protected abstract String getHBase10Classname();
+ protected abstract String getHBase10CDHClassname();
+ protected abstract String getHBase11Classname();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java b/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java
new file mode 100644
index 0000000..08b1545
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import com.google.common.primitives.Longs;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.persist.TransactionVisibilityState;
+
+import java.util.Map;
+
+/**
+ * Utility methods supporting transaction operations.
+ */
+public class TxUtils {
+
+ // Any cell with timestamp less than MAX_NON_TX_TIMESTAMP is assumed to be pre-existing data,
+ // i.e. data written before table was converted into transactional table using Tephra.
+ // Using 1.1 times current time to determine whether a timestamp is transactional timestamp or not is safe, and does
+ // not produce any false positives or false negatives.
+ //
+ // To prove this, let's say the earliest transactional timestamp written by Tephra was in year 2000, and the oldest
+ // that will be written is in year 2200.
+ // 01-Jan-2000 GMT is 946684800000 milliseconds since epoch.
+ // 31-Dec-2200 GMT is 7289654399000 milliseconds since epoch.
+ //
+ // Let's say that we enabled transactions on a table on 01-Jan-2000, then 1.1 * 946684800000 = 31-Dec-2002. Using
+ // 31-Dec-2002, we can safely say from 01-Jan-2000 onwards, whether a cell timestamp is
+ // non-transactional (<= 946684800000).
+ // Note that transactional timestamp will be greater than 946684800000000000 (> year 31969) at this instant.
+ //
+ // On the other end, let's say we enabled transactions on a table on 31-Dec-2200,
+ // then 1.1 * 7289654399000 = 07-Feb-2224. Again, we can use this time from 31-Dec-2200 onwards to say whether a
+ // cell timestamp is transactional (<= 7289654399000).
+ // Note that transactional timestamp will be greater than 7289654399000000000 (> year 232969) at this instant.
+ private static final long MAX_NON_TX_TIMESTAMP = (long) (System.currentTimeMillis() * 1.1);
+
+ /**
+ * Returns the oldest visible timestamp for the given transaction, based on the TTLs configured for each column
+ * family. If no TTL is set on any column family, the oldest visible timestamp will be {@code 0}.
+ * @param ttlByFamily A map of column family name to TTL value (in milliseconds)
+ * @param tx The current transaction
+ * @return The oldest timestamp that will be visible for the given transaction and TTL configuration
+ */
+ public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx) {
+ long maxTTL = getMaxTTL(ttlByFamily);
+ // we know that data will not be cleaned up while this tx is running up to this point as janitor uses it
+ return maxTTL < Long.MAX_VALUE ? tx.getVisibilityUpperBound() - maxTTL * TxConstants.MAX_TX_PER_MS : 0;
+ }
+
+ /**
+ * Returns the oldest visible timestamp for the given transaction, based on the TTLs configured for each column
+ * family. If no TTL is set on any column family, the oldest visible timestamp will be {@code 0}.
+ * @param ttlByFamily A map of column family name to TTL value (in milliseconds)
+ * @param tx The current transaction
+ * @param readNonTxnData indicates that the timestamp returned should allow reading non-transactional data
+ * @return The oldest timestamp that will be visible for the given transaction and TTL configuration
+ */
+ public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx, boolean readNonTxnData) {
+ if (readNonTxnData) {
+ long maxTTL = getMaxTTL(ttlByFamily);
+ return maxTTL < Long.MAX_VALUE ? System.currentTimeMillis() - maxTTL : 0;
+ }
+
+ return getOldestVisibleTimestamp(ttlByFamily, tx);
+ }
+
+ /**
+ * Returns the maximum timestamp to use for time-range operations, based on the given transaction.
+ * @param tx The current transaction
+ * @return The maximum timestamp (exclusive) to use for time-range operations
+ */
+ public static long getMaxVisibleTimestamp(Transaction tx) {
+ // NOTE: +1 here because we want read up to writepointer inclusive, but timerange's end is exclusive
+ // however, we also need to guard against overflow in the case write pointer is set to MAX_VALUE
+ return tx.getWritePointer() < Long.MAX_VALUE ?
+ tx.getWritePointer() + 1 : tx.getWritePointer();
+ }
+
+ /**
+ * Creates a "dummy" transaction based on the given txVisibilityState's state. This is not a "real" transaction in
+ * the sense that it has not been started, data should not be written with it, and it cannot be committed. However,
+ * this can still be useful for filtering data according to the txVisibilityState's state. Instead of the actual
+ * write pointer from the txVisibilityState, however, we use {@code Long.MAX_VALUE} to avoid mis-identifying any cells
+ * as being written by this transaction (and therefore visible).
+ */
+ public static Transaction createDummyTransaction(TransactionVisibilityState txVisibilityState) {
+ return new Transaction(txVisibilityState.getReadPointer(), Long.MAX_VALUE,
+ Longs.toArray(txVisibilityState.getInvalid()),
+ Longs.toArray(txVisibilityState.getInProgress().keySet()),
+ TxUtils.getFirstShortInProgress(txVisibilityState.getInProgress()), TransactionType.SHORT);
+ }
+
+ /**
+ * Returns the write pointer for the first "short" transaction that in the in-progress set, or
+ * {@link Transaction#NO_TX_IN_PROGRESS} if none.
+ */
+ public static long getFirstShortInProgress(Map<Long, TransactionManager.InProgressTx> inProgress) {
+ long firstShort = Transaction.NO_TX_IN_PROGRESS;
+ for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
+ if (!entry.getValue().isLongRunning()) {
+ firstShort = entry.getKey();
+ break;
+ }
+ }
+ return firstShort;
+ }
+
+ /**
+ * Returns the timestamp for calculating time to live for the given cell timestamp.
+ * This takes into account pre-existing non-transactional cells while calculating the time.
+ */
+ public static long getTimestampForTTL(long cellTs) {
+ return isPreExistingVersion(cellTs) ? cellTs * TxConstants.MAX_TX_PER_MS : cellTs;
+ }
+
+ /**
+ * Returns the max TTL for the given TTL values. Returns Long.MAX_VALUE if any of the column families has no TTL set.
+ */
+ private static long getMaxTTL(Map<byte[], Long> ttlByFamily) {
+ long maxTTL = 0;
+ for (Long familyTTL : ttlByFamily.values()) {
+ maxTTL = Math.max(familyTTL <= 0 ? Long.MAX_VALUE : familyTTL, maxTTL);
+ }
+ return maxTTL == 0 ? Long.MAX_VALUE : maxTTL;
+ }
+
+ /**
+ * Returns true if version was written before table was converted into transactional table, false otherwise.
+ */
+ public static boolean isPreExistingVersion(long version) {
+ return version < MAX_NON_TX_TIMESTAMP;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/visibility/DefaultFenceWait.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/visibility/DefaultFenceWait.java b/tephra-core/src/main/java/org/apache/tephra/visibility/DefaultFenceWait.java
new file mode 100644
index 0000000..7c1af8e
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/visibility/DefaultFenceWait.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.visibility;
+
+import com.google.common.base.Stopwatch;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Default implementation of {@link FenceWait}.
+ */
+public class DefaultFenceWait implements FenceWait {
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultFenceWait.class);
+
+ private final TransactionContext txContext;
+
+ DefaultFenceWait(TransactionContext txContext) {
+ this.txContext = txContext;
+ }
+
+ @Override
+ public void await(long timeout, TimeUnit timeUnit)
+ throws TransactionFailureException, InterruptedException, TimeoutException {
+ Stopwatch stopwatch = new Stopwatch();
+ stopwatch.start();
+ long sleepTimeMicros = timeUnit.toMicros(timeout) / 10;
+ // Have sleep time to be within 1 microsecond and 500 milliseconds
+ sleepTimeMicros = Math.max(Math.min(sleepTimeMicros, 500 * 1000), 1);
+ while (stopwatch.elapsedTime(timeUnit) < timeout) {
+ txContext.start();
+ try {
+ txContext.finish();
+ return;
+ } catch (TransactionFailureException e) {
+ LOG.error("Got exception waiting for fence. Sleeping for {} microseconds", sleepTimeMicros, e);
+ txContext.abort();
+ TimeUnit.MICROSECONDS.sleep(sleepTimeMicros);
+ }
+ }
+ throw new TimeoutException("Timeout waiting for fence");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/visibility/FenceWait.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/visibility/FenceWait.java b/tephra-core/src/main/java/org/apache/tephra/visibility/FenceWait.java
new file mode 100644
index 0000000..e1fb246
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/visibility/FenceWait.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.visibility;
+
+import org.apache.tephra.TransactionFailureException;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Used by a writer to wait on a fence so that changes are visible to all readers with in-progress transactions.
+ */
+public interface FenceWait {
+ /**
+ * Waits until the fence is complete, or till the timeout specified. The fence wait transaction will get re-tried
+ * several times until the timeout.
+ * <p>
+ *
+ * If a fence wait times out then it means there are still some readers with in-progress transactions that have not
+ * seen the change. In this case the wait will have to be retried using the same FenceWait object.
+ *
+ * @param timeout Maximum time to wait
+ * @param timeUnit {@link TimeUnit} for timeout and sleepTime
+ * @throws TransactionFailureException when not able to start fence wait transaction
+ * @throws InterruptedException on any interrupt
+ * @throws TimeoutException when timeout is reached
+ */
+ void await(long timeout, TimeUnit timeUnit)
+ throws TransactionFailureException, InterruptedException, TimeoutException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/visibility/ReadFence.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/visibility/ReadFence.java b/tephra-core/src/main/java/org/apache/tephra/visibility/ReadFence.java
new file mode 100644
index 0000000..a156d55
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/visibility/ReadFence.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.visibility;
+
+import com.google.common.primitives.Bytes;
+import com.google.common.primitives.Longs;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionAware;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Implementation of {@link VisibilityFence} used by reader.
+ */
+class ReadFence implements TransactionAware {
+ private final byte[] fenceId;
+ private Transaction tx;
+
+ public ReadFence(byte[] fenceId) {
+ this.fenceId = fenceId;
+ }
+
+ @Override
+ public void startTx(Transaction tx) {
+ this.tx = tx;
+ }
+
+ @Override
+ public void updateTx(Transaction tx) {
+ // Fences only need original transaction
+ }
+
+ @Override
+ public Collection<byte[]> getTxChanges() {
+ if (tx == null) {
+ throw new IllegalStateException("Transaction has not started yet");
+ }
+ return Collections.singleton(Bytes.concat(fenceId, Longs.toByteArray(tx.getTransactionId())));
+ }
+
+ @Override
+ public boolean commitTx() throws Exception {
+ // Nothing to persist
+ return true;
+ }
+
+ @Override
+ public void postTxCommit() {
+ tx = null;
+ }
+
+ @Override
+ public boolean rollbackTx() throws Exception {
+ // Nothing to rollback
+ return true;
+ }
+
+ @Override
+ public String getTransactionAwareName() {
+ return getClass().getSimpleName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/visibility/VisibilityFence.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/visibility/VisibilityFence.java b/tephra-core/src/main/java/org/apache/tephra/visibility/VisibilityFence.java
new file mode 100644
index 0000000..5d08246
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/visibility/VisibilityFence.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.visibility;
+
+import org.apache.tephra.TransactionAware;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.apache.tephra.TransactionSystemClient;
+
+import java.util.concurrent.TimeoutException;
+
+/**
+ * VisibilityFence is used to ensure that after a given point in time, all readers see an updated change
+ * that got committed.
+ * <p>
+ *
+ * Typically a reader will never conflict with a writer, since a reader only sees committed changes when its
+ * transaction started. However to ensure that after a given point all readers are aware of a change,
+ * we have to introduce a conflict between a reader and a writer that act on the same data concurrently.
+ *<p>
+ *
+ * This is done by the reader indicating that it is interested in changes to a piece of data by using a fence
+ * in its transaction. If there are no changes to the data when reader tries to commit the transaction
+ * containing the fence, the commit succeeds.
+ * <p>
+ *
+ * On the other hand, a writer updates the same data in a transaction. After the write transaction is committed,
+ * the writer then waits on the fence to ensure that all in-progress readers are aware of this update.
+ * When the wait on the fence returns successfully, it means that
+ * any in-progress readers that have not seen the change will not be allowed to commit anymore. This will
+ * force the readers to start a new transaction, and this ensures that the changes made by writer are visible
+ * to the readers.
+ *<p>
+ *
+ * In case an in-progress reader commits when the writer is waiting on the fence, then the wait method will retry
+ * until the given timeout.
+ * <p>
+ *
+ * Hence a successful await on a fence ensures that any reader (using the same fence) that successfully commits after
+ * this point onwards would see the change.
+ *
+ * <p>
+ * Sample reader code:
+ * <pre>
+ * <code>
+ * TransactionAware fence = VisibilityFence.create(fenceId);
+ * TransactionContext readTxContext = new TransactionContext(txClient, fence, table1, table2, ...);
+ * readTxContext.start();
+ *
+ * // do operations using table1, table2, etc.
+ *
+ * // finally commit
+ * try {
+ * readTxContext.finish();
+ * } catch (TransactionConflictException e) {
+ * // handle conflict by aborting and starting over with a new transaction
+ * }
+ * </code>
+ * </pre>
+ *<p>
+ *
+ * Sample writer code:
+ * <pre>
+ * <code>
+ * // start transaction
+ * // write change
+ * // commit transaction
+ *
+ * // Now wait on the fence (with the same fenceId as the readers) to ensure that all in-progress readers are
+ * aware of this change
+ * try {
+ * FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, txClient);
+ * fenceWait.await(50000, 50, TimeUnit.MILLISECONDS);
+ * } catch (TimeoutException e) {
+ * // await timed out, the change may not be visible to all in-progress readers.
+ * // Application has two options at this point:
+ * // 1. Revert the write. Re-try the write and fence wait again.
+ * // 2. Retry only the wait with the same fenceWait object (retry logic is not shown here).
+ * }
+ * </code>
+ * </pre>
+ *
+ * fenceId in the above samples refers to any id that both the readers and writer know for a given
+ * piece of data. Both readers and writer will have to use the same fenceId to synchronize on a given data.
+ * Typically fenceId uniquely identifies the data in question.
+ * For example, if the data is a table row, the fenceId can be composed of table name and row key.
+ * If the data is a table cell, the fenceId can be composed of table name, row key, and column key.
+ *<p>
+ *
+ * Note that in this implementation, any reader that starts a transaction after the write is committed, and
+ * while this read transaction is in-progress, if a writer successfully starts and completes an await on the fence then
+ * this reader will get a conflict while committing the fence even though this reader has seen the latest changes.
+ * This is because today there is no way to determine the commit time of a transaction.
+ */
+public final class VisibilityFence {
+ private VisibilityFence() {
+ // Cannot instantiate this class, all functionality is through static methods.
+ }
+
+ /**
+ * Used by a reader to get a fence that can be added to its transaction context.
+ *
+ * @param fenceId uniquely identifies the data that this fence is used to synchronize.
+ * If the data is a table cell then this id can be composed of the table name, row key
+ * and column key for the data.
+ * @return {@link TransactionAware} to be added to reader's transaction context.
+ */
+ public static TransactionAware create(byte[] fenceId) {
+ return new ReadFence(fenceId);
+ }
+
+ /**
+ * Used by a writer to wait on a fence so that changes are visible to all readers with in-progress transactions.
+ *
+ * @param fenceId uniquely identifies the data that this fence is used to synchronize.
+ * If the data is a table cell then this id can be composed of the table name, row key
+ * and column key for the data.
+ * @return {@link FenceWait} object
+ */
+ public static FenceWait prepareWait(byte[] fenceId, TransactionSystemClient txClient)
+ throws TransactionFailureException, InterruptedException, TimeoutException {
+ return new DefaultFenceWait(new TransactionContext(txClient, new WriteFence(fenceId)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/visibility/WriteFence.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/visibility/WriteFence.java b/tephra-core/src/main/java/org/apache/tephra/visibility/WriteFence.java
new file mode 100644
index 0000000..fe59efe
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/visibility/WriteFence.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.visibility;
+
+import com.google.common.primitives.Bytes;
+import com.google.common.primitives.Longs;
+import com.google.common.primitives.UnsignedBytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionAware;
+
+import java.util.Collection;
+import java.util.TreeSet;
+
+/**
+ * Implementation used by {@link FenceWait} to wait for a {@link VisibilityFence}.
+ */
+class WriteFence implements TransactionAware {
+ private final byte[] fenceId;
+ private Transaction tx;
+ private Collection<byte[]> inProgressChanges;
+
+ public WriteFence(byte[] fenceId) {
+ this.fenceId = fenceId;
+ }
+
+ @Override
+ public void startTx(Transaction tx) {
+ this.tx = tx;
+ if (inProgressChanges == null) {
+ inProgressChanges = new TreeSet<>(UnsignedBytes.lexicographicalComparator());
+ for (long inProgressTx : tx.getInProgress()) {
+ inProgressChanges.add(Bytes.concat(fenceId, Longs.toByteArray(inProgressTx)));
+ }
+ }
+ }
+
+ @Override
+ public void updateTx(Transaction tx) {
+ // Fences only need original transaction
+ }
+
+ @Override
+ public Collection<byte[]> getTxChanges() {
+ if (inProgressChanges == null || tx == null) {
+ throw new IllegalStateException("Transaction has not started yet");
+ }
+ return inProgressChanges;
+ }
+
+ @Override
+ public boolean commitTx() throws Exception {
+ // Nothing to persist
+ return true;
+ }
+
+ @Override
+ public void postTxCommit() {
+ tx = null;
+ }
+
+ @Override
+ public boolean rollbackTx() throws Exception {
+ // Nothing to rollback
+ return true;
+ }
+
+ @Override
+ public String getTransactionAwareName() {
+ return getClass().getSimpleName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicACLData.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicACLData.java b/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicACLData.java
new file mode 100644
index 0000000..8d95a42
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicACLData.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.zookeeper;
+
+import org.apache.twill.zookeeper.ACLData;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+
+import java.util.List;
+
+/**
+ * A straightforward implementation of {@link ACLData}.
+ */
+final class BasicACLData implements ACLData {
+
+ private final List<ACL> acl;
+ private final Stat stat;
+
+ BasicACLData(List<ACL> acl, Stat stat) {
+ this.acl = acl;
+ this.stat = stat;
+ }
+
+ @Override
+ public List<ACL> getACL() {
+ return acl;
+ }
+
+ @Override
+ public Stat getStat() {
+ return stat;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeChildren.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeChildren.java b/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeChildren.java
new file mode 100644
index 0000000..80edddb
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeChildren.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.zookeeper;
+
+import com.google.common.base.Objects;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.zookeeper.data.Stat;
+
+import java.util.List;
+
+/**
+ * Implementation of the {@link NodeChildren}.
+ */
+final class BasicNodeChildren implements NodeChildren {
+
+ private final Stat stat;
+ private final List<String> children;
+
+ BasicNodeChildren(List<String> children, Stat stat) {
+ this.stat = stat;
+ this.children = children;
+ }
+
+ @Override
+ public Stat getStat() {
+ return stat;
+ }
+
+ @Override
+ public List<String> getChildren() {
+ return children;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || !(o instanceof NodeChildren)) {
+ return false;
+ }
+
+ NodeChildren that = (NodeChildren) o;
+ return stat.equals(that.getStat()) && children.equals(that.getChildren());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(children, stat);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeData.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeData.java b/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeData.java
new file mode 100644
index 0000000..5df3475
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/zookeeper/BasicNodeData.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.zookeeper;
+
+import com.google.common.base.Objects;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.zookeeper.data.Stat;
+
+import java.util.Arrays;
+
+/**
+ * A straightforward implementation for {@link NodeData}.
+ */
+final class BasicNodeData implements NodeData {
+
+ private final byte[] data;
+ private final Stat stat;
+
+ BasicNodeData(byte[] data, Stat stat) {
+ this.data = data;
+ this.stat = stat;
+ }
+
+ @Override
+ public Stat getStat() {
+ return stat;
+ }
+
+ @Override
+ public byte[] getData() {
+ return data;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || !(o instanceof NodeData)) {
+ return false;
+ }
+
+ BasicNodeData that = (BasicNodeData) o;
+
+ return stat.equals(that.getStat()) && Arrays.equals(data, that.getData());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(data, stat);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/zookeeper/TephraZKClientService.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/zookeeper/TephraZKClientService.java b/tephra-core/src/main/java/org/apache/tephra/zookeeper/TephraZKClientService.java
new file mode 100644
index 0000000..2bc7a6a
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/zookeeper/TephraZKClientService.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.zookeeper;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.AbstractService;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.common.Threads;
+import org.apache.twill.internal.zookeeper.SettableOperationFuture;
+import org.apache.twill.zookeeper.ACLData;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.OperationFuture;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
+
+/**
+ * The implementation of {@link ZKClientService}.
+ */
+public class TephraZKClientService extends AbstractService implements ZKClientService, Watcher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TephraZKClientService.class);
+
+ private final String zkStr;
+ private final int sessionTimeout;
+ private final List<Watcher> connectionWatchers;
+ private final Multimap<String, byte[]> authInfos;
+ private final AtomicReference<ZooKeeper> zooKeeper;
+ private final Runnable stopTask;
+ private ExecutorService eventExecutor;
+
+ /**
+ * Create a new instance.
+ * @param zkStr zookeper connection string
+ * @param sessionTimeout timeout in milliseconds
+ * @param connectionWatcher watcher to set
+ * @param authInfos authorization bytes
+ */
+ public TephraZKClientService(String zkStr, int sessionTimeout,
+ Watcher connectionWatcher, Multimap<String, byte[]> authInfos) {
+ this.zkStr = zkStr;
+ this.sessionTimeout = sessionTimeout;
+ this.connectionWatchers = new CopyOnWriteArrayList<>();
+ this.authInfos = copyAuthInfo(authInfos);
+ addConnectionWatcher(connectionWatcher);
+
+ this.zooKeeper = new AtomicReference<>();
+ this.stopTask = createStopTask();
+ }
+
+ @Override
+ public Long getSessionId() {
+ ZooKeeper zk = zooKeeper.get();
+ return zk == null ? null : zk.getSessionId();
+ }
+
+ @Override
+ public String getConnectString() {
+ return zkStr;
+ }
+
+ @Override
+ public Cancellable addConnectionWatcher(final Watcher watcher) {
+ if (watcher == null) {
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ // No-op
+ }
+ };
+ }
+
+ // Invocation of connection watchers are already done inside the event thread,
+ // hence no need to wrap the watcher again.
+ connectionWatchers.add(watcher);
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ connectionWatchers.remove(watcher);
+ }
+ };
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode) {
+ return create(path, data, createMode, true);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data,
+ CreateMode createMode, boolean createParent) {
+ return create(path, data, createMode, createParent, ZooDefs.Ids.OPEN_ACL_UNSAFE);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data,
+ CreateMode createMode, Iterable<ACL> acl) {
+ return create(path, data, createMode, true, acl);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path) {
+ return exists(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path) {
+ return getChildren(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path) {
+ return getData(path, null);
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String path, byte[] data) {
+ return setData(path, data, -1);
+ }
+
+ @Override
+ public OperationFuture<String> delete(String path) {
+ return delete(path, -1);
+ }
+
+ @Override
+ public OperationFuture<Stat> setACL(String path, Iterable<ACL> acl) {
+ return setACL(path, acl, -1);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode,
+ boolean createParent, Iterable<ACL> acl) {
+ return doCreate(path, data, createMode, createParent, ImmutableList.copyOf(acl), false);
+ }
+
+ private OperationFuture<String> doCreate(final String path,
+ @Nullable final byte[] data,
+ final CreateMode createMode,
+ final boolean createParent,
+ final List<ACL> acl,
+ final boolean ignoreNodeExists) {
+ final SettableOperationFuture<String> createFuture = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().create(path, data, acl, createMode, Callbacks.STRING, createFuture);
+ if (!createParent) {
+ return createFuture;
+ }
+
+ // If create parent is request, return a different future
+ final SettableOperationFuture<String> result = SettableOperationFuture.create(path, eventExecutor);
+ // Watch for changes in the original future
+ Futures.addCallback(createFuture, new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String path) {
+ // Propagate if creation was successful
+ result.set(path);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // See if the failure can be handled
+ if (updateFailureResult(t, result, path, ignoreNodeExists)) {
+ return;
+ }
+ // Create the parent node
+ String parentPath = getParent(path);
+ if (parentPath.isEmpty()) {
+ result.setException(t);
+ return;
+ }
+ // Watch for parent creation complete. Parent is created with the unsafe ACL.
+ Futures.addCallback(doCreate(parentPath, null, CreateMode.PERSISTENT,
+ true, ZooDefs.Ids.OPEN_ACL_UNSAFE, true), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String parentPath) {
+ // Create the requested path again
+ Futures.addCallback(
+ doCreate(path, data, createMode, false, acl, ignoreNodeExists), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String pathResult) {
+ result.set(pathResult);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // handle the failure
+ updateFailureResult(t, result, path, ignoreNodeExists);
+ }
+ });
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ result.setException(t);
+ }
+ });
+ }
+
+ /**
+ * Updates the result future based on the given {@link Throwable}.
+ * @param t Cause of the failure
+ * @param result Future to be updated
+ * @param path Request path for the operation
+ * @return {@code true} if it is a failure, {@code false} otherwise.
+ */
+ private boolean updateFailureResult(Throwable t, SettableOperationFuture<String> result,
+ String path, boolean ignoreNodeExists) {
+ // Propagate if there is error
+ if (!(t instanceof KeeperException)) {
+ result.setException(t);
+ return true;
+ }
+ KeeperException.Code code = ((KeeperException) t).code();
+ // Node already exists, simply return success if it allows for ignoring node exists (for parent node creation).
+ if (ignoreNodeExists && code == KeeperException.Code.NODEEXISTS) {
+ // The requested path could be used because it only applies to non-sequential node
+ result.set(path);
+ return false;
+ }
+ if (code != KeeperException.Code.NONODE) {
+ result.setException(t);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Gets the parent of the given path.
+ * @param path Path for computing its parent
+ * @return Parent of the given path, or empty string if the given path is the root path already.
+ */
+ private String getParent(String path) {
+ String parentPath = path.substring(0, path.lastIndexOf('/'));
+ return (parentPath.isEmpty() && !"/".equals(path)) ? "/" : parentPath;
+ }
+ });
+
+ return result;
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path, Watcher watcher) {
+ SettableOperationFuture<Stat> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().exists(path, wrapWatcher(watcher), Callbacks.STAT_NONODE, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path, Watcher watcher) {
+ SettableOperationFuture<NodeChildren> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().getChildren(path, wrapWatcher(watcher), Callbacks.CHILDREN, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path, Watcher watcher) {
+ SettableOperationFuture<NodeData> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().getData(path, wrapWatcher(watcher), Callbacks.DATA, result);
+
+ return result;
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String dataPath, byte[] data, int version) {
+ SettableOperationFuture<Stat> result = SettableOperationFuture.create(dataPath, eventExecutor);
+ getZooKeeper().setData(dataPath, data, version, Callbacks.STAT, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<String> delete(String deletePath, int version) {
+ SettableOperationFuture<String> result = SettableOperationFuture.create(deletePath, eventExecutor);
+ getZooKeeper().delete(deletePath, version, Callbacks.VOID, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<ACLData> getACL(String path) {
+ SettableOperationFuture<ACLData> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().getACL(path, new Stat(), Callbacks.ACL, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<Stat> setACL(String path, Iterable<ACL> acl, int version) {
+ SettableOperationFuture<Stat> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().setACL(path, ImmutableList.copyOf(acl), version, Callbacks.STAT, result);
+ return result;
+ }
+
+ @Override
+ public Supplier<ZooKeeper> getZooKeeperSupplier() {
+ return new Supplier<ZooKeeper>() {
+ @Override
+ public ZooKeeper get() {
+ return getZooKeeper();
+ }
+ };
+ }
+
+ @Override
+ protected void doStart() {
+ // A single thread executor for all events
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(),
+ Threads.createDaemonThreadFactory("zk-client-EventThread"));
+ // Just discard the execution if the executor is closed
+ executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
+ eventExecutor = executor;
+
+ try {
+ zooKeeper.set(createZooKeeper());
+ } catch (IOException e) {
+ notifyFailed(e);
+ }
+ }
+
+ @Override
+ protected void doStop() {
+ // Submit a task to the executor to make sure all pending events in the executor are fired before
+ // transiting this Service into STOPPED state
+ eventExecutor.submit(stopTask);
+ eventExecutor.shutdown();
+ }
+
+ /**
+ * @return Current {@link ZooKeeper} client.
+ */
+ private ZooKeeper getZooKeeper() {
+ ZooKeeper zk = zooKeeper.get();
+ Preconditions.checkArgument(zk != null, "Not connected to zooKeeper.");
+ return zk;
+ }
+
+ /**
+ * Wraps the given watcher to be called from the event executor.
+ * @param watcher Watcher to be wrapped
+ * @return The wrapped Watcher
+ */
+ private Watcher wrapWatcher(final Watcher watcher) {
+ if (watcher == null) {
+ return null;
+ }
+ return new Watcher() {
+ @Override
+ public void process(final WatchedEvent event) {
+ if (eventExecutor.isShutdown()) {
+ LOG.debug("Already shutdown. Discarding event: {}", event);
+ return;
+ }
+ eventExecutor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ watcher.process(event);
+ } catch (Throwable t) {
+ LOG.error("Watcher throws exception.", t);
+ }
+ }
+ });
+ }
+ };
+ }
+
+ /**
+ * Creates a deep copy of the given authInfos multimap.
+ */
+ private Multimap<String, byte[]> copyAuthInfo(Multimap<String, byte[]> authInfos) {
+ Multimap<String, byte[]> result = ArrayListMultimap.create();
+
+ for (Map.Entry<String, byte[]> entry : authInfos.entries()) {
+ byte[] info = entry.getValue();
+ result.put(entry.getKey(), info == null ? null : Arrays.copyOf(info, info.length));
+ }
+
+ return result;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ State state = state();
+ if (state == State.TERMINATED || state == State.FAILED) {
+ return;
+ }
+
+ try {
+ if (event.getState() == Event.KeeperState.SyncConnected && state == State.STARTING) {
+ LOG.debug("Connected to ZooKeeper: {}", zkStr);
+ notifyStarted();
+ return;
+ }
+ if (event.getState() == Event.KeeperState.Expired) {
+ LOG.info("ZooKeeper session expired: {}", zkStr);
+
+ // When connection expired, simply reconnect again
+ if (state != State.RUNNING) {
+ return;
+ }
+ eventExecutor.submit(new Runnable() {
+ @Override
+ public void run() {
+ // Only reconnect if the current state is running
+ if (state() != State.RUNNING) {
+ return;
+ }
+ try {
+ LOG.info("Reconnect to ZooKeeper due to expiration: {}", zkStr);
+ closeZooKeeper(zooKeeper.getAndSet(createZooKeeper()));
+ } catch (IOException e) {
+ notifyFailed(e);
+ }
+ }
+ });
+ }
+ } finally {
+ if (event.getType() == Event.EventType.None) {
+ for (Watcher connectionWatcher : connectionWatchers) {
+ connectionWatcher.process(event);
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates a new ZooKeeper connection.
+ */
+ private ZooKeeper createZooKeeper() throws IOException {
+ ZooKeeper zk = new ZooKeeper(zkStr, sessionTimeout, wrapWatcher(this));
+ for (Map.Entry<String, byte[]> authInfo : authInfos.entries()) {
+ zk.addAuthInfo(authInfo.getKey(), authInfo.getValue());
+ }
+ return zk;
+ }
+
+ /**
+ * Closes the given {@link ZooKeeper} if it is not null. If there is InterruptedException,
+ * it will get logged.
+ */
+ private void closeZooKeeper(@Nullable ZooKeeper zk) {
+ try {
+ if (zk != null) {
+ zk.close();
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted when closing ZooKeeper", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Creates a {@link Runnable} task that will get executed in the event executor for transiting this
+ * Service into STOPPED state.
+ */
+ private Runnable createStopTask() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // Close the ZK connection in this task will make sure if there is ZK connection created
+ // after doStop() was called but before this task has been executed is also closed.
+ // It is possible to happen when the following sequence happens:
+ //
+ // 1. session expired, hence the expired event is triggered
+ // 2. The reconnect task executed. With Service.state() == RUNNING, it creates a new ZK client
+ // 3. Service.stop() gets called, Service.state() changed to STOPPING
+ // 4. The new ZK client created from the reconnect thread update the zooKeeper with the new one
+ closeZooKeeper(zooKeeper.getAndSet(null));
+ notifyStopped();
+ } catch (Exception e) {
+ notifyFailed(e);
+ }
+ }
+ };
+ }
+
+ /**
+ * Collection of generic callbacks that simply reflect results into OperationFuture.
+ */
+ private static final class Callbacks {
+ static final AsyncCallback.StringCallback STRING = new AsyncCallback.StringCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, String name) {
+ SettableOperationFuture<String> result = (SettableOperationFuture<String>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set((name == null || name.isEmpty()) ? path : name);
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.StatCallback STAT = new AsyncCallback.StatCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, Stat stat) {
+ SettableOperationFuture<Stat> result = (SettableOperationFuture<Stat>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(stat);
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ /**
+ * A stat callback that treats NONODE as success.
+ */
+ static final AsyncCallback.StatCallback STAT_NONODE = new AsyncCallback.StatCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, Stat stat) {
+ SettableOperationFuture<Stat> result = (SettableOperationFuture<Stat>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK || code == KeeperException.Code.NONODE) {
+ result.set(stat);
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.Children2Callback CHILDREN = new AsyncCallback.Children2Callback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ SettableOperationFuture<NodeChildren> result = (SettableOperationFuture<NodeChildren>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(new BasicNodeChildren(children, stat));
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.DataCallback DATA = new AsyncCallback.DataCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+ SettableOperationFuture<NodeData> result = (SettableOperationFuture<NodeData>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(new BasicNodeData(data, stat));
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.VoidCallback VOID = new AsyncCallback.VoidCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx) {
+ SettableOperationFuture<String> result = (SettableOperationFuture<String>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(result.getRequestPath());
+ return;
+ }
+ // Otherwise, it is an error
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.ACLCallback ACL = new AsyncCallback.ACLCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, List<ACL> acl, Stat stat) {
+ SettableOperationFuture<ACLData> result = (SettableOperationFuture<ACLData>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(new BasicACLData(acl, stat));
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/thrift/transaction.thrift
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/thrift/transaction.thrift b/tephra-core/src/main/thrift/transaction.thrift
index 39e6cec..0e3d712 100644
--- a/tephra-core/src/main/thrift/transaction.thrift
+++ b/tephra-core/src/main/thrift/transaction.thrift
@@ -16,7 +16,7 @@
# limitations under the License.
#
-namespace java co.cask.tephra.distributed.thrift
+namespace java org.apache.tephra.distributed.thrift
enum TTransactionType {
SHORT = 1,
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/ThriftTransactionSystemTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/ThriftTransactionSystemTest.java b/tephra-core/src/test/java/co/cask/tephra/ThriftTransactionSystemTest.java
deleted file mode 100644
index cf01d25..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/ThriftTransactionSystemTest.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.distributed.TransactionService;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Scopes;
-import com.google.inject.util.Modules;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.internal.zookeeper.InMemoryZKServer;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ThriftTransactionSystemTest extends TransactionSystemTest {
- private static final Logger LOG = LoggerFactory.getLogger(ThriftTransactionSystemTest.class);
-
- private static InMemoryZKServer zkServer;
- private static ZKClientService zkClientService;
- private static TransactionService txService;
- private static TransactionStateStorage storage;
- private static TransactionSystemClient txClient;
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @BeforeClass
- public static void start() throws Exception {
- zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
- zkServer.startAndWait();
-
- Configuration conf = new Configuration();
- conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
- conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
- conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
- conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
-
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- Modules.override(new TransactionModules().getDistributedModules())
- .with(new AbstractModule() {
- @Override
- protected void configure() {
- bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class).in(Scopes.SINGLETON);
- }
- }),
- new TransactionClientModule()
- );
-
- zkClientService = injector.getInstance(ZKClientService.class);
- zkClientService.startAndWait();
-
- // start a tx server
- txService = injector.getInstance(TransactionService.class);
- storage = injector.getInstance(TransactionStateStorage.class);
- txClient = injector.getInstance(TransactionSystemClient.class);
- try {
- LOG.info("Starting transaction service");
- txService.startAndWait();
- } catch (Exception e) {
- LOG.error("Failed to start service: ", e);
- }
- }
-
- @Before
- public void reset() throws Exception {
- getClient().resetState();
- }
-
- @AfterClass
- public static void stop() throws Exception {
- txService.stopAndWait();
- storage.stopAndWait();
- zkClientService.stopAndWait();
- zkServer.stopAndWait();
- }
-
- @Override
- protected TransactionSystemClient getClient() throws Exception {
- return txClient;
- }
-
- @Override
- protected TransactionStateStorage getStateStorage() throws Exception {
- return storage;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/TransactionAdminTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/TransactionAdminTest.java b/tephra-core/src/test/java/co/cask/tephra/TransactionAdminTest.java
deleted file mode 100644
index de02f80..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/TransactionAdminTest.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.distributed.TransactionService;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Scopes;
-import com.google.inject.util.Modules;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.internal.zookeeper.InMemoryZKServer;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.TimeUnit;
-
-public class TransactionAdminTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAdminTest.class);
-
- private static Configuration conf;
- private static InMemoryZKServer zkServer;
- private static ZKClientService zkClientService;
- private static TransactionService txService;
- private static TransactionSystemClient txClient;
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @BeforeClass
- public static void start() throws Exception {
- zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
- zkServer.startAndWait();
-
- conf = new Configuration();
- conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
- conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
- conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
- conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
-
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- Modules.override(new TransactionModules().getDistributedModules())
- .with(new AbstractModule() {
- @Override
- protected void configure() {
- bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class).in(Scopes.SINGLETON);
- }
- }),
- new TransactionClientModule()
- );
-
- zkClientService = injector.getInstance(ZKClientService.class);
- zkClientService.startAndWait();
-
- // start a tx server
- txService = injector.getInstance(TransactionService.class);
- txClient = injector.getInstance(TransactionSystemClient.class);
- try {
- LOG.info("Starting transaction service");
- txService.startAndWait();
- } catch (Exception e) {
- LOG.error("Failed to start service: ", e);
- }
- }
-
- @Before
- public void reset() throws Exception {
- txClient.resetState();
- }
-
- @AfterClass
- public static void stop() throws Exception {
- txService.stopAndWait();
- zkClientService.stopAndWait();
- zkServer.stopAndWait();
- }
-
- @Test
- public void testPrintUsage() throws Exception {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- ByteArrayOutputStream err = new ByteArrayOutputStream();
- TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(out), new PrintStream(err));
- int status = txAdmin.doMain(new String[0], conf);
- Assert.assertEquals(1, status);
- //noinspection ConstantConditions
- Assert.assertTrue(err.toString("UTF-8").startsWith("Usage:"));
- Assert.assertEquals(0, out.toByteArray().length);
- }
-
- @Test
- public void testTruncateInvalidTx() throws Exception {
- Transaction tx1 = txClient.startLong();
- Transaction tx2 = txClient.startShort();
- txClient.invalidate(tx1.getTransactionId());
- txClient.invalidate(tx2.getTransactionId());
- Assert.assertEquals(2, txClient.getInvalidSize());
-
- TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(System.out), new PrintStream(System.err));
- int status = txAdmin.doMain(new String[]{"--truncate-invalid-tx", String.valueOf(tx2.getTransactionId())}, conf);
- Assert.assertEquals(0, status);
- Assert.assertEquals(1, txClient.getInvalidSize());
- }
-
- @Test
- public void testTruncateInvalidTxBefore() throws Exception {
- Transaction tx1 = txClient.startLong();
- TimeUnit.MILLISECONDS.sleep(1);
- long beforeTx2 = System.currentTimeMillis();
- Transaction tx2 = txClient.startLong();
-
- // Try before invalidation
- Assert.assertEquals(0, txClient.getInvalidSize());
- TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(System.out), new PrintStream(System.err));
- int status = txAdmin.doMain(new String[]{"--truncate-invalid-tx-before", String.valueOf(beforeTx2)}, conf);
- // Assert command failed due to in-progress transactions
- Assert.assertEquals(1, status);
- // Assert no change to invalid size
- Assert.assertEquals(0, txClient.getInvalidSize());
-
- txClient.invalidate(tx1.getTransactionId());
- txClient.invalidate(tx2.getTransactionId());
- Assert.assertEquals(2, txClient.getInvalidSize());
-
- status = txAdmin.doMain(new String[]{"--truncate-invalid-tx-before", String.valueOf(beforeTx2)}, conf);
- Assert.assertEquals(0, status);
- Assert.assertEquals(1, txClient.getInvalidSize());
- }
-
- @Test
- public void testGetInvalidTxSize() throws Exception {
- Transaction tx1 = txClient.startShort();
- txClient.startLong();
- txClient.invalidate(tx1.getTransactionId());
-
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- ByteArrayOutputStream err = new ByteArrayOutputStream();
- TransactionAdmin txAdmin = new TransactionAdmin(new PrintStream(out), new PrintStream(err));
- int status = txAdmin.doMain(new String[]{"--get-invalid-tx-size"}, conf);
- Assert.assertEquals(0, status);
- //noinspection ConstantConditions
- Assert.assertTrue(out.toString("UTF-8").contains("Invalid list size: 1\n"));
- }
-}
[38/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCache.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCache.java b/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCache.java
new file mode 100644
index 0000000..f1d62b8
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCache.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.coprocessor;
+
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.HDFSTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.util.ConfigurationFactory;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Periodically refreshes transaction state from the latest stored snapshot. This is implemented as a singleton
+ * to allow a single cache to be shared by all regions on a regionserver.
+ */
+public class TransactionStateCache extends AbstractIdleService implements Configurable {
+ private static final Log LOG = LogFactory.getLog(TransactionStateCache.class);
+
+ // how frequently we should wake to check for changes (in seconds)
+ private static final long CHECK_FREQUENCY = 15;
+
+ private Configuration hConf;
+
+ private TransactionStateStorage storage;
+ private volatile TransactionVisibilityState latestState;
+
+ private Thread refreshService;
+ private long lastRefresh;
+ // snapshot refresh frequency in milliseconds
+ private long snapshotRefreshFrequency;
+ private boolean initialized;
+
+ public TransactionStateCache() {
+ }
+
+ @Override
+ public Configuration getConf() {
+ return hConf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.hConf = conf;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ refreshState();
+ startRefreshService();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ this.refreshService.interrupt();
+ this.storage.stop();
+ }
+
+ /**
+ * Try to initialize the Configuration and TransactionStateStorage instances. Obtaining the Configuration may
+ * fail until ReactorServiceMain has been started.
+ */
+ private void tryInit() {
+ try {
+ Configuration conf = getSnapshotConfiguration();
+ if (conf != null) {
+ // Since this is only used for background loading of transaction snapshots, we use the no-op metrics collector,
+ // as there are no relevant metrics to report
+ this.storage = new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf),
+ new TxMetricsCollector());
+ this.storage.startAndWait();
+ this.snapshotRefreshFrequency = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
+ TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL) * 1000;
+ this.initialized = true;
+ } else {
+ LOG.info("Could not load configuration");
+ }
+ } catch (Exception e) {
+ LOG.info("Failed to initialize TransactionStateCache due to: " + e.getMessage());
+ }
+ }
+
+ protected Configuration getSnapshotConfiguration() throws IOException {
+ Configuration conf = new ConfigurationFactory().get(hConf);
+ conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ return conf;
+ }
+
+ private void reset() {
+ this.storage.stop();
+ this.lastRefresh = 0;
+ this.initialized = false;
+ }
+
+ private void startRefreshService() {
+ this.refreshService = new Thread("tx-state-refresh") {
+ @Override
+ public void run() {
+ while (!isInterrupted()) {
+ if (latestState == null || System.currentTimeMillis() > (lastRefresh + snapshotRefreshFrequency)) {
+ try {
+ refreshState();
+ } catch (IOException ioe) {
+ LOG.info("Error refreshing transaction state cache: " + ioe.getMessage());
+ }
+ }
+ try {
+ TimeUnit.SECONDS.sleep(CHECK_FREQUENCY);
+ } catch (InterruptedException ie) {
+ // reset status
+ interrupt();
+ break;
+ }
+ }
+ LOG.info("Exiting thread " + getName());
+ }
+ };
+ this.refreshService.setDaemon(true);
+ this.refreshService.start();
+ }
+
+ private void refreshState() throws IOException {
+ if (!initialized) {
+ tryInit();
+ }
+
+ // only continue if initialization was successful
+ if (initialized) {
+ long now = System.currentTimeMillis();
+ TransactionVisibilityState currentState = storage.getLatestTransactionVisibilityState();
+ if (currentState != null) {
+ if (currentState.getTimestamp() < (now - 2 * snapshotRefreshFrequency)) {
+ LOG.info("Current snapshot is old, will force a refresh on next run.");
+ reset();
+ } else {
+ latestState = currentState;
+ LOG.info("Transaction state reloaded with snapshot from " + latestState.getTimestamp());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Latest transaction snapshot: " + latestState.toString());
+ }
+ lastRefresh = now;
+ }
+ } else {
+ LOG.info("No transaction state found.");
+ }
+ }
+ }
+
+ public TransactionVisibilityState getLatestState() {
+ return latestState;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCacheSupplier.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCacheSupplier.java b/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCacheSupplier.java
new file mode 100644
index 0000000..d19da36
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/coprocessor/TransactionStateCacheSupplier.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.coprocessor;
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Supplies instances of {@link TransactionStateCache} implementations.
+ */
+public class TransactionStateCacheSupplier implements Supplier<TransactionStateCache> {
+ protected static volatile TransactionStateCache instance;
+ protected static Object lock = new Object();
+
+ protected final Configuration conf;
+
+ public TransactionStateCacheSupplier(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Returns a singleton instance of the transaction state cache, performing lazy initialization if necessary.
+ * @return A shared instance of the transaction state cache.
+ */
+ @Override
+ public TransactionStateCache get() {
+ if (instance == null) {
+ synchronized (lock) {
+ if (instance == null) {
+ instance = new TransactionStateCache();
+ instance.setConf(conf);
+ instance.start();
+ }
+ }
+ }
+ return instance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/coprocessor/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/coprocessor/package-info.java b/tephra-core/src/main/java/org/apache/tephra/coprocessor/package-info.java
new file mode 100644
index 0000000..c554e37
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/coprocessor/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains HBase coprocessor implementations for the transaction system.
+ */
+package org.apache.tephra.coprocessor;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/AbstractClientProvider.java b/tephra-core/src/main/java/org/apache/tephra/distributed/AbstractClientProvider.java
new file mode 100644
index 0000000..3abdafa
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/AbstractClientProvider.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TFramedTransport;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.twill.discovery.Discoverable;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * An abstract tx client provider that implements common functionality.
+ */
+public abstract class AbstractClientProvider implements ThriftClientProvider {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractClientProvider.class);
+
+ // Discovery service. If null, no service discovery.
+ private final DiscoveryServiceClient discoveryServiceClient;
+ protected final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ // the configuration
+ final Configuration configuration;
+
+ // the endpoint strategy for service discovery.
+ EndpointStrategy endpointStrategy;
+
+ protected AbstractClientProvider(Configuration configuration, DiscoveryServiceClient discoveryServiceClient) {
+ this.configuration = configuration;
+ this.discoveryServiceClient = discoveryServiceClient;
+ }
+
+ public void initialize() throws TException {
+ if (initialized.compareAndSet(false, true)) {
+ this.initDiscovery();
+ }
+ }
+
+ /**
+ * Initialize the service discovery client, we will reuse that
+ * every time we need to create a new client.
+ */
+ private void initDiscovery() {
+ if (discoveryServiceClient == null) {
+ LOG.info("No DiscoveryServiceClient provided. Skipping service discovery.");
+ return;
+ }
+
+ endpointStrategy = new TimeLimitEndpointStrategy(
+ new RandomEndpointStrategy(
+ discoveryServiceClient.discover(
+ configuration.get(TxConstants.Service.CFG_DATA_TX_DISCOVERY_SERVICE_NAME,
+ TxConstants.Service.DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME))),
+ 2, TimeUnit.SECONDS);
+ }
+
+ protected TransactionServiceThriftClient newClient() throws TException {
+ return newClient(-1);
+ }
+
+ protected TransactionServiceThriftClient newClient(int timeout) throws TException {
+ initialize();
+ String address;
+ int port;
+
+ if (endpointStrategy == null) {
+ // if there is no discovery service, try to read host and port directly
+ // from the configuration
+ LOG.info("Reading address and port from configuration.");
+ address = configuration.get(TxConstants.Service.CFG_DATA_TX_BIND_ADDRESS,
+ TxConstants.Service.DEFAULT_DATA_TX_BIND_ADDRESS);
+ port = configuration.getInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT,
+ TxConstants.Service.DEFAULT_DATA_TX_BIND_PORT);
+ LOG.info("Service assumed at " + address + ":" + port);
+ } else {
+ Discoverable endpoint = endpointStrategy.pick();
+ if (endpoint == null) {
+ LOG.error("Unable to discover tx service.");
+ throw new TException("Unable to discover tx service.");
+ }
+ address = endpoint.getSocketAddress().getHostName();
+ port = endpoint.getSocketAddress().getPort();
+ LOG.info("Service discovered at " + address + ":" + port);
+ }
+
+ // now we have an address and port, try to connect a client
+ if (timeout < 0) {
+ timeout = configuration.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_TIMEOUT,
+ TxConstants.Service.DEFAULT_DATA_TX_CLIENT_TIMEOUT_MS);
+ }
+ LOG.info("Attempting to connect to tx service at " +
+ address + ":" + port + " with timeout " + timeout + " ms.");
+ // thrift transport layer
+ TTransport transport =
+ new TFramedTransport(new TSocket(address, port, timeout));
+ try {
+ transport.open();
+ } catch (TTransportException e) {
+ LOG.error("Unable to connect to tx service: " + e.getMessage());
+ throw e;
+ }
+ // and create a thrift client
+ TransactionServiceThriftClient newClient = new TransactionServiceThriftClient(transport);
+
+ LOG.info("Connected to tx service at " +
+ address + ":" + port);
+ return newClient;
+ }
+
+ /**
+ * This class helps picking up an endpoint from a list of Discoverable.
+ */
+ public interface EndpointStrategy {
+
+ /**
+ * Picks a {@link Discoverable} using its strategy.
+ * @return A {@link Discoverable} based on the stragegy or {@code null} if no endpoint can be found.
+ */
+ Discoverable pick();
+ }
+
+ /**
+ * An {@link EndpointStrategy} that make sure it picks an endpoint within the given
+ * timeout limit.
+ */
+ public final class TimeLimitEndpointStrategy implements EndpointStrategy {
+
+ private final EndpointStrategy delegate;
+ private final long timeout;
+ private final TimeUnit timeoutUnit;
+
+ public TimeLimitEndpointStrategy(EndpointStrategy delegate, long timeout, TimeUnit timeoutUnit) {
+ this.delegate = delegate;
+ this.timeout = timeout;
+ this.timeoutUnit = timeoutUnit;
+ }
+
+ @Override
+ public Discoverable pick() {
+ Discoverable pick = delegate.pick();
+ try {
+ long count = 0;
+ while (pick == null && count++ < timeout) {
+ timeoutUnit.sleep(1);
+ pick = delegate.pick();
+ }
+ } catch (InterruptedException e) {
+ // Simply propagate the interrupt.
+ Thread.currentThread().interrupt();
+ }
+ return pick;
+ }
+ }
+
+ /**
+ * Randomly picks endpoint from the list of available endpoints.
+ */
+ public final class RandomEndpointStrategy implements EndpointStrategy {
+
+ private final Iterable<Discoverable> endpoints;
+
+ /**
+ * Constructs a random endpoint strategy.
+ * @param endpoints Endpoints for the strategy to use. Note that this strategy will
+ * invoke {@link Iterable#iterator()} and traverse through it on
+ * every call to the {@link #pick()} method. One could leverage this
+ * behavior with the live {@link Iterable} as provided by
+ * {@link org.apache.twill.discovery.DiscoveryServiceClient#discover(String)} method.
+ */
+ public RandomEndpointStrategy(Iterable<Discoverable> endpoints) {
+ this.endpoints = endpoints;
+ }
+
+ @Override
+ public Discoverable pick() {
+ // Reservoir sampling
+ Discoverable result = null;
+ Iterator<Discoverable> itor = endpoints.iterator();
+ Random random = new Random();
+ int count = 0;
+ while (itor.hasNext()) {
+ Discoverable next = itor.next();
+ if (random.nextInt(++count) == 0) {
+ result = next;
+ }
+ }
+ return result;
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/CloseableThriftClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/CloseableThriftClient.java b/tephra-core/src/main/java/org/apache/tephra/distributed/CloseableThriftClient.java
new file mode 100644
index 0000000..ee73ef0
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/CloseableThriftClient.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+/**
+ * An {@link AutoCloseable} to automatically return the thrift client to the ThriftClientProvider.
+ */
+public class CloseableThriftClient implements AutoCloseable {
+
+ private final ThriftClientProvider provider;
+ private final TransactionServiceThriftClient thriftClient;
+
+ public CloseableThriftClient(ThriftClientProvider provider, TransactionServiceThriftClient thriftClient) {
+ this.provider = provider;
+ this.thriftClient = thriftClient;
+ }
+
+ public TransactionServiceThriftClient getThriftClient() {
+ return thriftClient;
+ }
+
+ @Override
+ public void close() {
+ // in any case, the client must be returned to the pool. The pool is
+ // responsible for discarding the client if it is in a bad state.
+ provider.returnClient(thriftClient);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/ElasticPool.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/ElasticPool.java b/tephra-core/src/main/java/org/apache/tephra/distributed/ElasticPool.java
new file mode 100644
index 0000000..8ba3218
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/ElasticPool.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * An Elastic Pool is an object pool that can dynamically grow.
+ * Normally, an element is obtained by a client and then returned to the pool
+ * after use. However, if the element gets into a bad state, the element can
+ * be discarded, based upon the recycle() method returning false. This will
+ * cause the element to be removed from the pool, and for a subsequent request,
+ * a new element can be created on the fly to replace the discarded one.
+ *
+ * The pool starts with zero (active) elements. Every time a client attempts
+ * to obtain an element, an element from the pool is returned if available.
+ * Otherwise, if the number of active elements is less than the pool's limit,
+ * a new element is created (using abstract method create(), this must be
+ * overridden by all implementations), and the number of active elements is
+ * increased by one. If the limit is reached, then obtain() blocks until
+ * either an element is returned to the pool or, if the obtain method with timeout
+ * parameters is used, a timeout occurs.
+ *
+ * Every time an element is returned to the pool, it is "recycled" to restore its
+ * fresh state for the next use or destroyed, depending on its state.
+ *
+ * @param <T> the type of the elements
+ * @param <E> the type of exception thrown by create()
+ */
+public abstract class ElasticPool<T, E extends Exception> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ElasticPool.class);
+
+ /**
+ * A method to create a new element. Will be called every time the pool
+ * of available elements is empty but the limit of active elements is
+ * not exceeded.
+ * @return a new element
+ */
+ protected abstract T create() throws E;
+
+ /**
+ * A method to recycle an existing element when it is returned to the pool.
+ * This methods ensures that the element is in a fresh state before it can
+ * be reused by the next agent. If the element is not to be returned to the pool,
+ * calling code is responsible for destroying it and returning false.
+ *
+ * @param element the element to recycle
+ * @return true to reuse element, false to remove it from the pool
+ */
+ protected boolean recycle(T element) {
+ // by default, simply return true
+ return true;
+ }
+
+ // holds all currently available elements
+ private final ConcurrentLinkedQueue<T> elements;
+
+ // we keep track of elements via the permits of a semaphore, because there can
+ // be elements in a queue, but also elements that are "loaned out" count towards
+ // the pool's size limit
+ private final Semaphore semaphore;
+
+ public ElasticPool(int sizeLimit) {
+ elements = new ConcurrentLinkedQueue<>();
+ semaphore = new Semaphore(sizeLimit, true);
+ }
+
+ /**
+ * Get a element from the pool. If there is an available element in
+ * the pool, it will be returned. Otherwise, if the number of active
+ * elements does not exceed the limit, a new element is created with
+ * create() and returned. Otherwise, blocks until an element is either
+ * released and returned to the pool, or an element is discarded,
+ * allowing for the creation of a new element.
+ *
+ * @return an element
+ */
+ public T obtain() throws E, InterruptedException {
+ semaphore.acquire();
+ return getOrCreate();
+ }
+
+ /**
+ * Get a element from the pool. If there is an available element in
+ * the pool, it will be returned. Otherwise, if the number of active
+ * elements does ot exceed the limit, a new element is created with
+ * create() and returned. Otherwise, blocks until an element is either
+ * released and returned to the pool, an element is discarded,
+ * allowing for the creation of a new element, or a timeout occurs.
+ *
+ * @param timeout the timeout for trying to obtain an element
+ * @param unit the timeout unit for trying to obtain an element
+ * @return an element
+ * @throws TimeoutException if a client is not able to be obtained within the given timeout
+ */
+ public T obtain(long timeout, TimeUnit unit) throws E, TimeoutException, InterruptedException {
+ if (!semaphore.tryAcquire(1, timeout, unit)) {
+ throw new TimeoutException(String.format("Failed to obtain client within %d %s.",
+ timeout, unit));
+ }
+ return getOrCreate();
+ }
+
+ // gets an element from the queue, or creates one if there is none available in the queue.
+ // the semaphore must be acquired before calling this method. The semaphore will be released from within
+ // this method if it throws any exception
+ private T getOrCreate() throws E {
+ try {
+ T client = elements.poll();
+ // a client was available, all good. otherwise, create one
+ if (client != null) {
+ return client;
+ }
+ return create();
+ } catch (Exception e) {
+ // if an exception is thrown after acquiring the semaphore, release the
+ // semaphore before propagating the exception
+ semaphore.release();
+ throw e;
+ }
+ }
+
+ /**
+ * Returns an element to the pool of available elements. The element must
+ * have been obtained from this pool through obtain(). The recycle() method
+ * is called before the element is available for obtain().
+ *
+ * @param element the element to be returned
+ */
+ public void release(T element) {
+ try {
+ if (recycle(element)) {
+ elements.add(element);
+ }
+ } finally {
+ semaphore.release();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/PooledClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/PooledClientProvider.java b/tephra-core/src/main/java/org/apache/tephra/distributed/PooledClientProvider.java
new file mode 100644
index 0000000..9df1441
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/PooledClientProvider.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.thrift.TException;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * This is an tx client provider that uses a bounded size pool of connections.
+ */
+public class PooledClientProvider extends AbstractClientProvider {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PooledClientProvider.class);
+
+ // we will use this as a pool of tx clients
+ class TxClientPool extends ElasticPool<TransactionServiceThriftClient, TException> {
+ TxClientPool(int sizeLimit) {
+ super(sizeLimit);
+ }
+
+ @Override
+ protected TransactionServiceThriftClient create() throws TException {
+ return newClient();
+ }
+
+ @Override
+ protected boolean recycle(TransactionServiceThriftClient client) {
+ if (!client.isValid()) {
+ client.close();
+ return false;
+ }
+ return true;
+ }
+ }
+
+ // we will use this as a pool of tx clients
+ private volatile TxClientPool clients;
+
+ // the limit for the number of active clients
+ private int maxClients;
+ // timeout, for obtaining a client
+ private long obtainClientTimeoutMs;
+
+ public PooledClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient) {
+ super(conf, discoveryServiceClient);
+ }
+
+ private void initializePool() throws TException {
+ // initialize the super class (needed for service discovery)
+ super.initialize();
+
+ // create a (empty) pool of tx clients
+ maxClients = configuration.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT,
+ TxConstants.Service.DEFAULT_DATA_TX_CLIENT_COUNT);
+ if (maxClients < 1) {
+ LOG.warn("Configuration of " + TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT +
+ " is invalid: value is " + maxClients + " but must be at least 1. " +
+ "Using 1 as a fallback. ");
+ maxClients = 1;
+ }
+
+ obtainClientTimeoutMs =
+ configuration.getLong(TxConstants.Service.CFG_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS,
+ TxConstants.Service.DEFAULT_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS);
+ if (obtainClientTimeoutMs < 0) {
+ LOG.warn("Configuration of " + TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT +
+ " is invalid: value is " + obtainClientTimeoutMs + " but must be at least 0. " +
+ "Using 0 as a fallback. ");
+ obtainClientTimeoutMs = 0;
+ }
+ this.clients = new TxClientPool(maxClients);
+ }
+
+ @Override
+ public CloseableThriftClient getCloseableClient() throws TException, TimeoutException, InterruptedException {
+ TransactionServiceThriftClient client = getClientPool().obtain(obtainClientTimeoutMs, TimeUnit.MILLISECONDS);
+ return new CloseableThriftClient(this, client);
+ }
+
+ @Override
+ public void returnClient(TransactionServiceThriftClient client) {
+ getClientPool().release(client);
+ }
+
+ @Override
+ public String toString() {
+ return "Elastic pool of size " + this.maxClients +
+ ", with timeout (in milliseconds): " + this.obtainClientTimeoutMs;
+ }
+
+ private TxClientPool getClientPool() {
+ if (clients != null) {
+ return clients;
+ }
+
+ synchronized (this) {
+ if (clients == null) {
+ try {
+ initializePool();
+ } catch (TException e) {
+ LOG.error("Failed to initialize Tx client provider", e);
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+ return clients;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/RetryNTimes.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/RetryNTimes.java b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryNTimes.java
new file mode 100644
index 0000000..8655c44
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryNTimes.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+
+/**
+ * A retry strategy that makes N attempts and then gives up. This does
+ * not do anything before the re-attempt - extend this class to add a
+ * sleep or similar.
+ */
+public class RetryNTimes extends RetryStrategy {
+
+ int attempts = 0;
+ int limit;
+
+ /**
+ * @param maxAttempts the number of attempts after which to stop
+ */
+ protected RetryNTimes(int maxAttempts) {
+ limit = maxAttempts;
+ }
+
+ @Override
+ boolean failOnce() {
+ ++attempts;
+ return attempts < limit;
+ }
+
+ /**
+ * A retry strategy provider for this strategy.
+ */
+ public static class Provider implements RetryStrategyProvider {
+
+ int nTimes;
+
+ public Provider() {
+ this.nTimes = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_ATTEMPTS;
+ }
+
+ @Override
+ public void configure(Configuration config) {
+ nTimes = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, nTimes);
+ }
+
+ @Override
+ public RetryStrategy newRetryStrategy() {
+ return new RetryNTimes(nTimes);
+ }
+
+ @Override
+ public String toString() {
+ return nTimes + " attempts without delay";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategy.java b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategy.java
new file mode 100644
index 0000000..6fd6c9f
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategy.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+/**
+ * A retry strategy is an abstraction over how the remote tx client shuold retry operations after connection
+ * failures.
+ */
+public abstract class RetryStrategy {
+
+ /**
+ * Increments the number of failed attempts.
+ * @return whether another attempt should be made
+ */
+ abstract boolean failOnce();
+
+ /**
+ * Should be called before re-attempting. This can, for instance
+ * inject a sleep time between retries. Default implementation is
+ * to do nothing.
+ */
+ void beforeRetry() {
+ // do nothinhg
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategyProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategyProvider.java b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategyProvider.java
new file mode 100644
index 0000000..7763ea5
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryStrategyProvider.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A retry strategy provider is used by the tx client to get a new retry strategy for every call.
+ */
+public interface RetryStrategyProvider {
+
+ /**
+ * Provides a new instance of a retry strategy.
+ * @return a retry strategy
+ */
+ RetryStrategy newRetryStrategy();
+
+ /**
+ * Configure the strategy.
+ * @param config the configuration
+ */
+ void configure(Configuration config);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/RetryWithBackoff.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/RetryWithBackoff.java b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryWithBackoff.java
new file mode 100644
index 0000000..ce1130e
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/RetryWithBackoff.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A retry strategy that makes N attempts and then gives up. This does
+ * not do anything before the re-attempt - extend this class to add a
+ * sleep or similar.
+ */
+public class RetryWithBackoff extends RetryStrategy {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RetryWithBackoff.class);
+
+ int initialSleep; // initial sleep time
+ int backoffFactor; // factor by which to increase sleep for each retry
+ int maxSleep; // max sleep time. stop retrying when we exceed this
+ int sleep; // current sleep time
+
+ /**
+ * @param initial the initial sleep time (before first retry)
+ * @param backoff the backoff factor by which sleep time is multiplied
+ * after each retry
+ * @param limit the max sleep time. if sleep time reaches this limit, we
+ * stop retrying
+ */
+ protected RetryWithBackoff(int initial, int backoff, int limit) {
+ initialSleep = initial;
+ backoffFactor = backoff;
+ maxSleep = limit;
+ sleep = initialSleep;
+ }
+
+ @Override
+ boolean failOnce() {
+ return sleep < maxSleep;
+ }
+
+ @Override
+ void beforeRetry() {
+ LOG.info("Sleeping " + sleep + " ms before retry.");
+ long current = System.currentTimeMillis();
+ long end = current + sleep;
+ while (current < end) {
+ try {
+ Thread.sleep(end - current);
+ } catch (InterruptedException e) {
+ // do nothing
+ }
+ current = System.currentTimeMillis();
+ }
+ sleep = sleep * backoffFactor;
+ }
+
+ /**
+ * A provider for this retry strategy.
+ */
+ public static class Provider implements RetryStrategyProvider {
+
+ int initialSleep; // initial sleep time
+ int backoffFactor; // factor by which to increase sleep for each retry
+ int maxSleep; // max sleep time. stop retrying when we exceed this
+
+ public Provider() {
+ initialSleep = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_BACKOFF_INITIAL;
+ backoffFactor = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_BACKOFF_FACTOR;
+ maxSleep = TxConstants.Service.DEFAULT_DATA_TX_CLIENT_BACKOFF_LIMIT;
+ }
+
+ public void configure(Configuration config) {
+ initialSleep = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_BACKOFF_INITIAL, initialSleep);
+ backoffFactor = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_BACKOFF_FACTOR, backoffFactor);
+ maxSleep = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_BACKOFF_LIMIT, maxSleep);
+ }
+
+ @Override
+ public RetryStrategy newRetryStrategy() {
+ return new RetryWithBackoff(initialSleep, backoffFactor, maxSleep);
+ }
+
+ @Override
+ public String toString() {
+ return "sleep " + initialSleep + " ms with back off factor " +
+ backoffFactor + " and limit " + maxSleep + " ms";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/SingleUseClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/SingleUseClientProvider.java b/tephra-core/src/main/java/org/apache/tephra/distributed/SingleUseClientProvider.java
new file mode 100644
index 0000000..d55da5e
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/SingleUseClientProvider.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.thrift.TException;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeoutException;
+
+/**
+ * An tx client provider that creates a new connection every time.
+ */
+public class SingleUseClientProvider extends AbstractClientProvider {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SingleUseClientProvider.class);
+
+ public SingleUseClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient, int timeout) {
+ super(conf, discoveryServiceClient);
+ this.timeout = timeout;
+ }
+
+ final int timeout;
+
+ @Override
+ public CloseableThriftClient getCloseableClient() throws TException, TimeoutException, InterruptedException {
+ try {
+ return new CloseableThriftClient(this, newClient(timeout));
+ } catch (TException e) {
+ LOG.error("Unable to create new tx client: " + e.getMessage());
+ throw e;
+ }
+ }
+
+ @Override
+ public void returnClient(TransactionServiceThriftClient client) {
+ client.close();
+ }
+
+ @Override
+ public String toString() {
+ return "Single-use(timeout = " + timeout + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/ThreadLocalClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/ThreadLocalClientProvider.java b/tephra-core/src/main/java/org/apache/tephra/distributed/ThreadLocalClientProvider.java
new file mode 100644
index 0000000..dedaffe
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/ThreadLocalClientProvider.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.thrift.TException;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeoutException;
+
+/**
+ * An tx client provider that uses thread local to maintain at most one open connection per thread.
+ * Note that there can be a connection leak if the threads are recycled.
+ */
+public class ThreadLocalClientProvider extends AbstractClientProvider {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ThreadLocalClientProvider.class);
+
+ ThreadLocal<TransactionServiceThriftClient> clients = new ThreadLocal<>();
+
+ public ThreadLocalClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient) {
+ super(conf, discoveryServiceClient);
+ }
+
+ @Override
+ public CloseableThriftClient getCloseableClient() throws TException, TimeoutException, InterruptedException {
+ TransactionServiceThriftClient client = this.clients.get();
+ if (client == null) {
+ try {
+ client = this.newClient();
+ clients.set(client);
+ } catch (TException e) {
+ LOG.error("Unable to create new tx client for thread: "
+ + e.getMessage());
+ throw e;
+ }
+ }
+ return new CloseableThriftClient(this, client);
+ }
+
+ @Override
+ public void returnClient(TransactionServiceThriftClient client) {
+ if (!client.isValid()) {
+ client.close();
+ clients.remove();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Thread-local";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/ThriftClientProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/ThriftClientProvider.java b/tephra-core/src/main/java/org/apache/tephra/distributed/ThriftClientProvider.java
new file mode 100644
index 0000000..6a41912
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/ThriftClientProvider.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import org.apache.thrift.TException;
+
+import java.util.concurrent.TimeoutException;
+
+/**
+ * This interface is used to provide thrift tx service clients:
+ * there is only one (singleton)
+ * tx service per JVM, but many threads may use it concurrently.
+ * However, being a thrift client, it is not thread-safe. In
+ * order to avoid serializing all tx calls by synchronizing
+ * on the tx service client, we employ a pool of clients. But
+ * in different scenarios there are different strategies for
+ * pooling: If there are many short-lived threads, it is wise
+ * to have a shared pool between all threads. But if there are
+ * few long-lived threads, it may be better to have thread-local
+ * client for each thread.
+ *
+ * This interface provides an abstraction of the pooling strategy.
+ */
+public interface ThriftClientProvider {
+
+ /**
+ * Initialize the provider. At this point, it should be verified
+ * that tx service is up and running and getClient() can
+ * create new clients when necessary.
+ */
+ void initialize() throws TException;
+
+ /**
+ * Retrieve an AutoCloseable wrapper around tx client for exclusive use by the
+ * current thread. The client must be closed (returned) to the provider after use.
+ * @return an tx client, connected and fully functional
+ */
+ CloseableThriftClient getCloseableClient() throws TException,
+ TimeoutException, InterruptedException;
+
+ /**
+ * Release an tx client back to the provider's pool, if the client is valid.
+ * If the client becomes disfunctional, for instance, due to a socket
+ * exception. The provider must make sure to close the client, and it
+ * must remove the client from its arsenal and be prepared to create
+ * a new client subsequently.
+ *
+ * @param client The client to return
+ */
+ void returnClient(TransactionServiceThriftClient client);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionConverterUtils.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionConverterUtils.java b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionConverterUtils.java
new file mode 100644
index 0000000..66e463d
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionConverterUtils.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.primitives.Longs;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.distributed.thrift.TTransaction;
+import org.apache.tephra.distributed.thrift.TTransactionType;
+import org.apache.tephra.distributed.thrift.TVisibilityLevel;
+
+/**
+ * Utility methods to convert to thrift and back.
+ */
+public final class TransactionConverterUtils {
+ private static final long[] EMPTY_LONG_ARRAY = {};
+
+ public static TTransaction wrap(Transaction tx) {
+ return new TTransaction(tx.getTransactionId(), tx.getReadPointer(),
+ Longs.asList(tx.getInvalids()), Longs.asList(tx.getInProgress()),
+ tx.getFirstShortInProgress(), getTTransactionType(tx.getType()),
+ tx.getWritePointer(), Longs.asList(tx.getCheckpointWritePointers()),
+ getTVisibilityLevel(tx.getVisibilityLevel()));
+ }
+
+ public static Transaction unwrap(TTransaction thriftTx) {
+ return new Transaction(thriftTx.getReadPointer(), thriftTx.getTransactionId(), thriftTx.getWritePointer(),
+ thriftTx.getInvalids() == null ? EMPTY_LONG_ARRAY : Longs.toArray(thriftTx.getInvalids()),
+ thriftTx.getInProgress() == null ? EMPTY_LONG_ARRAY :
+ Longs.toArray(thriftTx.getInProgress()),
+ thriftTx.getFirstShort(), getTransactionType(thriftTx.getType()),
+ thriftTx.getCheckpointWritePointers() == null ? EMPTY_LONG_ARRAY :
+ Longs.toArray(thriftTx.getCheckpointWritePointers()),
+ getVisibilityLevel(thriftTx.getVisibilityLevel()));
+ }
+
+ private static TransactionType getTransactionType(TTransactionType tType) {
+ return tType == TTransactionType.SHORT ? TransactionType.SHORT : TransactionType.LONG;
+ }
+
+ private static TTransactionType getTTransactionType(TransactionType type) {
+ return type == TransactionType.SHORT ? TTransactionType.SHORT : TTransactionType.LONG;
+ }
+
+ private static Transaction.VisibilityLevel getVisibilityLevel(TVisibilityLevel tLevel) {
+ // default to SNAPSHOT
+ if (tLevel == null) {
+ return Transaction.VisibilityLevel.SNAPSHOT;
+ }
+
+ switch (tLevel) {
+ case SNAPSHOT:
+ return Transaction.VisibilityLevel.SNAPSHOT;
+ case SNAPSHOT_EXCLUDE_CURRENT:
+ return Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT;
+ case SNAPSHOT_ALL:
+ return Transaction.VisibilityLevel.SNAPSHOT_ALL;
+ default:
+ throw new IllegalArgumentException("Unknown TVisibilityLevel: " + tLevel);
+ }
+ }
+
+ private static TVisibilityLevel getTVisibilityLevel(Transaction.VisibilityLevel level) {
+ switch (level) {
+ case SNAPSHOT:
+ return TVisibilityLevel.SNAPSHOT;
+ case SNAPSHOT_EXCLUDE_CURRENT:
+ return TVisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT;
+ case SNAPSHOT_ALL:
+ return TVisibilityLevel.SNAPSHOT_ALL;
+ default:
+ throw new IllegalArgumentException("Unknown VisibilityLevel: " + level);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionService.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionService.java b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionService.java
new file mode 100644
index 0000000..6fbf926
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionService.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.distributed.thrift.TTransactionServer;
+import org.apache.tephra.inmemory.InMemoryTransactionService;
+import org.apache.tephra.rpc.ThriftRPCServer;
+import org.apache.twill.api.ElectionHandler;
+import org.apache.twill.discovery.DiscoveryService;
+import org.apache.twill.internal.ServiceListenerAdapter;
+import org.apache.twill.internal.zookeeper.LeaderElection;
+import org.apache.twill.zookeeper.ZKClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ *
+ */
+public final class TransactionService extends InMemoryTransactionService {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionService.class);
+ private LeaderElection leaderElection;
+ private final ZKClient zkClient;
+
+ private ThriftRPCServer<TransactionServiceThriftHandler, TTransactionServer> server;
+
+ @Inject
+ public TransactionService(Configuration conf,
+ ZKClient zkClient,
+ DiscoveryService discoveryService,
+ Provider<TransactionManager> txManagerProvider) {
+ super(conf, discoveryService, txManagerProvider);
+ this.zkClient = zkClient;
+ }
+
+ @Override
+ protected InetSocketAddress getAddress() {
+ if (address.equals("0.0.0.0")) {
+ // resolve hostname
+ try {
+ return new InetSocketAddress(InetAddress.getLocalHost().getHostName(), server.getBindAddress().getPort());
+ } catch (UnknownHostException x) {
+ LOG.error("Cannot resolve hostname for 0.0.0.0", x);
+ }
+ }
+ return server.getBindAddress();
+ }
+
+ @Override
+ protected void doStart() {
+ leaderElection = new LeaderElection(zkClient, "/tx.service/leader", new ElectionHandler() {
+ @Override
+ public void leader() {
+ // if the txManager fails, we should stop the server
+ txManager = txManagerProvider.get();
+ txManager.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void failed(State from, Throwable failure) {
+ LOG.error("Transaction manager aborted, stopping transaction service");
+ TransactionService.this.abort(failure);
+ }
+ }, MoreExecutors.sameThreadExecutor());
+
+ server = ThriftRPCServer.builder(TTransactionServer.class)
+ .setHost(address)
+ .setPort(port)
+ .setWorkerThreads(threads)
+ .setMaxReadBufferBytes(maxReadBufferBytes)
+ .setIOThreads(ioThreads)
+ .build(new TransactionServiceThriftHandler(txManager));
+ try {
+ server.startAndWait();
+ doRegister();
+ LOG.info("Transaction Thrift Service started successfully on " + getAddress());
+ } catch (Throwable t) {
+ LOG.info("Transaction Thrift Service didn't start on " + server.getBindAddress());
+ leaderElection.stop();
+ notifyFailed(t);
+ }
+ }
+
+ @Override
+ public void follower() {
+ // First stop the transaction server as un-registering from discovery can block sometimes.
+ // That can lead to multiple transaction servers being active at the same time.
+ if (server != null && server.isRunning()) {
+ server.stopAndWait();
+ }
+ undoRegister();
+ }
+ });
+ leaderElection.start();
+
+ notifyStarted();
+ }
+
+ @VisibleForTesting
+ State thriftRPCServerState() {
+ return server.state();
+ }
+
+ @Override
+ protected void doStop() {
+ internalStop();
+ notifyStopped();
+ }
+
+ protected void abort(Throwable cause) {
+ // try to clear leader status and shutdown RPC
+ internalStop();
+ notifyFailed(cause);
+ }
+
+ protected void internalStop() {
+ if (leaderElection != null) {
+ // NOTE: if was a leader this will cause loosing of leadership which in callback above will
+ // de-register service in discovery service and stop the service if needed
+ try {
+ Uninterruptibles.getUninterruptibly(leaderElection.stop(), 5, TimeUnit.SECONDS);
+ } catch (TimeoutException te) {
+ LOG.warn("Timed out waiting for leader election cancellation to complete");
+ } catch (ExecutionException e) {
+ LOG.error("Exception when cancelling leader election.", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceClient.java b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceClient.java
new file mode 100644
index 0000000..ca15e9e
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceClient.java
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.InvalidTruncateTimeException;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCouldNotTakeSnapshotException;
+import org.apache.tephra.TransactionNotInProgressException;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.tephra.util.ConfigurationFactory;
+import org.apache.thrift.TException;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * A tx service client
+ */
+public class TransactionServiceClient implements TransactionSystemClient {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TransactionServiceClient.class);
+
+ // we will use this to provide every call with an tx client
+ private ThriftClientProvider clientProvider;
+
+ // the retry strategy we will use
+ private final RetryStrategyProvider retryStrategyProvider;
+
+ /**
+ * Utility to be used for basic verification of transaction system availability and functioning
+ * @param args arguments list, accepts single option "-v" that makes it to print out more details about started tx
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length > 1 || (args.length == 1 && !"-v".equals(args[0]))) {
+ System.out.println("USAGE: TransactionServiceClient [-v]");
+ }
+
+ boolean verbose = false;
+ if (args.length == 1 && "-v".equals(args[0])) {
+ verbose = true;
+ }
+ doMain(verbose, new ConfigurationFactory().get());
+ }
+
+ @VisibleForTesting
+ public static void doMain(boolean verbose, Configuration conf) throws Exception {
+ LOG.info("Starting tx server client test.");
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ new TransactionModules().getDistributedModules(),
+ new TransactionClientModule()
+ );
+
+ ZKClientService zkClient = injector.getInstance(ZKClientService.class);
+ zkClient.startAndWait();
+
+ try {
+ TransactionServiceClient client = injector.getInstance(TransactionServiceClient.class);
+ LOG.info("Starting tx...");
+ Transaction tx = client.startShort();
+ if (verbose) {
+ LOG.info("Started tx details: " + tx.toString());
+ } else {
+ LOG.info("Started tx: " + tx.getTransactionId() +
+ ", readPointer: " + tx.getReadPointer() +
+ ", invalids: " + tx.getInvalids().length +
+ ", inProgress: " + tx.getInProgress().length);
+ }
+ LOG.info("Checking if canCommit tx...");
+ boolean canCommit = client.canCommit(tx, Collections.<byte[]>emptyList());
+ LOG.info("canCommit: " + canCommit);
+ if (canCommit) {
+ LOG.info("Committing tx...");
+ boolean committed = client.commit(tx);
+ LOG.info("Committed tx: " + committed);
+ if (!committed) {
+ LOG.info("Aborting tx...");
+ client.abort(tx);
+ LOG.info("Aborted tx...");
+ }
+ } else {
+ LOG.info("Aborting tx...");
+ client.abort(tx);
+ LOG.info("Aborted tx...");
+ }
+ } finally {
+ zkClient.stopAndWait();
+ }
+ }
+
+ /**
+ * Create from a configuration. This will first attempt to find a zookeeper
+ * for service discovery. Otherwise it will look for the port in the
+ * config and use localhost.
+ * @param config a configuration containing the zookeeper properties
+ */
+ @Inject
+ public TransactionServiceClient(Configuration config,
+ ThriftClientProvider clientProvider) {
+
+ // initialize the retry logic
+ String retryStrat = config.get(
+ TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY,
+ TxConstants.Service.DEFAULT_DATA_TX_CLIENT_RETRY_STRATEGY);
+ if ("backoff".equals(retryStrat)) {
+ this.retryStrategyProvider = new RetryWithBackoff.Provider();
+ } else if ("n-times".equals(retryStrat)) {
+ this.retryStrategyProvider = new RetryNTimes.Provider();
+ } else {
+ String message = "Unknown Retry Strategy '" + retryStrat + "'.";
+ LOG.error(message);
+ throw new IllegalArgumentException(message);
+ }
+ this.retryStrategyProvider.configure(config);
+ LOG.debug("Retry strategy is " + this.retryStrategyProvider);
+
+ this.clientProvider = clientProvider;
+ }
+
+ /**
+ * This is an abstract class that encapsulates an operation. It provides a
+ * method to attempt the actual operation, and it can throw an operation
+ * exception.
+ * @param <T> The return type of the operation
+ */
+ abstract static class Operation<T> {
+
+ /** the name of the operation. */
+ String name;
+
+ /** constructor with name of operation. */
+ Operation(String name) {
+ this.name = name;
+ }
+
+ /** return the name of the operation. */
+ String getName() {
+ return name;
+ }
+
+ /** execute the operation, given an tx client. */
+ abstract T execute(TransactionServiceThriftClient client)
+ throws Exception;
+ }
+
+ /** see execute(operation, client). */
+ private <T> T execute(Operation<T> operation) throws Exception {
+ return execute(operation, null);
+ }
+
+ /**
+ * This is a generic method implementing the somewhat complex execution
+ * and retry logic for operations, to avoid repetitive code.
+ *
+ * Attempts to execute one operation, by obtaining an tx client from
+ * the client provider and passing the operation to the client. If the
+ * call fails with a Thrift exception, apply the retry strategy. If no
+ * more retries are to be made according to the strategy, call the
+ * operation's error method to obtain a value to return. Note that error()
+ * may also throw an exception. Note also that the retry logic is only
+ * applied for thrift exceptions.
+ *
+ * @param operation The operation to be executed
+ * @param provider An tx client provider. If null, then a client will be
+ * obtained using the client provider
+ * @param <T> The return type of the operation
+ * @return the result of the operation, or a value returned by error()
+ */
+ private <T> T execute(Operation<T> operation, ThriftClientProvider provider) throws Exception {
+ RetryStrategy retryStrategy = retryStrategyProvider.newRetryStrategy();
+ while (true) {
+ // did we get a custom client provider or do we use the default?
+ if (provider == null) {
+ provider = this.clientProvider;
+ }
+ // this will throw a TException if it cannot get a client
+ try (CloseableThriftClient closeable = provider.getCloseableClient()) {
+ // note that this can throw exceptions other than TException
+ return operation.execute(closeable.getThriftClient());
+
+ } catch (TException te) {
+ // determine whether we should retry
+ boolean retry = retryStrategy.failOnce();
+ if (!retry) {
+ // retry strategy is exceeded, throw an operation exception
+ String message =
+ "Thrift error for " + operation + ": " + te.getMessage();
+ LOG.error(message);
+ LOG.debug(message, te);
+ throw new Exception(message, te);
+ } else {
+ // call retry strategy before retrying
+ retryStrategy.beforeRetry();
+ String msg = "Retrying " + operation.getName() + " after Thrift error: " + te.getMessage();
+ LOG.info(msg);
+ LOG.debug(msg, te);
+ }
+
+ }
+ }
+ }
+
+ @Override
+ public Transaction startLong() {
+ try {
+ return execute(
+ new Operation<Transaction>("startLong") {
+ @Override
+ public Transaction execute(TransactionServiceThriftClient client)
+ throws TException {
+ return client.startLong();
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public Transaction startShort() {
+ try {
+ return execute(
+ new Operation<Transaction>("startShort") {
+ @Override
+ public Transaction execute(TransactionServiceThriftClient client)
+ throws TException {
+ return client.startShort();
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public Transaction startShort(final int timeout) {
+ try {
+ return execute(
+ new Operation<Transaction>("startShort") {
+ @Override
+ public Transaction execute(TransactionServiceThriftClient client)
+ throws TException {
+ return client.startShort(timeout);
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public boolean canCommit(final Transaction tx, final Collection<byte[]> changeIds)
+ throws TransactionNotInProgressException {
+
+ try {
+ return execute(
+ new Operation<Boolean>("canCommit") {
+ @Override
+ public Boolean execute(TransactionServiceThriftClient client)
+ throws Exception {
+ return client.canCommit(tx, changeIds);
+ }
+ });
+ } catch (TransactionNotInProgressException e) {
+ throw e;
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public boolean commit(final Transaction tx) throws TransactionNotInProgressException {
+ try {
+ return this.execute(
+ new Operation<Boolean>("commit") {
+ @Override
+ public Boolean execute(TransactionServiceThriftClient client)
+ throws Exception {
+ return client.commit(tx);
+ }
+ });
+ } catch (TransactionNotInProgressException e) {
+ throw e;
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void abort(final Transaction tx) {
+ try {
+ this.execute(
+ new Operation<Boolean>("abort") {
+ @Override
+ public Boolean execute(TransactionServiceThriftClient client)
+ throws TException {
+ client.abort(tx);
+ return true;
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public boolean invalidate(final long tx) {
+ try {
+ return this.execute(
+ new Operation<Boolean>("invalidate") {
+ @Override
+ public Boolean execute(TransactionServiceThriftClient client)
+ throws TException {
+ return client.invalidate(tx);
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public Transaction checkpoint(final Transaction tx) throws TransactionNotInProgressException {
+ try {
+ return this.execute(
+ new Operation<Transaction>("checkpoint") {
+ @Override
+ Transaction execute(TransactionServiceThriftClient client) throws Exception {
+ return client.checkpoint(tx);
+ }
+ }
+ );
+ } catch (TransactionNotInProgressException e) {
+ throw e;
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
+ try {
+ return this.execute(
+ new Operation<InputStream>("takeSnapshot") {
+ @Override
+ public InputStream execute(TransactionServiceThriftClient client)
+ throws Exception {
+ return client.getSnapshotStream();
+ }
+ });
+ } catch (TransactionCouldNotTakeSnapshotException e) {
+ throw e;
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public String status() {
+ try {
+ return this.execute(
+ new Operation<String>("status") {
+ @Override
+ public String execute(TransactionServiceThriftClient client) throws Exception {
+ return client.status();
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void resetState() {
+ try {
+ this.execute(
+ new Operation<Boolean>("resetState") {
+ @Override
+ public Boolean execute(TransactionServiceThriftClient client)
+ throws TException {
+ client.resetState();
+ return true;
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public boolean truncateInvalidTx(final Set<Long> invalidTxIds) {
+ try {
+ return this.execute(
+ new Operation<Boolean>("truncateInvalidTx") {
+ @Override
+ public Boolean execute(TransactionServiceThriftClient client) throws TException {
+ return client.truncateInvalidTx(invalidTxIds);
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public boolean truncateInvalidTxBefore(final long time) throws InvalidTruncateTimeException {
+ try {
+ return this.execute(
+ new Operation<Boolean>("truncateInvalidTxBefore") {
+ @Override
+ public Boolean execute(TransactionServiceThriftClient client) throws Exception {
+ return client.truncateInvalidTxBefore(time);
+ }
+ });
+ } catch (InvalidTruncateTimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public int getInvalidSize() {
+ try {
+ return this.execute(
+ new Operation<Integer>("getInvalidSize") {
+ @Override
+ public Integer execute(TransactionServiceThriftClient client) throws TException {
+ return client.getInvalidSize();
+ }
+ });
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftClient.java b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftClient.java
new file mode 100644
index 0000000..a7e9592
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftClient.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import org.apache.tephra.InvalidTruncateTimeException;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCouldNotTakeSnapshotException;
+import org.apache.tephra.TransactionNotInProgressException;
+import org.apache.tephra.distributed.thrift.TInvalidTruncateTimeException;
+import org.apache.tephra.distributed.thrift.TTransactionCouldNotTakeSnapshotException;
+import org.apache.tephra.distributed.thrift.TTransactionNotInProgressException;
+import org.apache.tephra.distributed.thrift.TTransactionServer;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.TTransport;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This class is a wrapper around the thrift tx service client, it takes
+ * Operations, converts them into thrift objects, calls the thrift
+ * client, and converts the results back to data fabric classes.
+ * This class also instruments the thrift calls with metrics.
+ */
+public class TransactionServiceThriftClient {
+ private static final Function<byte[], ByteBuffer> BYTES_WRAPPER = new Function<byte[], ByteBuffer>() {
+ @Override
+ public ByteBuffer apply(byte[] input) {
+ return ByteBuffer.wrap(input);
+ }
+ };
+
+ /**
+ * The thrift transport layer. We need this when we close the connection.
+ */
+ TTransport transport;
+
+ /**
+ * The actual thrift client.
+ */
+ TTransactionServer.Client client;
+
+ /**
+ * Whether this client is valid for use.
+ */
+ private final AtomicBoolean isValid = new AtomicBoolean(true);
+
+ /**
+ * Constructor from an existing, connected thrift transport.
+ *
+ * @param transport the thrift transport layer. It must already be connected
+ */
+ public TransactionServiceThriftClient(TTransport transport) {
+ this.transport = transport;
+ // thrift protocol layer, we use binary because so does the service
+ TProtocol protocol = new TBinaryProtocol(transport);
+ // and create a thrift client
+ this.client = new TTransactionServer.Client(protocol);
+ }
+
+ /**
+ * close this client. may be called multiple times
+ */
+ public void close() {
+ if (this.transport.isOpen()) {
+ this.transport.close();
+ }
+ }
+
+ public Transaction startLong() throws TException {
+ try {
+ return TransactionConverterUtils.unwrap(client.startLong());
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public Transaction startShort() throws TException {
+ try {
+ return TransactionConverterUtils.unwrap(client.startShort());
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public Transaction startShort(int timeout) throws TException {
+ try {
+ return TransactionConverterUtils.unwrap(client.startShortTimeout(timeout));
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public boolean canCommit(Transaction tx, Collection<byte[]> changeIds)
+ throws TException, TransactionNotInProgressException {
+ try {
+ return client.canCommitTx(TransactionConverterUtils.wrap(tx),
+ ImmutableSet.copyOf(Iterables.transform(changeIds, BYTES_WRAPPER))).isValue();
+ } catch (TTransactionNotInProgressException e) {
+ throw new TransactionNotInProgressException(e.getMessage());
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+
+
+ public boolean commit(Transaction tx) throws TException, TransactionNotInProgressException {
+ try {
+ return client.commitTx(TransactionConverterUtils.wrap(tx)).isValue();
+ } catch (TTransactionNotInProgressException e) {
+ throw new TransactionNotInProgressException(e.getMessage());
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public void abort(Transaction tx) throws TException {
+ try {
+ client.abortTx(TransactionConverterUtils.wrap(tx));
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public boolean invalidate(long tx) throws TException {
+ try {
+ return client.invalidateTx(tx);
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public Transaction checkpoint(Transaction tx) throws TException {
+ try {
+ return TransactionConverterUtils.unwrap(client.checkpoint(TransactionConverterUtils.wrap(tx)));
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public InputStream getSnapshotStream() throws TException, TransactionCouldNotTakeSnapshotException {
+ try {
+ ByteBuffer buffer = client.getSnapshot();
+ if (buffer.hasArray()) {
+ return new ByteArrayInputStream(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
+ }
+
+ // The ByteBuffer is not backed by array. Read the content to a new byte array and return an InputStream of that.
+ byte[] snapshot = new byte[buffer.remaining()];
+ buffer.get(snapshot);
+ return new ByteArrayInputStream(snapshot);
+ } catch (TTransactionCouldNotTakeSnapshotException e) {
+ throw new TransactionCouldNotTakeSnapshotException(e.getMessage());
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public String status() throws TException {
+ try {
+ return client.status();
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public void resetState() throws TException {
+ try {
+ client.resetState();
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public boolean truncateInvalidTx(Set<Long> invalidTxIds) throws TException {
+ try {
+ return client.truncateInvalidTx(invalidTxIds).isValue();
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public boolean truncateInvalidTxBefore(long time) throws TException, InvalidTruncateTimeException {
+ try {
+ return client.truncateInvalidTxBefore(time).isValue();
+ } catch (TInvalidTruncateTimeException e) {
+ throw new InvalidTruncateTimeException(e.getMessage());
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public int getInvalidSize() throws TException {
+ try {
+ return client.invalidTxSize();
+ } catch (TException e) {
+ isValid.set(false);
+ throw e;
+ }
+ }
+
+ public boolean isValid() {
+ return isValid.get();
+ }
+}
[36/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java
new file mode 100644
index 0000000..d7a09bc
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+import org.apache.thrift.scheme.TupleScheme;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TTransactionCouldNotTakeSnapshotException extends TException implements org.apache.thrift.TBase<TTransactionCouldNotTakeSnapshotException, TTransactionCouldNotTakeSnapshotException._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTransactionCouldNotTakeSnapshotException");
+
+ private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TTransactionCouldNotTakeSnapshotExceptionStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TTransactionCouldNotTakeSnapshotExceptionTupleSchemeFactory());
+ }
+
+ public String message; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ MESSAGE((short)1, "message");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // MESSAGE
+ return MESSAGE;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTransactionCouldNotTakeSnapshotException.class, metaDataMap);
+ }
+
+ public TTransactionCouldNotTakeSnapshotException() {
+ }
+
+ public TTransactionCouldNotTakeSnapshotException(
+ String message)
+ {
+ this();
+ this.message = message;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TTransactionCouldNotTakeSnapshotException(TTransactionCouldNotTakeSnapshotException other) {
+ if (other.isSetMessage()) {
+ this.message = other.message;
+ }
+ }
+
+ public TTransactionCouldNotTakeSnapshotException deepCopy() {
+ return new TTransactionCouldNotTakeSnapshotException(this);
+ }
+
+ @Override
+ public void clear() {
+ this.message = null;
+ }
+
+ public String getMessage() {
+ return this.message;
+ }
+
+ public TTransactionCouldNotTakeSnapshotException setMessage(String message) {
+ this.message = message;
+ return this;
+ }
+
+ public void unsetMessage() {
+ this.message = null;
+ }
+
+ /** Returns true if field message is set (has been assigned a value) and false otherwise */
+ public boolean isSetMessage() {
+ return this.message != null;
+ }
+
+ public void setMessageIsSet(boolean value) {
+ if (!value) {
+ this.message = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case MESSAGE:
+ if (value == null) {
+ unsetMessage();
+ } else {
+ setMessage((String)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case MESSAGE:
+ return getMessage();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case MESSAGE:
+ return isSetMessage();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TTransactionCouldNotTakeSnapshotException)
+ return this.equals((TTransactionCouldNotTakeSnapshotException)that);
+ return false;
+ }
+
+ public boolean equals(TTransactionCouldNotTakeSnapshotException that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_message = true && this.isSetMessage();
+ boolean that_present_message = true && that.isSetMessage();
+ if (this_present_message || that_present_message) {
+ if (!(this_present_message && that_present_message))
+ return false;
+ if (!this.message.equals(that.message))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(TTransactionCouldNotTakeSnapshotException other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ TTransactionCouldNotTakeSnapshotException typedOther = (TTransactionCouldNotTakeSnapshotException)other;
+
+ lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetMessage()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TTransactionCouldNotTakeSnapshotException(");
+ boolean first = true;
+
+ sb.append("message:");
+ if (this.message == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.message);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TTransactionCouldNotTakeSnapshotExceptionStandardSchemeFactory implements SchemeFactory {
+ public TTransactionCouldNotTakeSnapshotExceptionStandardScheme getScheme() {
+ return new TTransactionCouldNotTakeSnapshotExceptionStandardScheme();
+ }
+ }
+
+ private static class TTransactionCouldNotTakeSnapshotExceptionStandardScheme extends StandardScheme<TTransactionCouldNotTakeSnapshotException> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // MESSAGE
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.message != null) {
+ oprot.writeFieldBegin(MESSAGE_FIELD_DESC);
+ oprot.writeString(struct.message);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TTransactionCouldNotTakeSnapshotExceptionTupleSchemeFactory implements SchemeFactory {
+ public TTransactionCouldNotTakeSnapshotExceptionTupleScheme getScheme() {
+ return new TTransactionCouldNotTakeSnapshotExceptionTupleScheme();
+ }
+ }
+
+ private static class TTransactionCouldNotTakeSnapshotExceptionTupleScheme extends TupleScheme<TTransactionCouldNotTakeSnapshotException> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetMessage()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetMessage()) {
+ oprot.writeString(struct.message);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionNotInProgressException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionNotInProgressException.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionNotInProgressException.java
new file mode 100644
index 0000000..6ca6dd4
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionNotInProgressException.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+import org.apache.thrift.scheme.TupleScheme;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TTransactionNotInProgressException extends TException implements org.apache.thrift.TBase<TTransactionNotInProgressException, TTransactionNotInProgressException._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTransactionNotInProgressException");
+
+ private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TTransactionNotInProgressExceptionStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TTransactionNotInProgressExceptionTupleSchemeFactory());
+ }
+
+ public String message; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ MESSAGE((short)1, "message");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // MESSAGE
+ return MESSAGE;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTransactionNotInProgressException.class, metaDataMap);
+ }
+
+ public TTransactionNotInProgressException() {
+ }
+
+ public TTransactionNotInProgressException(
+ String message)
+ {
+ this();
+ this.message = message;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TTransactionNotInProgressException(TTransactionNotInProgressException other) {
+ if (other.isSetMessage()) {
+ this.message = other.message;
+ }
+ }
+
+ public TTransactionNotInProgressException deepCopy() {
+ return new TTransactionNotInProgressException(this);
+ }
+
+ @Override
+ public void clear() {
+ this.message = null;
+ }
+
+ public String getMessage() {
+ return this.message;
+ }
+
+ public TTransactionNotInProgressException setMessage(String message) {
+ this.message = message;
+ return this;
+ }
+
+ public void unsetMessage() {
+ this.message = null;
+ }
+
+ /** Returns true if field message is set (has been assigned a value) and false otherwise */
+ public boolean isSetMessage() {
+ return this.message != null;
+ }
+
+ public void setMessageIsSet(boolean value) {
+ if (!value) {
+ this.message = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case MESSAGE:
+ if (value == null) {
+ unsetMessage();
+ } else {
+ setMessage((String)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case MESSAGE:
+ return getMessage();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case MESSAGE:
+ return isSetMessage();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TTransactionNotInProgressException)
+ return this.equals((TTransactionNotInProgressException)that);
+ return false;
+ }
+
+ public boolean equals(TTransactionNotInProgressException that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_message = true && this.isSetMessage();
+ boolean that_present_message = true && that.isSetMessage();
+ if (this_present_message || that_present_message) {
+ if (!(this_present_message && that_present_message))
+ return false;
+ if (!this.message.equals(that.message))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(TTransactionNotInProgressException other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ TTransactionNotInProgressException typedOther = (TTransactionNotInProgressException)other;
+
+ lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetMessage()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TTransactionNotInProgressException(");
+ boolean first = true;
+
+ sb.append("message:");
+ if (this.message == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.message);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TTransactionNotInProgressExceptionStandardSchemeFactory implements SchemeFactory {
+ public TTransactionNotInProgressExceptionStandardScheme getScheme() {
+ return new TTransactionNotInProgressExceptionStandardScheme();
+ }
+ }
+
+ private static class TTransactionNotInProgressExceptionStandardScheme extends StandardScheme<TTransactionNotInProgressException> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // MESSAGE
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.message != null) {
+ oprot.writeFieldBegin(MESSAGE_FIELD_DESC);
+ oprot.writeString(struct.message);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TTransactionNotInProgressExceptionTupleSchemeFactory implements SchemeFactory {
+ public TTransactionNotInProgressExceptionTupleScheme getScheme() {
+ return new TTransactionNotInProgressExceptionTupleScheme();
+ }
+ }
+
+ private static class TTransactionNotInProgressExceptionTupleScheme extends TupleScheme<TTransactionNotInProgressException> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetMessage()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetMessage()) {
+ oprot.writeString(struct.message);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ }
+ }
+ }
+
+}
+
[52/56] [abbrv] incubator-tephra git commit: Adding NOTICE,
DISCLAIMER and KEYS. Update README to remove some references to Cask
Posted by po...@apache.org.
Adding NOTICE, DISCLAIMER and KEYS.
Update README to remove some references to Cask
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/26ca8e1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/26ca8e1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/26ca8e1a
Branch: refs/heads/master
Commit: 26ca8e1ac8f99b902fb9effef4faebe174543ff1
Parents: 9c69374
Author: poorna <po...@cask.co>
Authored: Fri May 6 17:06:19 2016 -0700
Committer: poorna <po...@cask.co>
Committed: Fri May 6 17:06:19 2016 -0700
----------------------------------------------------------------------
DISCLAIMER | 1 +
KEYS | 161 +++++++++++++++++++-------------------------------------
NOTICE | 5 ++
README.rst | 24 ++++-----
4 files changed, 73 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/26ca8e1a/DISCLAIMER
----------------------------------------------------------------------
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..4ce5c53
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1 @@
+Apache Tephra is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/26ca8e1a/KEYS
----------------------------------------------------------------------
diff --git a/KEYS b/KEYS
index 4dfe358..1725d26 100644
--- a/KEYS
+++ b/KEYS
@@ -8,114 +8,63 @@ Developers:
(gpg --list-sigs <your name>
&& gpg --armor --export <your name>) >> this file.
-Type bits/keyID Date User ID
-pub 4096R/BEB03B4B 2014-07-21 Terence Yim (CONTINUUITY CODE SIGNING KEY) <te...@continuuity.com>
+----------------------------------------------------------------
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.13 (Darwin)
-
-mQINBFPNhbwBEADYehBuvYsFZ0lvXl2Rr9T8gkbYsdSqPd223cz7K7tFblS4gsRo
-MsDGg2I/h4Jgrqm0c2D3cLidpkzGqOtd9dSgDDI+yRPbsO28uKesjbvlk84bLeqp
-7cUMg8lqG0AMp+IE6ddQMjqCJYJWtYeSuQUnqUmvGB9ixz3WRNmYDAYfwguTPTyi
-LkHMuOC+ktaKMoocoerbnida12a23jYkrGolUMKzWb+z0WTuZ2PXyIaqOoF8frqX
-x8eiD/xa5FKvR8R9M3NVn+BkbrbQfI4AymY/XKY4t8H7bx/MGGhrm8BhJXiBXECR
-XoEpx61OrDLTMvGRtvhSSdQB8ZRgBuJIWFEuY9ufyK5Mp4Bte/5PTvuP1jOPZv2g
-p7D0ffuekW4c6QydkcpU6m9+fOvHliAhDfl2le1aOThmjaHN0d022IikMT7bps2W
-46JYMwBj46PNeZVVxyjJzmMN06BpFc9lSI+3lmpy3z6btDvin/6OWaC78myau8Zc
-6skL0Wsg/JXzRgogeJo/AhznibrgHxyRKQbXkKPj82WV8roUWswIaQI1bWuSvRkc
-21JEtE8lV6zhlAyAwdouuAJZQ0cf5FyfE/RFscNlyZT8COkODwE1yV9+BTVz4qA/
-rdMDGSTRqQLPxrqnX+IM5Y35LYU+hoxaHqjftqW7CR94bBZ9F2L62GyWDwARAQAB
-tERUZXJlbmNlIFlpbSAoQ09OVElOVVVJVFkgQ09ERSBTSUdOSU5HIEtFWSkgPHRl
-cmVuY2VAY29udGludXVpdHkuY29tPokCOAQTAQIAIgUCU82FvAIbAwYLCQgHAwIG
-FQgCCQoLBBYCAwECHgECF4AACgkQfmiTXb6wO0tRsg/+P++H6jZ7ZFwc0e3pbcov
-REu25QUDB6L1z4vlb9kaD+6fRw3GYn+hTSFepszuNiTMhgjPhBP6kx10uaMLETje
-eBWW1l59QUxZ/5NwpTwAF7v6ggPUW0Hkjdw5L17thO2VWr3OWEFNFaSCgkplqU7U
-tDgIO1TLRhsnGN/Sbrq63MLvjn/q4SS5iRnPK+VfkcSDq+/yvBSoMnJRP1GKc8OR
-tC/5JRj+EJyFNTD55EhPzGK5NP+dh7FMI7ck7zX7zjvH/5fuEY9MDXRKytB/EzUi
-m/sDhkG5AwCjCJJST3UeAnpI1OkXcDaPk/OmyBb+ZSRPavaggovl9/KRt/CfM38l
-p7HpUEOtXnKrNmxO0USixvA4ygRwZ/FhChcs2L4XgPIjU/ubLSSJFuZ1nTTP3x3o
-Zgag3UkDJLilddjqsXCZNT3yQHheAG9DDSPt5+4s17J1mCzLN5UbWlx9ZX3wPJK0
-0nKlb4QE3+aXoximsFdQu4/oRogVcjXyjOefcepkirSRP9ZSrdDr6LqOuq4d58Vd
-BF6ZA7MA2vQct93aymDGn9AfWADWyNFHVrX2IGS/l7Lq6XcpzvKWtHmBX1KCnjo9
-x2/oL7p8ra7lMa9zsl4qaPrG9jCSX6aLWuaYvb8uS64mcbeJ21WK/57n27xShAvx
-2zwJv9HwdpILqVVxUn/jora5Ag0EU82FvAEQAMEI++mvIkcx7zjucgbfhFnUbqR3
-/YM88nSoQDYh32Zf+5rLz8qbpOA0qvjM6P/eaNtmTWgOOyFJaTcOH3k6HEfv67zt
-pAViy6eiBpUcl7ucs3IU+9qfXB+glNfqISsDDYkmb41kN7S7ntqZD955cdB98tO4
-+oxw17peoU5pA8Drmw69+C3zzlJBiu+wBu0ls24FPvev+5ZgHylgPWk98P9THHNK
-b0Pacgr1oV2awhRlKPXK1JPeCl2DMf+9sgFItgRWvTWLngIBHsHgdlKbvHHlf0QQ
-962724mSUkI/8wPM7WZ+iEOSkKI7OhLAtiHIgSBO3TxGIWgWCpR9M8ggWRSFWj6p
-HbKOAwxQVjatoMYX247ZuF4U4WhJeqaF3cUgQmf8KErUWSvqA0X274zq/SP9Ws9K
-Zp4ogxAZdZoYhB/QBrcqZFdrNx4JVQkZ0qaddPBmU9tExaDiUaeDrhta5VHV9FYy
-zHs6Qm/siTi+Hs76q1WCjsUovKoQ3vrFqe94mZuzDjnb+00zdU3fyCCcxW/PPlxD
-XIIat+4PH6B4vrlh8uy2RijFeJ2L+8CWuloeC+sFGDMlkT9bdVleJdQB2Mgm8pD9
-yOfTJjnt5oX7RV9yuqAWxHiGcGSbwL/euEb/FHweYOL2VJtBGszu3n+XJs3CwIC2
-n11yNzjd5NmmTdgZABEBAAGJAh8EGAECAAkFAlPNhbwCGwwACgkQfmiTXb6wO0ve
-GQ//QGZ+qWg9WFyESABmYpTrgHgP/AfHk/MfRcV4dEQ2yUxvVd0Ta1/8VDmGoK0f
-FUiP8gogWuRKbUOYKwVuHczyoFaiZmXZkhCe4MzKLAJU+9n/MPug/U1BCUxhhgRw
-M9Z3VXGLzKa3w9QFgR+4iB6Eemx+ClNbgJvI26FvC+JDR8hsdlf8VARMEoeeysWl
-K8RLpLqfQHrJWZsFq/AdzC4z/hrtIZk+s1vf9wmEycV+PYoiTEK64LLTr4UoQ3//
-wiA+jowVJeNePVav8af063dFJyi6pIo0rLqVRKyjabQw4hGa5krmF8KfdVMug2uY
-u5cQd+GC1AdLbT9++QLe3L01rr8NVLzwFyrFEi8OwiJZ1iBXtwz1esKTw5fLjvSm
-8qtnFU/GyIhkDgCd3qhAWNWOOC8USK43FdUzrVoj9kshzx5+zq0aUY333XdRt+ot
-qC7/75SMcRoaE1dHiewRE9JveeHoesV0WgBHnufhbS6dMZOmGDYsLEp3UomdLTVL
-EPHJpSc+cYmMWk6uVikIQ/ijRd/tx68WEV+eEJYj2m4W2kWpGWIaZ/pFdx/O/kTG
-/9HeozN1icT5WRIQIM1xuwCA69YBosrP+Rbt71FV01EdDauf/9dHYK3QYK7etbdk
-wo0NWDrrknMOr6CNJJMAnw47oP4wvudruF9PALYLOA3tNx4=
-=NpUI
------END PGP PUBLIC KEY BLOCK-----
-
-Type bits/keyID Date User ID
-pub 4096R/51B1F840 2014-07-21 Continuuity Dev (Continuuity Code Signing Key) <de...@continuuity.com>
+pub 4096R/0990A464 2015-08-26 [expires: 2019-08-26]
+uid [ultimate] Poorna Chandra <po...@apache.org>
+sig 3 0990A464 2015-08-26 Poorna Chandra <po...@apache.org>
+sub 4096R/E5997ABA 2015-08-26 [expires: 2019-08-26]
+sig 0990A464 2015-08-26 Poorna Chandra <po...@apache.org>
-----BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v2.0.14 (GNU/Linux)
+Comment: GPGTools - https://gpgtools.org
-mQINBFPNnlABEADEjU/ZUSZubLa6CpOOKDlUZWG8gcCh47ODsH0GhPnyc7utqXT0
-28DZ8HSzo9ca1B+WNdKK7zR89Bu1lEIXtabRzZZkDDPAoZUvef0knZzG4ywPDlCh
-HjLs/NHzmy4JmyNiguIuu+FSuLjC/rr+3gyY9N+/i2BBX8ZPDvJsDKFx85WzAumi
-dY84+fxcTbZZBKXM67tSGnWl8f7gNBbtwx6A5fLWRhgylTFHYTzSZHLgbkH4zUsM
-nAqF5eOMkdJ3X7aaUb4BEbKIYEw0HziYuTDoSf61kAxMjm/1JAkwWVX7rZHM0Kms
-zrA3Fcq6oeV3eNt8sLO0b8qMWTAt1/MMLcW7GLfFFvxLcU2g2H5mTD3GOkjaPOO+
-pCngwjHI8J6xDZT4/YK0bxN3TPgueertFGyHULMJ4Ci5PXr+zURwIE88aki/BnaG
-KmymLvAP2ag54gaR8UAg84MtPqECOQehprkMqYOjQ5qbArZ9grgUx++EL16aTnod
-XzsbABaaGARzGMwrQ78FqbnB+L6mnMLSWXzlkWf08PsU59TlfGrMQ2IZVTqimk5r
-YmK4dVrXk9LORP3lT/8vlpqLy/rZrB9GHyv+J4mrW4RbkfsJLW5RGecQnfDMyoi1
-Sot9+14HOeSGsojhGTOUJr09EtDY+qrilVns33HjLP8NrZvIk4kYSPCqHwARAQAB
-tERDb250aW51dWl0eSBEZXYgKENvbnRpbnV1aXR5IENvZGUgU2lnbmluZyBLZXkp
-IDxkZXZAY29udGludXVpdHkuY29tPokCOAQTAQIAIgUCU82eUAIbAwYLCQgHAwIG
-FQgCCQoLBBYCAwECHgECF4AACgkQoa7LJlGx+EDjlA/9FffKO3e6DgsURcAZ9XFE
-5+NuQTPO8coCLyjiUHWn5tjNhXgvApw4NTdoc7TZn3FehnUWR+U8T1HZN/ObVw22
-5QtxXBQ3wEqbjKLmtiJ37ZNqjiXEk8dez+Hbw7kZDcxQiXw7GnsrQGPOyo8JDg9U
-NUlNIioCPp2/ib0D98NgDUN2uGq7cjuHbnMRBiYXMYtG9abvDeOi817sY0NJaz03
-V1RtptvQsY/qS1fCD0IzWI/ve5Sv9qhhEevodD0vvAsjepwWaFkLh+Owc53xnQg8
-5KbdZedAmfwxHPF6FK6hxwWayIgBsRYePI/LnO4MAx3WVat3ytfErnPuKy0dR7P5
-vazT5IU7sONqI81vXs5hrWWSw05M304qZOqTp8xjInzzKk/nBZuaxse2VSjvzPtK
-35mVMDwFvGq3T/3bcAE3gzAgD8iuz0CnEjq4ke2JaRoZpdna16PgWmSMH9x5csDb
-A0Y7+KGSELLosG9xFMrTcNQZfkxeyInZDzU9yu1yijukA/zihQlz1UR36+vlAEoW
-9ehqA3vWkpnF3EAUNQetyZhxHQ+pM27dh333vGuJGERwEfoR5iaS8Y+tuq94iR/V
-Gk/4tAJ+f4/okGpWa9A4LIYhSsRHQ9prFWx7HfmsftoDkNTAnHblwDMo2P1f7lRj
-KJYFHBczVcnSqVwMayOrpXC5Ag0EU82eUAEQANZstwFsRiWi5Tmr9KlKwOTx0VfD
-D2YyV53RktZXZ12Oyk0k2hc0An+5wyP7XoHLh36mxQ76yqRRNOQj+mAJX/t/pOn0
-zz3WofslLhYttAOmsEd0O/X/GYZkni2pG6JFgQQJJ3ZbjdOCBChDCLeysme/eE/b
-yzVU//kweugVHAbeHjeFNaUF26nw1bV9btxM6rMSAlhRbTb1DmmtU+giGwAEJP7M
-B7wVLA6BwB42gFd0SurQ1b2lZjKkrNz4vOMJQ46kcqzSBbCYPYji6bFgA0mr6wai
-gCE1i5CwzyvoV1I3qytTinQG+Ev5R+zq9sie8AA8C1HvYYKAfYOsYOc4IHK/d7Jc
-3Gps+JCHsaCTmbUmtw0nJGxr7N7SIYFDgWxfOPGQZJpZCAPRhij4BelZfUh95QRk
-mP3uD35rF89AKXfCmOdjoI0f5N8m7BmuFUyvE50oneZZr7zZkad8ypvNmp1bm8P9
-739DRJk+lwQ3HcEYJb8/gqeTKRW5MisGWebBTr//oz5niwOr0lkssXxlmTbAXvjO
-ea4L815B8jJB+omhyI9JDuzsPZmI89Lt4gpjfeOL8KAJReVX8OtPOQQRKYkE6TAn
-6Ne9+OF4NFS97wJmFswr1rcD0YyfLmrBvqwdKlAESrn3WnynaUaXpKKaWGJJ1fxD
-37RWmrVnSBMQ7wSZABEBAAGJAh8EGAECAAkFAlPNnlACGwwACgkQoa7LJlGx+ECY
-iA//c99Vwl2KfwuKX0DOVkjrN1CcIoe+vv96PES4U8CQY8MkeOqM3pX5Dsqdr/Q5
-IgPbve2hawXj20CIRMlRyPfrV4BLyGh44mbvS1o70dR7RmPYp2cYWJ1+fgNeNeQT
-AgIcIA85pAe1f7A4ElumuiM+De7wnjiWKGDYlkgqhGcLAM34Fywz+Vb0aASOIrFS
-yqYfmzxoCtXCzKvUTdoKvRQvDXlb5qG3BUS0Myl1pGvwiVxxfPwg3lJEYrPWef1K
-gGTf9BfXstfTkwobzGN4N+a6upIT0giaZruXwFYji/4RkE42N9XajmXzIgjy6GJ4
-1yw9kL4SdWPIQMhusP6DyFoyMuZy5SVcLQ4SECqWg+3H2wJL5Nib7GSIUAt9xzrs
-ltYdAv3RVS3O6BuE1aCZz6klmglujq1Kzt1HWOTOGMbhEndyw8IavR+176d9LQLX
-tgEnjvBQtrpj2m3YDEjEHUmR+oJFGL6J47Ixznz4kNFuU/GeFnhQXmyI8WntQ5NX
-g2tpADcLsPrAX0U6Js6jrrig0Td6ezIImF45R3yOHrqjBwTBcibywsqUoyXOsXDY
-gCWXihCWJ09Kpxzc7itkzC16Fd1qWsujkntba1jmlzcdsVxB0+OUGNeS8k0Xpymc
-TLSkCChM9AabgJrsbVqsDSqDBZqK8XgL+ruqwUiZTi47HWM=
-=Nltw
+mQINBFXdbbUBEAD4fGbEYaLpvL0otbiecuhqCAE6ccRgaA89fauPEYigbkqZJPIS
+XXm/VzxmDPwx9SfYRPYP++ElSfPXCVnSExe7FB90bM8WL6t6M29qCUtcNS56VePs
+iq/t5NVRBJYeaThkYjANSzF4rPJBASkAxXKBQ9ciad/P8VhgAlzQK4sHCWGBe9pk
+ixT8lZZKWOYU3GFkB7BnlERlQwPbZloGoP/FSNkMXCDyOi/kAZtaxM4Z6v2beQP8
+7vxYpAgzZobCNfG+SxudlqEoe3+2QAzmxK8ERjtQYMcNJ3iLhC2LZ9SLt3F66yqh
+SxUhUgFxao3PBYKk6rCUETT0OYfzCNZR4YKo6L9nU4cN9WXtFT+Es0/XeqH2PmzJ
+AItab9PxU21bwb0T2UPzy6UrDfV1Ua9qNK5JXRf4kDYUuTVYo0LEZu+8hj8MVdWw
+Lw+wJBuOEyP3pVZeI/9WEQI4w23asQmwdZxdkW5fgpVmsnh8AP7+bBu5jOfN+my6
+gyyDFhxzopUWw1fuucDNkCzFgROGp/yCZNFFedhpTNEwuxoSNxn020It9i51LHLw
+FKflmJjJyT/2zqLLIYLyq9zLCTfs8Zmpgw75zDgNsVDosuI7sb1kFpY7lsdAZr6B
+/j1KpefXqnj+cRwoNiYQrHY16UKKPVzW0/HsSvaG0/9JhlhjOLWNKxgBVQARAQAB
+tCJQb29ybmEgQ2hhbmRyYSA8cG9vcm5hQGFwYWNoZS5vcmc+iQI9BBMBCgAnBQJV
+3W21AhsDBQkHhh+ABQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI7EosYJkKRk
+4gIQAKga03hKwSe78hCcminnYyT97a/9k5qPG5IkEFW1Cda5T3NJqFJfzfFjQx+c
+Wa4siGeX30pBnFrLfMPgtw5zUOqKHe88Krl9agxWBeZQIeAUn65YkKOzBEXp02wr
+eIk9l+p/p7fYRr9tj+m2l+Mzf2kfCda4epat1gKF/nmgRR/4L+O4bLxxbK4MoWNe
+/VA/H47UloH1N4OAqX7pyAbirg5JD8G9YZVM8yq9q1lK3RiHvGi3+kG4runkZs82
+4bLcGkCKeOegdPdIHxNNzs5S2wGi3tNMJwQh7PR2YAJ6ddk2uuJIhx1U/0Q9+iiE
+uIFBxIxYJ0wmERJgI1UBZlIjdpnYvffHI3PO+iDVJ+I3+73LC/Z8s3kKBQHn9c0f
+cjZ+jfs7MQ28yWpbNACeUOo7n35RdxL8K4jN0KgnMpHGaSQCgzzNg/LcAoI4DVQI
+f+CNTlWZTnYFrhrs55VzXcYqSAEIDVe2y/wefCB4KGG5PQ4GrmDW7Ib+xHr2R/bP
+c2MM65CxfKWUTAKFN6Oq2P5V286pN6WKhlN2ZwuIlhfnHB+4HHWRip81nTUDYE2u
+OabwbHJ9IZQT3B+yky1F+cmXoJKFj+ndM401AIkx6cY0ezeekPGs1fxoEoDi28Cs
+99QE1N9+jLIJNj/74jjK+m+ocfx/J1NDahj4gJCeFuhprXQkuQINBFXdbbUBEADK
+JoeImDMTNVDoGahm6fsA0UA0HoAh+gPeuX0Esr+nIXx1i2gNzCo35X3aSzlskJvR
+AqIPMHh0eUTbwoU8Eyf5JurUCOgqJlXXrTdrA0+iblSz1OUkpKY2VfGLUxDnAZpx
+3YwxFQaNkqiMo0APw5z70rh/naHZxGIJPUBvYdqHYVR5ugRFQueIkngUprYQyrA5
+rj8+GYtYaUG9p/7Tw0bSKsUuTZJS3ZKE+2rPOxqtQxRFE6mFTbtU1oemaPa/R5Pf
+j0kugIQF3n2DOoUD+aVmZSWLORM6lc8KpNbx8Rg6SC+cuo/jEjyY2hwLjWV2EW1h
+54Wj97GI309e3YBxbqutxxsL+/eitaUM8/ovfoKB/aRTXOELZRUWUsOveGI/r1ZE
+x89qVqFXnj7fEp5aoy5xzLhQ8u3cZUNPZHpZv1YYjlT7/oaiKrz/p1ixn2lljDII
+vcE7/6WVHAZjXiwiqM0bJ3V0s8+5H9j1P9tD1Rnpzj7ag7H+83QMuTpv9VJnXnFd
+TCDh6SJoHaHwNZzxSrTeHme8Mrgc2tC3qJV3BvBCmcPPMvagusDqc7HxH4tI1jZO
+AizevNFa6a41hUaNBffvvsXLF+CwBYTmg+BrbTNfivaNaMOKqM4e3TxtkaDIkIXm
+fRbTytkvzCo508rkaNqEeDq0FHKsE8aoLawmr23b1QARAQABiQIlBBgBCgAPBQJV
+3W21AhsMBQkHhh+AAAoJEI7EosYJkKRka+AQALlpS4uBqeXOCpvSPAyacEoX0dcN
+CQogpBYIBuzYeQM4CNMaK0QRnGKOiRgjSfuJcY5D842diihSscEdIC1T4L3v3jLI
+pWScZF6qQHVUJ7SdonP1n3RgN+LCfnmQwm5xw9tbj1dRsa2XyXQYOqASkk364App
+C54R1dcMtjDHxmollGAiFrqo1jUlUTPZTEQc4dKtWCzIhhA4PqlVxf34mWr+027w
+wC1jcUhi9zxTLyCWqOIZ9SOZ5XPJ3mJc3xo9J66pwa+ESQvm6pogF/ixQXDL9m/q
+QW8nu2pnfXC9A9U1JpN8biXCI0TCLjrJ7qvvSSsi+EDrBTZOs6M/ok8ZQIhX/3au
+twLHCZ6YQuz7o5yBKqRBTQ6qv9kYX/rhmI6EGxyjnds/if4F9b6dq0xHsOLxSQ7m
+GJH7yQCngOgRxfLIZw71oK2Li5BOXwmSsT1s7E6YnU8wt2peRXvidhNFyXDIcgt8
+I/20jVk3AY9aTP3GPYiiuxRXsWbb+imn3rICTABT7rxLhCJ2w5zphEmtEq3qqn3A
+QPK3epXdsxVGvdaqpbDEdbEzToCkawNDeKPoeAZ6XB1ASFGAaSi1mxVCq5+DxkwE
+FdpOSb9Kc7aFyhYvhwDO0aXBYQ44gOxiiDdYc8NJZoKSJ9mMGj/If2R87hmdGjKP
+sTRdXFul3HuO3LKj
+=qoUL
-----END PGP PUBLIC KEY BLOCK-----
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/26ca8e1a/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..bb171fd
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,5 @@
+Apache Tephra
+Copyright 2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/26ca8e1a/README.rst
----------------------------------------------------------------------
diff --git a/README.rst b/README.rst
index e1221b4..df097a4 100644
--- a/README.rst
+++ b/README.rst
@@ -1,11 +1,11 @@
.. ==================
-.. Cask Tephra
+.. Apache Tephra
.. ==================
|(Tephra)|
**Transactions for Apache HBase** |(TM)|:
-Cask Tephra provides globally consistent transactions on top of Apache HBase. While HBase
+Apache Tephra provides globally consistent transactions on top of Apache HBase. While HBase
provides strong consistency with row- or region-level ACID operations, it sacrifices
cross-region and cross-table consistency in favor of scalability. This trade-off requires
application developers to handle the complexity of ensuring consistency when their modifications
@@ -130,8 +130,8 @@ Getting Started
You can get started with Tephra by building directly from the latest source code::
- git clone https://github.com/caskdata/tephra.git
- cd tephra
+ git clone https://git-wip-us.apache.org/repos/asf/incubator-tephra.git
+ cd incubator-tephra
mvn clean package
After the build completes, you will have a full binary distribution of Tephra under the
@@ -144,12 +144,12 @@ build system's equivalent configuration), in order to make use of Tephra classes
<dependency>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-api</artifactId>
- <version>0.7.1</version>
+ <version>${tephra.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-core</artifactId>
- <version>0.7.1</version>
+ <version>${tephra.version}</version>
</dependency>
Since the HBase APIs have changed between versions, you will need to select the
@@ -160,7 +160,7 @@ For HBase 0.96.x::
<dependency>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-0.96</artifactId>
- <version>0.7.1</version>
+ <version>${tephra.version}</version>
</dependency>
For HBase 0.98.x::
@@ -168,7 +168,7 @@ For HBase 0.98.x::
<dependency>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-0.98</artifactId>
- <version>0.7.1</version>
+ <version>${tephra.version}</version>
</dependency>
For HBase 1.0.x::
@@ -176,7 +176,7 @@ For HBase 1.0.x::
<dependency>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.0</artifactId>
- <version>0.7.1</version>
+ <version>${tephra.version}</version>
</dependency>
If you are running the CDH 5.4, 5.5, or 5.6 version of HBase 1.0.x (this version contains API incompatibilities
@@ -185,7 +185,7 @@ with Apache HBase 1.0.x)::
<dependency>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.0-cdh</artifactId>
- <version>0.7.1</version>
+ <version>${tephra.version}</version>
</dependency>
For HBase 1.1.x or CDH 5.7 version of HBase 1.2.x::
@@ -193,7 +193,7 @@ For HBase 1.1.x or CDH 5.7 version of HBase 1.2.x::
<dependency>
<groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1</artifactId>
- <version>0.7.1</version>
+ <version>${tephra.version}</version>
</dependency>
Deployment and Configuration
@@ -699,7 +699,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-Cask, Cask Tephra and Tephra are trademarks of Cask Data, Inc. All rights reserved.
+Tephra is a trademark of Cask Data, Inc. All rights reserved.
Apache, Apache HBase, and HBase are trademarks of The Apache Software Foundation. Used with permission.
No endorsement by The Apache Software Foundation is implied by the use of these marks.
[34/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionType.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionType.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionType.java
new file mode 100644
index 0000000..8abfc3b
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionType.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+
+public enum TTransactionType implements org.apache.thrift.TEnum {
+ SHORT(1),
+ LONG(2);
+
+ private final int value;
+
+ private TTransactionType(int value) {
+ this.value = value;
+ }
+
+ /**
+ * Get the integer value of this enum value, as defined in the Thrift IDL.
+ */
+ public int getValue() {
+ return value;
+ }
+
+ /**
+ * Find a the enum type by its integer value, as defined in the Thrift IDL.
+ * @return null if the value is not found.
+ */
+ public static TTransactionType findByValue(int value) {
+ switch (value) {
+ case 1:
+ return SHORT;
+ case 2:
+ return LONG;
+ default:
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TVisibilityLevel.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TVisibilityLevel.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TVisibilityLevel.java
new file mode 100644
index 0000000..ed081b5
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TVisibilityLevel.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+
+public enum TVisibilityLevel implements org.apache.thrift.TEnum {
+ SNAPSHOT(1),
+ SNAPSHOT_EXCLUDE_CURRENT(2),
+ SNAPSHOT_ALL(3);
+
+ private final int value;
+
+ private TVisibilityLevel(int value) {
+ this.value = value;
+ }
+
+ /**
+ * Get the integer value of this enum value, as defined in the Thrift IDL.
+ */
+ public int getValue() {
+ return value;
+ }
+
+ /**
+ * Find a the enum type by its integer value, as defined in the Thrift IDL.
+ * @return null if the value is not found.
+ */
+ public static TVisibilityLevel findByValue(int value) {
+ switch (value) {
+ case 1:
+ return SNAPSHOT;
+ case 2:
+ return SNAPSHOT_EXCLUDE_CURRENT;
+ case 3:
+ return SNAPSHOT_ALL;
+ default:
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/inmemory/DetachedTxSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/inmemory/DetachedTxSystemClient.java b/tephra-core/src/main/java/org/apache/tephra/inmemory/DetachedTxSystemClient.java
new file mode 100644
index 0000000..c8bf22a
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/inmemory/DetachedTxSystemClient.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.inmemory;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.tephra.InvalidTruncateTimeException;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCouldNotTakeSnapshotException;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Implementation of the tx system client that doesn't talk to any global service and tries to do its best to meet the
+ * tx system requirements/expectations. In fact it implements enough logic to support running flows (when each flowlet
+ * uses its own detached tx system client, without talking to each other and sharing any state) with "process exactly
+ * once" guarantee if no failures happen.
+ *
+ * NOTE: Will NOT detect conflicts. May leave inconsistent state when process crashes. Does NOT provide even read
+ * isolation guarantees.
+ *
+ * Good for performance testing. For demoing high throughput. For use-cases with relaxed tx guarantees.
+ */
+public class DetachedTxSystemClient implements TransactionSystemClient {
+ // Dataset and queue logic relies on tx id to grow monotonically even after restart. Hence we need to start with
+ // value that is for sure bigger than the last one used before restart.
+ // NOTE: with code below we assume we don't do more than InMemoryTransactionManager.MAX_TX_PER_MS tx/ms
+ // by single client
+ private AtomicLong generator = new AtomicLong(System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS);
+
+ @Override
+ public Transaction startShort() {
+ long wp = getWritePointer();
+ // NOTE: -1 here is because we have logic that uses (readpointer + 1) as a "exclusive stop key" in some datasets
+ return new Transaction(
+ Long.MAX_VALUE - 1, wp, new long[0], new long[0],
+ Transaction.NO_TX_IN_PROGRESS, TransactionType.SHORT);
+ }
+
+ private long getWritePointer() {
+ long wp = generator.incrementAndGet();
+ // NOTE: using InMemoryTransactionManager.MAX_TX_PER_MS to be at least close to real one
+ long now = System.currentTimeMillis();
+ if (wp < now * TxConstants.MAX_TX_PER_MS) {
+ // trying to advance to align with timestamp, but only once: if failed, we'll just try again later with next tx
+ long advanced = now * TxConstants.MAX_TX_PER_MS;
+ if (generator.compareAndSet(wp, advanced)) {
+ wp = advanced;
+ }
+ }
+ return wp;
+ }
+
+ @Override
+ public Transaction startShort(int timeout) {
+ return startShort();
+ }
+
+ @Override
+ public Transaction startLong() {
+ return startShort();
+ }
+
+ @Override
+ public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) {
+ return true;
+ }
+
+ @Override
+ public boolean commit(Transaction tx) {
+ return true;
+ }
+
+ @Override
+ public void abort(Transaction tx) {
+ // do nothing
+ }
+
+ @Override
+ public boolean invalidate(long tx) {
+ return true;
+ }
+
+ @Override
+ public Transaction checkpoint(Transaction tx) {
+ long newWritePointer = getWritePointer();
+ LongArrayList newCheckpointPointers = new LongArrayList(tx.getCheckpointWritePointers());
+ newCheckpointPointers.add(newWritePointer);
+ return new Transaction(tx, newWritePointer, newCheckpointPointers.toLongArray());
+ }
+
+ @Override
+ public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
+ throw new TransactionCouldNotTakeSnapshotException(
+ "Snapshot not implemented in detached transaction system client");
+ }
+
+ @Override
+ public String status() {
+ return TxConstants.STATUS_OK;
+ }
+
+ @Override
+ public void resetState() {
+ // do nothing
+ }
+
+ @Override
+ public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
+ return true;
+ }
+
+ @Override
+ public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
+ return true;
+ }
+
+ @Override
+ public int getInvalidSize() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTransactionService.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTransactionService.java b/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTransactionService.java
new file mode 100644
index 0000000..823f934
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTransactionService.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.inmemory;
+
+import com.google.common.util.concurrent.AbstractService;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TxConstants;
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.discovery.Discoverable;
+import org.apache.twill.discovery.DiscoveryService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Transaction server that manages transaction data for the Reactor.
+ * <p>
+ * Transaction server is HA, one can start multiple instances, only one of which is active and will register itself in
+ * discovery service.
+ * </p>
+ */
+public class InMemoryTransactionService extends AbstractService {
+ private static final Logger LOG = LoggerFactory.getLogger(InMemoryTransactionService.class);
+
+ private final DiscoveryService discoveryService;
+ private final String serviceName;
+ protected final Provider<TransactionManager> txManagerProvider;
+ private Cancellable cancelDiscovery;
+ protected TransactionManager txManager;
+
+ // thrift server config
+ protected final String address;
+ protected final int port;
+ protected final int threads;
+ protected final int ioThreads;
+ protected final int maxReadBufferBytes;
+
+ @Inject
+ public InMemoryTransactionService(Configuration conf,
+ DiscoveryService discoveryService,
+ Provider<TransactionManager> txManagerProvider) {
+
+ this.discoveryService = discoveryService;
+ this.txManagerProvider = txManagerProvider;
+ this.serviceName = conf.get(TxConstants.Service.CFG_DATA_TX_DISCOVERY_SERVICE_NAME,
+ TxConstants.Service.DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME);
+
+ address = conf.get(TxConstants.Service.CFG_DATA_TX_BIND_ADDRESS, TxConstants.Service.DEFAULT_DATA_TX_BIND_ADDRESS);
+ port = conf.getInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, TxConstants.Service.DEFAULT_DATA_TX_BIND_PORT);
+
+ // Retrieve the number of threads for the service
+ threads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS,
+ TxConstants.Service.DEFAULT_DATA_TX_SERVER_THREADS);
+ ioThreads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS,
+ TxConstants.Service.DEFAULT_DATA_TX_SERVER_IO_THREADS);
+
+ maxReadBufferBytes = conf.getInt(TxConstants.Service.CFG_DATA_TX_THRIFT_MAX_READ_BUFFER,
+ TxConstants.Service.DEFAULT_DATA_TX_THRIFT_MAX_READ_BUFFER);
+
+ LOG.info("Configuring TransactionService" +
+ ", address: " + address +
+ ", port: " + port +
+ ", threads: " + threads +
+ ", io threads: " + ioThreads +
+ ", max read buffer (bytes): " + maxReadBufferBytes);
+ }
+
+ protected void undoRegister() {
+ if (cancelDiscovery != null) {
+ cancelDiscovery.cancel();
+ }
+ }
+
+ protected void doRegister() {
+ cancelDiscovery = discoveryService.register(new Discoverable() {
+ @Override
+ public String getName() {
+ return serviceName;
+ }
+
+ @Override
+ public InetSocketAddress getSocketAddress() {
+ return getAddress();
+ }
+ });
+ }
+
+ protected InetSocketAddress getAddress() {
+ return new InetSocketAddress(1);
+ }
+
+ @Override
+ protected void doStart() {
+ try {
+ txManager = txManagerProvider.get();
+ txManager.startAndWait();
+ doRegister();
+ LOG.info("Transaction Thrift service started successfully on " + getAddress());
+ notifyStarted();
+ } catch (Throwable t) {
+ LOG.info("Transaction Thrift service didn't start on " + getAddress());
+ notifyFailed(t);
+ }
+ }
+
+ @Override
+ protected void doStop() {
+ undoRegister();
+ txManager.stopAndWait();
+ notifyStopped();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTxSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTxSystemClient.java b/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTxSystemClient.java
new file mode 100644
index 0000000..da38dd2
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/inmemory/InMemoryTxSystemClient.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.inmemory;
+
+import com.google.inject.Inject;
+import org.apache.tephra.InvalidTruncateTimeException;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCouldNotTakeSnapshotException;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionNotInProgressException;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TxConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ *
+ */
+public class InMemoryTxSystemClient implements TransactionSystemClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(InMemoryTxSystemClient.class);
+
+ TransactionManager txManager;
+
+ @Inject
+ public InMemoryTxSystemClient(TransactionManager txmgr) {
+ txManager = txmgr;
+ }
+
+ @Override
+ public Transaction startLong() {
+ return txManager.startLong();
+ }
+
+ @Override
+ public Transaction startShort() {
+ return txManager.startShort();
+ }
+
+ @Override
+ public Transaction startShort(int timeout) {
+ return txManager.startShort(timeout);
+ }
+
+ @Override
+ public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
+ return changeIds.isEmpty() || txManager.canCommit(tx, changeIds);
+ }
+
+ @Override
+ public boolean commit(Transaction tx) throws TransactionNotInProgressException {
+ return txManager.commit(tx);
+ }
+
+ @Override
+ public void abort(Transaction tx) {
+ txManager.abort(tx);
+ }
+
+ @Override
+ public boolean invalidate(long tx) {
+ return txManager.invalidate(tx);
+ }
+
+ @Override
+ public Transaction checkpoint(Transaction tx) throws TransactionNotInProgressException {
+ return txManager.checkpoint(tx);
+ }
+
+ @Override
+ public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
+ try {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try {
+ boolean snapshotTaken = txManager.takeSnapshot(out);
+ if (!snapshotTaken) {
+ throw new TransactionCouldNotTakeSnapshotException("Transaction manager did not take a snapshot.");
+ }
+ } finally {
+ out.close();
+ }
+ return new ByteArrayInputStream(out.toByteArray());
+ } catch (IOException e) {
+ LOG.error("Snapshot could not be taken", e);
+ throw new TransactionCouldNotTakeSnapshotException(e.getMessage());
+ }
+ }
+
+ @Override
+ public String status() {
+ return txManager.isRunning() ? TxConstants.STATUS_OK : TxConstants.STATUS_NOTOK;
+ }
+
+ @Override
+ public void resetState() {
+ txManager.resetState();
+ }
+
+ @Override
+ public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
+ return txManager.truncateInvalidTx(invalidTxIds);
+ }
+
+ @Override
+ public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
+ return txManager.truncateInvalidTxBefore(time);
+ }
+
+ @Override
+ public int getInvalidSize() {
+ return txManager.getInvalidSize();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/inmemory/MinimalTxSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/inmemory/MinimalTxSystemClient.java b/tephra-core/src/main/java/org/apache/tephra/inmemory/MinimalTxSystemClient.java
new file mode 100644
index 0000000..2f60225
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/inmemory/MinimalTxSystemClient.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.inmemory;
+
+import org.apache.tephra.InvalidTruncateTimeException;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCouldNotTakeSnapshotException;
+import org.apache.tephra.TransactionNotInProgressException;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * Dummy implementation of TxSystemClient. May be useful for perf testing.
+ */
+public class MinimalTxSystemClient implements TransactionSystemClient {
+ private long currentTxPointer = 1;
+
+ @Override
+ public Transaction startShort() {
+ long wp = currentTxPointer++;
+ // NOTE: -1 here is because we have logic that uses (readpointer + 1) as a "exclusive stop key" in some datasets
+ return new Transaction(
+ Long.MAX_VALUE - 1, wp, new long[0], new long[0],
+ Transaction.NO_TX_IN_PROGRESS, TransactionType.SHORT);
+ }
+
+ @Override
+ public Transaction startShort(int timeout) {
+ return startShort();
+ }
+
+ @Override
+ public Transaction startLong() {
+ return startShort();
+ }
+
+ @Override
+ public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) {
+ return true;
+ }
+
+ @Override
+ public boolean commit(Transaction tx) {
+ return true;
+ }
+
+ @Override
+ public void abort(Transaction tx) {
+ // do nothing
+ }
+
+ @Override
+ public boolean invalidate(long tx) {
+ return true;
+ }
+
+ @Override
+ public Transaction checkpoint(Transaction tx) throws TransactionNotInProgressException {
+ return tx;
+ }
+
+ @Override
+ public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
+ throw new TransactionCouldNotTakeSnapshotException("Not snapshot to take.");
+ }
+
+ @Override
+ public String status() {
+ return TxConstants.STATUS_OK;
+ }
+
+ @Override
+ public void resetState() {
+ // do nothing
+ }
+
+ @Override
+ public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
+ return true;
+ }
+
+ @Override
+ public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
+ return true;
+ }
+
+ @Override
+ public int getInvalidSize() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/inmemory/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/inmemory/package-info.java b/tephra-core/src/main/java/org/apache/tephra/inmemory/package-info.java
new file mode 100644
index 0000000..2e396fe
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/inmemory/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains in memory implementation of the transaction system v2.
+ */
+package org.apache.tephra.inmemory;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/metrics/DefaultMetricsCollector.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/metrics/DefaultMetricsCollector.java b/tephra-core/src/main/java/org/apache/tephra/metrics/DefaultMetricsCollector.java
new file mode 100644
index 0000000..86d2afa
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/metrics/DefaultMetricsCollector.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.metrics;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Default metrics collector implementation using <a href="http://metrics.dropwizard.io">Yammer Metrics</a>.
+ *
+ * <p>The reporting frequency for this collector can be configured by setting the
+ * {@code data.tx.metrics.period} configuration property to the reporting frequency in seconds.
+ * </p>
+ */
+public class DefaultMetricsCollector extends TxMetricsCollector {
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultMetricsCollector.class);
+
+ private final MetricRegistry metrics = new MetricRegistry();
+ private JmxReporter jmxReporter;
+ private ScheduledReporter reporter;
+ private int reportPeriod;
+ private ConcurrentMap<String, AtomicLong> gauges = Maps.newConcurrentMap();
+
+ @Override
+ public void configure(Configuration conf) {
+ // initialize selected output reporter
+ reportPeriod = conf.getInt(TxConstants.Metrics.REPORT_PERIOD_KEY, TxConstants.Metrics.REPORT_PERIOD_DEFAULT);
+ LOG.info("Configured metrics report to emit every {} seconds", reportPeriod);
+ // TODO: reporters should be pluggable based on injection
+ jmxReporter = JmxReporter.forRegistry(metrics).build();
+ reporter = Slf4jReporter.forRegistry(metrics)
+ .outputTo(LoggerFactory.getLogger("tephra-metrics"))
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ }
+
+
+ @Override
+ public void gauge(String metricName, int value, String... tags) {
+ AtomicLong gauge = gauges.get(metricName);
+ if (gauge == null) {
+ final AtomicLong newValue = new AtomicLong();
+ if (gauges.putIfAbsent(metricName, newValue) == null) {
+ // first to set the value, need to register the metric
+ metrics.register(metricName, new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return newValue.get();
+ }
+ });
+ gauge = newValue;
+ } else {
+ // someone else set it first
+ gauge = gauges.get(metricName);
+ }
+ }
+ gauge.set(value);
+ }
+
+ @Override
+ public void histogram(String metricName, int value) {
+ metrics.histogram(metricName).update(value);
+ }
+
+ @Override
+ public void rate(String metricName) {
+ metrics.meter(metricName).mark();
+ }
+
+ @Override
+ public void rate(String metricName, int count) {
+ metrics.meter(metricName).mark(count);
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ jmxReporter.start();
+ reporter.start(reportPeriod, TimeUnit.SECONDS);
+ LOG.info("Started metrics reporter");
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ jmxReporter.stop();
+ reporter.stop();
+ LOG.info("Stopped metrics reporter");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/metrics/MetricsCollector.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/metrics/MetricsCollector.java b/tephra-core/src/main/java/org/apache/tephra/metrics/MetricsCollector.java
new file mode 100644
index 0000000..45668e1
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/metrics/MetricsCollector.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.metrics;
+
+import com.google.common.util.concurrent.Service;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Basic API for Tephra to support system metrics.
+ */
+public interface MetricsCollector extends Service {
+ /**
+ * Report a metric as an absolute value.
+ */
+ void gauge(String metricName, int value, String... tags);
+
+ /**
+ * Report a metric as a count over a given time duration. This method uses an implicit count of 1.
+ */
+ void rate(String metricName);
+
+ /**
+ * Report a metric as a count over a given time duration.
+ */
+ void rate(String metricName, int count);
+
+ /**
+ * Report a metric calculating the distribution of the value.
+ */
+ void histogram(String metricName, int value);
+
+ /**
+ * Called before the collector service is started, allowing the collector to setup any
+ * required configuration.
+ */
+ void configure(Configuration conf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/metrics/TxMetricsCollector.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/metrics/TxMetricsCollector.java b/tephra-core/src/main/java/org/apache/tephra/metrics/TxMetricsCollector.java
new file mode 100644
index 0000000..4dc7e2f
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/metrics/TxMetricsCollector.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.metrics;
+
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Metrics Collector Class, to emit Transaction Related Metrics.
+ * Note: This default implementation is a no-op and doesn't emit any metrics
+ */
+public class TxMetricsCollector extends AbstractIdleService implements MetricsCollector {
+
+ @Override
+ public void gauge(String metricName, int value, String... tags) {
+ //no-op
+ }
+
+ @Override
+ public void rate(String metricName) {
+ // no-op
+ }
+
+ @Override
+ public void rate(String metricName, int count) {
+ // no-op
+ }
+
+ @Override
+ public void histogram(String metricName, int value) {
+ // no-op
+ }
+
+ @Override
+ public void configure(Configuration conf) {
+ // no-op
+ }
+
+ /* Service methods */
+
+ @Override
+ protected void startUp() throws Exception {
+ // no-op
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/package-info.java b/tephra-core/src/main/java/org/apache/tephra/package-info.java
new file mode 100644
index 0000000..dea84ac
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains implementations of the transaction system v2.
+ */
+package org.apache.tephra;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionLog.java b/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionLog.java
new file mode 100644
index 0000000..b1e0978
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionLog.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Common implementation of a transaction log, backed by file reader and writer based storage. Classes extending
+ * this class, must also implement {@link TransactionLogWriter} and {@link TransactionLogReader}.
+ */
+public abstract class AbstractTransactionLog implements TransactionLog {
+ /** Time limit, in milliseconds, of an append to the transaction log before we log it as "slow". */
+ private static final long SLOW_APPEND_THRESHOLD = 1000L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionLog.class);
+
+ private final AtomicLong logSequence = new AtomicLong();
+ private final MetricsCollector metricsCollector;
+ protected long timestamp;
+ private volatile boolean initialized;
+ private volatile boolean closed;
+ private AtomicLong syncedUpTo = new AtomicLong();
+ private List<Entry> pendingWrites = Lists.newLinkedList();
+ private TransactionLogWriter writer;
+
+ public AbstractTransactionLog(long timestamp, MetricsCollector metricsCollector) {
+ this.timestamp = timestamp;
+ this.metricsCollector = metricsCollector;
+ }
+
+ /**
+ * Initializes the log file, opening a file writer. Clients calling {@code init()} should ensure that they
+ * also call {@link HDFSTransactionLog#close()}.
+ * @throws java.io.IOException If an error is encountered initializing the file writer.
+ */
+ public synchronized void init() throws IOException {
+ if (initialized) {
+ return;
+ }
+ this.writer = createWriter();
+ this.initialized = true;
+ }
+
+ /**
+ * Returns a log writer to be used for appending any new {@link TransactionEdit} objects.
+ */
+ protected abstract TransactionLogWriter createWriter() throws IOException;
+
+ @Override
+ public abstract String getName();
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public void append(TransactionEdit edit) throws IOException {
+ long startTime = System.nanoTime();
+ synchronized (this) {
+ ensureAvailable();
+
+ Entry entry = new Entry(new LongWritable(logSequence.getAndIncrement()), edit);
+
+ // add to pending edits
+ append(entry);
+ }
+
+ // wait for sync to complete
+ sync();
+ long durationMillis = (System.nanoTime() - startTime) / 1000000L;
+ if (durationMillis > SLOW_APPEND_THRESHOLD) {
+ LOG.info("Slow append to log " + getName() + ", took " + durationMillis + " msec.");
+ }
+ }
+
+ @Override
+ public void append(List<TransactionEdit> edits) throws IOException {
+ long startTime = System.nanoTime();
+ synchronized (this) {
+ ensureAvailable();
+
+ for (TransactionEdit edit : edits) {
+ Entry entry = new Entry(new LongWritable(logSequence.getAndIncrement()), edit);
+
+ // add to pending edits
+ append(entry);
+ }
+ }
+
+ // wait for sync to complete
+ sync();
+ long durationMillis = (System.nanoTime() - startTime) / 1000000L;
+ if (durationMillis > SLOW_APPEND_THRESHOLD) {
+ LOG.info("Slow append to log " + getName() + ", took " + durationMillis + " msec.");
+ }
+ }
+
+ private void ensureAvailable() throws IOException {
+ if (closed) {
+ throw new IOException("Log " + getName() + " is already closed, cannot append!");
+ }
+ if (!initialized) {
+ init();
+ }
+ }
+
+ /*
+ * Appends new writes to the pendingWrites. It is better to keep it in
+ * our own queue rather than writing it to the HDFS output stream because
+ * HDFSOutputStream.writeChunk is not lightweight at all.
+ */
+ private void append(Entry e) throws IOException {
+ pendingWrites.add(e);
+ }
+
+ // Returns all currently pending writes. New writes
+ // will accumulate in a new list.
+ private List<Entry> getPendingWrites() {
+ synchronized (this) {
+ List<Entry> save = this.pendingWrites;
+ this.pendingWrites = new LinkedList<>();
+ return save;
+ }
+ }
+
+ private void sync() throws IOException {
+ // writes out pending entries to the HLog
+ TransactionLogWriter tmpWriter = null;
+ long latestSeq = 0;
+ int entryCount = 0;
+ synchronized (this) {
+ if (closed) {
+ return;
+ }
+ // prevent writer being dereferenced
+ tmpWriter = writer;
+
+ List<Entry> currentPending = getPendingWrites();
+ if (!currentPending.isEmpty()) {
+ tmpWriter.commitMarker(currentPending.size());
+ }
+
+ // write out all accumulated entries to log.
+ for (Entry e : currentPending) {
+ tmpWriter.append(e);
+ entryCount++;
+ latestSeq = Math.max(latestSeq, e.getKey().get());
+ }
+ }
+
+ long lastSynced = syncedUpTo.get();
+ // someone else might have already synced our edits, avoid double syncing
+ if (lastSynced < latestSeq) {
+ tmpWriter.sync();
+ metricsCollector.histogram("wal.sync.size", entryCount);
+ syncedUpTo.compareAndSet(lastSynced, latestSeq);
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ // perform a final sync if any outstanding writes
+ if (!pendingWrites.isEmpty()) {
+ sync();
+ }
+ // NOTE: writer is lazy-inited, so it can be null
+ if (writer != null) {
+ this.writer.close();
+ }
+ this.closed = true;
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public abstract TransactionLogReader getReader() throws IOException;
+
+ /**
+ * Represents an entry in the transaction log. Each entry consists of a key, generated from an incrementing sequence
+ * number, and a value, the {@link TransactionEdit} being stored.
+ */
+ public static class Entry implements Writable {
+ private LongWritable key;
+ private TransactionEdit edit;
+
+ // for Writable
+ public Entry() {
+ this.key = new LongWritable();
+ this.edit = new TransactionEdit();
+ }
+
+ public Entry(LongWritable key, TransactionEdit edit) {
+ this.key = key;
+ this.edit = edit;
+ }
+
+ public LongWritable getKey() {
+ return this.key;
+ }
+
+ public TransactionEdit getEdit() {
+ return this.edit;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ this.key.write(out);
+ this.edit.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.key.readFields(in);
+ this.edit.readFields(in);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionStateStorage.java b/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionStateStorage.java
new file mode 100644
index 0000000..1c51ccd
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/AbstractTransactionStateStorage.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Common base class for all transaction storage implementations. This implement logic to prefix a snapshot
+ * with a version when encoding, and to select the correct codec for decoding based on this version prefix.
+ */
+public abstract class AbstractTransactionStateStorage extends AbstractIdleService implements TransactionStateStorage {
+
+ protected final SnapshotCodecProvider codecProvider;
+
+ protected AbstractTransactionStateStorage(SnapshotCodecProvider codecProvider) {
+ this.codecProvider = codecProvider;
+ }
+
+ @Override
+ public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException {
+ codecProvider.encode(out, snapshot);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/CommitMarkerCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/CommitMarkerCodec.java b/tephra-core/src/main/java/org/apache/tephra/persist/CommitMarkerCodec.java
new file mode 100644
index 0000000..c4f02e5
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/CommitMarkerCodec.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.common.primitives.Bytes;
+import com.google.common.primitives.Ints;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.tephra.TxConstants;
+
+import java.io.Closeable;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+/**
+ * Class to read and write commit markers used in {@link HDFSTransactionLogReaderV2} and above.
+ */
+public class CommitMarkerCodec implements Closeable {
+ private static final byte[] KEY_BYTES = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED.getBytes(Charsets.UTF_8);
+ private final DataOutputBuffer rawKey;
+ private final DataOutputBuffer rawValue;
+ private SequenceFile.ValueBytes valueBytes;
+
+ public CommitMarkerCodec() {
+ this.rawKey = new DataOutputBuffer();
+ this.rawValue = new DataOutputBuffer();
+ }
+
+ @Override
+ public void close() throws IOException {
+ rawKey.close();
+ rawValue.close();
+ }
+
+ // 1. Returns the count when the marker is written correctly
+ // 2. If data is incorrect (for ex, incorrect key, mismatch in key/value/record length), we throw IOException
+ // since this indicates corrupted log file
+ // 3. If data is incomplete, then we throw EOFException which is handled gracefully by the calling method
+ // since we can recover without any consequence
+ public int readMarker(SequenceFile.Reader reader) throws IOException {
+ if (valueBytes == null) {
+ valueBytes = reader.createValueBytes();
+ }
+ rawKey.reset();
+ rawValue.reset();
+
+ // valueBytes need not be reset since nextRaw call does it (and it is a private method)
+ int status = reader.nextRaw(rawKey, valueBytes);
+
+ // if we reach EOF, return -1
+ if (status == -1) {
+ return -1;
+ }
+
+ // Check if the marker key is valid and return the count
+ if (isMarkerValid()) {
+ valueBytes.writeUncompressedBytes(rawValue);
+ rawValue.flush();
+ // rawValue.getData() may return a larger byte array but Ints.fromByteArray will only read the first four bytes
+ return Ints.fromByteArray(rawValue.getData());
+ }
+
+ // EOF not reached and marker is not valid, then thrown an IOException since we can't make progress
+ throw new IOException(String.format("Invalid key for num entries appended found %s, expected : %s",
+ new String(rawKey.getData()), TxConstants.TransactionLog.NUM_ENTRIES_APPENDED));
+ }
+
+ private boolean isMarkerValid() {
+ // rawKey should have the expected length and the matching bytes should start at index 0
+ return rawKey.getLength() == KEY_BYTES.length && Bytes.indexOf(rawKey.getData(), KEY_BYTES) == 0;
+ }
+
+ public static void writeMarker(SequenceFile.Writer writer, int count) throws IOException {
+ writer.appendRaw(KEY_BYTES, 0, KEY_BYTES.length, new CommitEntriesCount(count));
+ }
+
+ @VisibleForTesting
+ static final class CommitEntriesCount implements SequenceFile.ValueBytes {
+ private final int numEntries;
+
+ public CommitEntriesCount(int numEntries) {
+ this.numEntries = numEntries;
+ }
+
+ @Override
+ public void writeUncompressedBytes(DataOutputStream outStream) throws IOException {
+ outStream.write(Ints.toByteArray(numEntries));
+ }
+
+ @Override
+ public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException {
+ throw new IllegalArgumentException("Commit Entries count writing is not expected to be compressed.");
+ }
+
+ @Override
+ public int getSize() {
+ return Ints.BYTES;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLog.java b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLog.java
new file mode 100644
index 0000000..ba781ac
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLog.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * Allows reading from and writing to a transaction write-ahead log stored in HDFS.
+ */
+public class HDFSTransactionLog extends AbstractTransactionLog {
+ private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionLog.class);
+
+ private final FileSystem fs;
+ private final Configuration hConf;
+ private final Path logPath;
+
+ /**
+ * Creates a new HDFS-backed write-ahead log for storing transaction state.
+ * @param fs Open FileSystem instance for opening log files in HDFS.
+ * @param hConf HDFS cluster configuration.
+ * @param logPath Path to the log file.
+ */
+ public HDFSTransactionLog(final FileSystem fs, final Configuration hConf,
+ final Path logPath, long timestamp, MetricsCollector metricsCollector) {
+ super(timestamp, metricsCollector);
+ this.fs = fs;
+ this.hConf = hConf;
+ this.logPath = logPath;
+ }
+
+ @Override
+ protected TransactionLogWriter createWriter() throws IOException {
+ return new LogWriter(fs, hConf, logPath);
+ }
+
+ @Override
+ public String getName() {
+ return logPath.getName();
+ }
+
+ @Override
+ public TransactionLogReader getReader() throws IOException {
+ FileStatus status = fs.getFileStatus(logPath);
+ long length = status.getLen();
+
+ TransactionLogReader reader = null;
+ // check if this file needs to be recovered due to failure
+ // Check for possibly empty file. With appends, currently Hadoop reports a
+ // zero length even if the file has been sync'd. Revisit if HDFS-376 or
+ // HDFS-878 is committed.
+ if (length <= 0) {
+ LOG.warn("File " + logPath + " might be still open, length is 0");
+ }
+
+ try {
+ HDFSUtil hdfsUtil = new HDFSUtil();
+ hdfsUtil.recoverFileLease(fs, logPath, hConf);
+ try {
+ FileStatus newStatus = fs.getFileStatus(logPath);
+ LOG.info("New file size for " + logPath + " is " + newStatus.getLen());
+ SequenceFile.Reader fileReader = new SequenceFile.Reader(fs, logPath, hConf);
+ reader = new HDFSTransactionLogReaderSupplier(fileReader).get();
+ } catch (EOFException e) {
+ if (length <= 0) {
+ // TODO should we ignore an empty, not-last log file if skip.errors
+ // is false? Either way, the caller should decide what to do. E.g.
+ // ignore if this is the last log in sequence.
+ // TODO is this scenario still possible if the log has been
+ // recovered (i.e. closed)
+ LOG.warn("Could not open " + logPath + " for reading. File is empty", e);
+ return null;
+ } else {
+ // EOFException being ignored
+ return null;
+ }
+ }
+ } catch (IOException e) {
+ throw e;
+ }
+ return reader;
+ }
+
+ @VisibleForTesting
+ static final class LogWriter implements TransactionLogWriter {
+ private final SequenceFile.Writer internalWriter;
+ public LogWriter(FileSystem fs, Configuration hConf, Path logPath) throws IOException {
+ // TODO: retry a few times to ride over transient failures?
+ SequenceFile.Metadata metadata = new SequenceFile.Metadata();
+ metadata.set(new Text(TxConstants.TransactionLog.VERSION_KEY),
+ new Text(Byte.toString(TxConstants.TransactionLog.CURRENT_VERSION)));
+
+ this.internalWriter = SequenceFile.createWriter(fs, hConf, logPath, LongWritable.class, TransactionEdit.class,
+ SequenceFile.CompressionType.NONE, null, null, metadata);
+ LOG.debug("Created a new TransactionLog writer for " + logPath);
+ }
+
+ @Override
+ public void append(Entry entry) throws IOException {
+ internalWriter.append(entry.getKey(), entry.getEdit());
+ }
+
+ @Override
+ public void commitMarker(int count) throws IOException {
+ CommitMarkerCodec.writeMarker(internalWriter, count);
+ }
+
+ @Override
+ public void sync() throws IOException {
+ internalWriter.syncFs();
+ }
+
+ @Override
+ public void close() throws IOException {
+ internalWriter.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderSupplier.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderSupplier.java b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderSupplier.java
new file mode 100644
index 0000000..a517903
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderSupplier.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.tephra.TxConstants;
+
+/**
+ * Provides the correct version of {@link TransactionLogReader}, based on the log's version metadata,
+ * to read HDFS Transaction Logs.
+ */
+public class HDFSTransactionLogReaderSupplier implements Supplier<TransactionLogReader> {
+ private final SequenceFile.Reader reader;
+ private final byte version;
+ private TransactionLogReader logReader;
+
+ public HDFSTransactionLogReaderSupplier(SequenceFile.Reader reader) {
+ this.reader = reader;
+ Text versionInfo = reader.getMetadata().get(new Text(TxConstants.TransactionLog.VERSION_KEY));
+ this.version = versionInfo == null ? 1 : Byte.parseByte(versionInfo.toString());
+ }
+
+ @Override
+ public TransactionLogReader get() {
+ if (logReader != null) {
+ return logReader;
+ }
+
+ switch (version) {
+ case 2:
+ logReader = new HDFSTransactionLogReaderV2(reader);
+ return logReader;
+ case 1:
+ logReader = new HDFSTransactionLogReaderV1(reader);
+ return logReader;
+ default:
+ throw new IllegalArgumentException(String.format("Invalid version %s found in the Transaction Log", version));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV1.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV1.java b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV1.java
new file mode 100644
index 0000000..faefaec
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV1.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * {@link TransactionLogReader} that can read v1 (default) version of Transaction logs. The logs are expected to
+ * have a sequence of {@link TransactionEdit}s.
+ */
+public class HDFSTransactionLogReaderV1 implements TransactionLogReader {
+ private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionLogReaderV1.class);
+ private final SequenceFile.Reader reader;
+ private final LongWritable key;
+ private boolean closed;
+
+ public HDFSTransactionLogReaderV1(SequenceFile.Reader reader) {
+ this.reader = reader;
+ this.key = new LongWritable();
+ }
+
+ @Override
+ public TransactionEdit next() throws IOException {
+ return next(new TransactionEdit());
+ }
+
+ @Override
+ public TransactionEdit next(TransactionEdit reuse) throws IOException {
+ if (closed) {
+ return null;
+ }
+
+ try {
+ boolean successful = reader.next(key, reuse);
+ return successful ? reuse : null;
+ } catch (EOFException e) {
+ LOG.warn("Hit an unexpected EOF while trying to read the Transaction Edit. Skipping the entry.", e);
+ return null;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ reader.close();
+ closed = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV2.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV2.java b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV2.java
new file mode 100644
index 0000000..ce50da8
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionLogReaderV2.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Queue;
+
+/**
+ * {@link TransactionLogReader} that can read v2 version of Transaction logs. The logs are expected to
+ * have commit markers that indicates the size of the batch of {@link TransactionEdit}s (follows the marker),
+ * that were synced together. If the expected number of {@link TransactionEdit}s are not present then that set of
+ * {@link TransactionEdit}s are discarded.
+ */
+public class HDFSTransactionLogReaderV2 implements TransactionLogReader {
+ private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionLogReaderV2.class);
+
+ private final SequenceFile.Reader reader;
+ private final Queue<TransactionEdit> transactionEdits;
+ private final CommitMarkerCodec commitMarkerCodec;
+ private final LongWritable key;
+
+ private boolean closed;
+
+ public HDFSTransactionLogReaderV2(SequenceFile.Reader reader) {
+ this.reader = reader;
+ this.transactionEdits = new ArrayDeque<>();
+ this.key = new LongWritable();
+ this.commitMarkerCodec = new CommitMarkerCodec();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ try {
+ commitMarkerCodec.close();
+ } finally {
+ reader.close();
+ closed = true;
+ }
+ }
+
+ @Override
+ public TransactionEdit next() throws IOException {
+ return next(null);
+ }
+
+ @Override
+ public TransactionEdit next(TransactionEdit reuse) throws IOException {
+ if (closed) {
+ return null;
+ }
+
+ if (!transactionEdits.isEmpty()) {
+ return transactionEdits.remove();
+ }
+
+ // Fetch the 'marker' and read 'marker' number of edits
+ populateTransactionEdits();
+ return transactionEdits.poll();
+ }
+
+ private void populateTransactionEdits() throws IOException {
+ // read the marker to determine numEntries to read.
+ int numEntries = 0;
+ try {
+ // can throw EOFException if reading of incomplete commit marker, no other action required since we can safely
+ // ignore this
+ numEntries = commitMarkerCodec.readMarker(reader);
+ } catch (EOFException e) {
+ LOG.warn("Reached EOF in log while trying to read commit marker", e);
+ }
+
+ for (int i = 0; i < numEntries; i++) {
+ TransactionEdit edit = new TransactionEdit();
+ try {
+ if (reader.next(key, edit)) {
+ transactionEdits.add(edit);
+ } else {
+ throw new EOFException("Attempt to read TransactionEdit failed.");
+ }
+ } catch (EOFException e) {
+ // we have reached EOF before reading back numEntries, we clear the partial list and return.
+ LOG.warn("Reached EOF in log before reading {} entries. Ignoring all {} edits since the last marker",
+ numEntries, transactionEdits.size(), e);
+ transactionEdits.clear();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionStateStorage.java b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionStateStorage.java
new file mode 100644
index 0000000..d751ae2
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/persist/HDFSTransactionStateStorage.java
@@ -0,0 +1,492 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.persist;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.io.CountingInputStream;
+import com.google.common.primitives.Longs;
+import com.google.inject.Inject;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.util.ConfigurationFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * Handles persistence of transaction snapshot and logs to a directory in HDFS.
+ *
+ * The directory used for file storage is configured using the {@code data.tx.snapshot.dir} configuration property.
+ * Both snapshot and transaction log files are suffixed with a timestamp to allow easy ordering. Snapshot files
+ * are written with the filename "snapshot.<timestamp>". Transaction log files are written with the filename
+ * "txlog.<timestamp>".
+ */
+public class HDFSTransactionStateStorage extends AbstractTransactionStateStorage {
+ private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionStateStorage.class);
+
+ private static final String SNAPSHOT_FILE_PREFIX = "snapshot.";
+ private static final String TMP_SNAPSHOT_FILE_PREFIX = ".in-progress.snapshot.";
+ private static final String LOG_FILE_PREFIX = "txlog.";
+
+ private static final PathFilter SNAPSHOT_FILE_FILTER = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith(SNAPSHOT_FILE_PREFIX);
+ }
+ };
+
+ // buffer size used for HDFS reads and writes
+ private static final int BUFFER_SIZE = 16384;
+
+ private final Configuration hConf;
+ private final String configuredSnapshotDir;
+ private final MetricsCollector metricsCollector;
+ private FileSystem fs;
+ private Path snapshotDir;
+
+ @Inject
+ public HDFSTransactionStateStorage(Configuration hConf, SnapshotCodecProvider codecProvider,
+ MetricsCollector metricsCollector) {
+ super(codecProvider);
+ this.hConf = hConf;
+ this.configuredSnapshotDir = hConf.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR);
+ this.metricsCollector = metricsCollector;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ Preconditions.checkState(configuredSnapshotDir != null,
+ "Snapshot directory is not configured. Please set " + TxConstants.Manager.CFG_TX_SNAPSHOT_DIR +
+ " in configuration.");
+ String hdfsUser = hConf.get(TxConstants.Manager.CFG_TX_HDFS_USER);
+ if (hdfsUser == null || UserGroupInformation.isSecurityEnabled()) {
+ if (hdfsUser != null && LOG.isDebugEnabled()) {
+ LOG.debug("Ignoring configuration {}={}, running on secure Hadoop",
+ TxConstants.Manager.CFG_TX_HDFS_USER, hdfsUser);
+ }
+ // NOTE: we can start multiple times this storage. As hdfs uses per-jvm cache, we want to create new fs instead
+ // of getting closed one
+ fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf);
+ } else {
+ fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf, hdfsUser);
+ }
+ snapshotDir = new Path(configuredSnapshotDir);
+ LOG.info("Using snapshot dir " + snapshotDir);
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ fs.close();
+ }
+
+ @Override
+ public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
+ // create a temporary file, and save the snapshot
+ Path snapshotTmpFile = new Path(snapshotDir, TMP_SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
+ LOG.debug("Writing snapshot to temporary file {}", snapshotTmpFile);
+
+ FSDataOutputStream out = fs.create(snapshotTmpFile, false, BUFFER_SIZE);
+ // encode the snapshot and stream the serialized version to the file
+ try {
+ codecProvider.encode(out, snapshot);
+ } finally {
+ out.close();
+ }
+
+ // move the temporary file into place with the correct filename
+ Path finalFile = new Path(snapshotDir, SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
+ fs.rename(snapshotTmpFile, finalFile);
+ LOG.debug("Completed snapshot to file {}", finalFile);
+ }
+
+ @Override
+ public TransactionSnapshot getLatestSnapshot() throws IOException {
+ InputStream in = getLatestSnapshotInputStream();
+ if (in == null) {
+ return null;
+ }
+ try {
+ return readSnapshotInputStream(in);
+ } finally {
+ in.close();
+ }
+ }
+
+ @Override
+ public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
+ InputStream in = getLatestSnapshotInputStream();
+ if (in == null) {
+ return null;
+ }
+ try {
+ return readTransactionVisibilityStateFromInputStream(in);
+ } finally {
+ in.close();
+ }
+ }
+
+ private InputStream getLatestSnapshotInputStream() throws IOException {
+ TimestampedFilename[] snapshots = listSnapshotFiles();
+ Arrays.sort(snapshots);
+ if (snapshots.length > 0) {
+ // last is the most recent
+ return fs.open(snapshots[snapshots.length - 1].getPath(), BUFFER_SIZE);
+ }
+
+ LOG.info("No snapshot files found in {}", snapshotDir);
+ return null;
+ }
+
+ private TransactionSnapshot readSnapshotInputStream(InputStream in) throws IOException {
+ CountingInputStream countingIn = new CountingInputStream(in);
+ TransactionSnapshot snapshot = codecProvider.decode(countingIn);
+ LOG.info("Read encoded transaction snapshot of {} bytes", countingIn.getCount());
+ return snapshot;
+ }
+
+ private TransactionVisibilityState readTransactionVisibilityStateFromInputStream(InputStream in) throws IOException {
+ CountingInputStream countingIn = new CountingInputStream(in);
+ TransactionVisibilityState state = codecProvider.decodeTransactionVisibilityState(countingIn);
+ LOG.info("Read encoded transaction snapshot of {} bytes", countingIn.getCount());
+ return state;
+ }
+
+ private TransactionSnapshot readSnapshotFile(Path filePath) throws IOException {
+ FSDataInputStream in = fs.open(filePath, BUFFER_SIZE);
+ try {
+ return readSnapshotInputStream(in);
+ } finally {
+ in.close();
+ }
+ }
+
+ private TimestampedFilename[] listSnapshotFiles() throws IOException {
+ FileStatus[] snapshotFileStatuses = fs.listStatus(snapshotDir, SNAPSHOT_FILE_FILTER);
+ TimestampedFilename[] snapshotFiles = new TimestampedFilename[snapshotFileStatuses.length];
+ for (int i = 0; i < snapshotFileStatuses.length; i++) {
+ snapshotFiles[i] = new TimestampedFilename(snapshotFileStatuses[i].getPath());
+ }
+ return snapshotFiles;
+ }
+
+ @Override
+ public long deleteOldSnapshots(int numberToKeep) throws IOException {
+ TimestampedFilename[] snapshots = listSnapshotFiles();
+ if (snapshots.length == 0) {
+ return -1;
+ }
+ Arrays.sort(snapshots, Collections.reverseOrder());
+ if (snapshots.length <= numberToKeep) {
+ // nothing to remove, oldest timestamp is the last snapshot
+ return snapshots[snapshots.length - 1].getTimestamp();
+ }
+ int toRemoveCount = snapshots.length - numberToKeep;
+ TimestampedFilename[] toRemove = new TimestampedFilename[toRemoveCount];
+ System.arraycopy(snapshots, numberToKeep, toRemove, 0, toRemoveCount);
+
+ for (TimestampedFilename f : toRemove) {
+ LOG.debug("Removing old snapshot file {}", f.getPath());
+ fs.delete(f.getPath(), false);
+ }
+ long oldestTimestamp = snapshots[numberToKeep - 1].getTimestamp();
+ LOG.debug("Removed {} old snapshot files prior to {}", toRemoveCount, oldestTimestamp);
+ return oldestTimestamp;
+ }
+
+ @Override
+ public List<String> listSnapshots() throws IOException {
+ FileStatus[] files = fs.listStatus(snapshotDir, SNAPSHOT_FILE_FILTER);
+ return Lists.transform(Arrays.asList(files), new Function<FileStatus, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable FileStatus input) {
+ return input.getPath().getName();
+ }
+ });
+ }
+
+ @Override
+ public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
+ FileStatus[] statuses = fs.listStatus(snapshotDir, new LogFileFilter(timestamp, Long.MAX_VALUE));
+ TimestampedFilename[] timestampedFiles = new TimestampedFilename[statuses.length];
+ for (int i = 0; i < statuses.length; i++) {
+ timestampedFiles[i] = new TimestampedFilename(statuses[i].getPath());
+ }
+ return Lists.transform(Arrays.asList(timestampedFiles), new Function<TimestampedFilename, TransactionLog>() {
+ @Nullable
+ @Override
+ public TransactionLog apply(@Nullable TimestampedFilename input) {
+ return openLog(input.getPath(), input.getTimestamp());
+ }
+ });
+ }
+
+ @Override
+ public TransactionLog createLog(long timestamp) throws IOException {
+ Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timestamp);
+ return openLog(newLog, timestamp);
+ }
+
+ private TransactionLog openLog(Path path, long timestamp) {
+ return new HDFSTransactionLog(fs, hConf, path, timestamp, metricsCollector);
+ }
+
+ @Override
+ public void deleteLogsOlderThan(long timestamp) throws IOException {
+ FileStatus[] statuses = fs.listStatus(snapshotDir, new LogFileFilter(0, timestamp));
+ int removedCnt = 0;
+ for (FileStatus status : statuses) {
+ LOG.debug("Removing old transaction log {}", status.getPath());
+ if (fs.delete(status.getPath(), false)) {
+ removedCnt++;
+ } else {
+ LOG.error("Failed to delete transaction log file {}", status.getPath());
+ }
+ }
+ LOG.debug("Removed {} transaction logs older than {}", removedCnt, timestamp);
+ }
+
+ @Override
+ public void setupStorage() throws IOException {
+ if (!fs.exists(snapshotDir)) {
+ LOG.info("Creating snapshot dir at {}", snapshotDir);
+ fs.mkdirs(snapshotDir);
+ } else {
+ Preconditions.checkState(fs.isDirectory(snapshotDir),
+ "Configured snapshot directory " + snapshotDir + " is not a directory!");
+ }
+ }
+
+ @Override
+ public List<String> listLogs() throws IOException {
+ FileStatus[] files = fs.listStatus(snapshotDir, new LogFileFilter(0, Long.MAX_VALUE));
+ return Lists.transform(Arrays.asList(files), new Function<FileStatus, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable FileStatus input) {
+ return input.getPath().getName();
+ }
+ });
+ }
+
+ @Override
+ public String getLocation() {
+ return snapshotDir.toString();
+ }
+
+ private static class LogFileFilter implements PathFilter {
+ // starting time of files to include (inclusive)
+ private final long startTime;
+ // ending time of files to include (exclusive)
+ private final long endTime;
+
+ public LogFileFilter(long startTime, long endTime) {
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ @Override
+ public boolean accept(Path path) {
+ if (path.getName().startsWith(LOG_FILE_PREFIX)) {
+ String[] parts = path.getName().split("\\.");
+ if (parts.length == 2) {
+ try {
+ long fileTime = Long.parseLong(parts[1]);
+ return fileTime >= startTime && fileTime < endTime;
+ } catch (NumberFormatException ignored) {
+ LOG.warn("Filename {} did not match the expected pattern prefix.<timestamp>", path.getName());
+ }
+ }
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Represents a filename composed of a prefix and a ".timestamp" suffix. This is useful for manipulating both
+ * snapshot and transaction log filenames.
+ */
+ private static class TimestampedFilename implements Comparable<TimestampedFilename> {
+ private Path path;
+ private String prefix;
+ private long timestamp;
+
+ public TimestampedFilename(Path path) {
+ this.path = path;
+ String[] parts = path.getName().split("\\.");
+ if (parts.length != 2) {
+ throw new IllegalArgumentException("Filename " + path.getName() +
+ " did not match the expected pattern prefix.timestamp");
+ }
+ prefix = parts[0];
+ timestamp = Long.parseLong(parts[1]);
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public int compareTo(TimestampedFilename other) {
+ int res = prefix.compareTo(other.getPrefix());
+ if (res == 0) {
+ res = Longs.compare(timestamp, other.getTimestamp());
+ }
+ return res;
+ }
+ }
+
+ // TODO move this out as a separate command line tool
+ private enum CLIMode { SNAPSHOT, TXLOG };
+ /**
+ * Reads a transaction state snapshot or transaction log from HDFS and prints the entries to stdout.
+ *
+ * Supports the following options:
+ * -s read snapshot state (defaults to the latest)
+ * -l read a transaction log
+ * [filename] reads the given file
+ * @param args
+ */
+ public static void main(String[] args) {
+ List<String> filenames = Lists.newArrayList();
+ CLIMode mode = null;
+ for (String arg : args) {
+ if ("-s".equals(arg)) {
+ mode = CLIMode.SNAPSHOT;
+ } else if ("-l".equals(arg)) {
+ mode = CLIMode.TXLOG;
+ } else if ("-h".equals(arg)) {
+ printUsage(null);
+ } else {
+ filenames.add(arg);
+ }
+ }
+
+ if (mode == null) {
+ printUsage("ERROR: Either -s or -l is required to set mode.", 1);
+ }
+
+ Configuration config = new ConfigurationFactory().get();
+
+ // Use the no-op metrics collector. We are being run as a command line tool, so there are no relevant metrics
+ // to report
+ HDFSTransactionStateStorage storage =
+ new HDFSTransactionStateStorage(config, new SnapshotCodecProvider(config), new TxMetricsCollector());
+ storage.startAndWait();
+ try {
+ switch (mode) {
+ case SNAPSHOT:
+ try {
+ if (filenames.isEmpty()) {
+ TransactionSnapshot snapshot = storage.getLatestSnapshot();
+ printSnapshot(snapshot);
+ }
+ for (String file : filenames) {
+ Path path = new Path(file);
+ TransactionSnapshot snapshot = storage.readSnapshotFile(path);
+ printSnapshot(snapshot);
+ System.out.println();
+ }
+ } catch (IOException ioe) {
+ System.err.println("Error reading snapshot files: " + ioe.getMessage());
+ ioe.printStackTrace();
+ System.exit(1);
+ }
+ break;
+ case TXLOG:
+ if (filenames.isEmpty()) {
+ printUsage("ERROR: At least one transaction log filename is required!", 1);
+ }
+ for (String file : filenames) {
+ TimestampedFilename timestampedFilename = new TimestampedFilename(new Path(file));
+ TransactionLog log = storage.openLog(timestampedFilename.getPath(), timestampedFilename.getTimestamp());
+ printLog(log);
+ System.out.println();
+ }
+ break;
+ }
+ } finally {
+ storage.stop();
+ }
+ }
+
+ private static void printUsage(String message) {
+ printUsage(message, 0);
+ }
+
+ private static void printUsage(String message, int exitCode) {
+ if (message != null) {
+ System.out.println(message);
+ }
+ System.out.println("Usage: java " + HDFSTransactionStateStorage.class.getName() + " (-s|-l) file1 [file2...]");
+ System.out.println();
+ System.out.println("\t-s\tRead files as transaction state snapshots (will default to latest if no file given)");
+ System.out.println("\t-l\tRead files as transaction logs [filename is required]");
+ System.out.println("\t-h\tPrint this message");
+ System.exit(exitCode);
+ }
+
+ private static void printSnapshot(TransactionSnapshot snapshot) {
+ Date snapshotDate = new Date(snapshot.getTimestamp());
+ System.out.println("TransactionSnapshot at " + snapshotDate.toString());
+ System.out.println("\t" + snapshot.toString());
+ }
+
+ private static void printLog(TransactionLog log) {
+ try {
+ System.out.println("TransactionLog " + log.getName());
+ TransactionLogReader reader = log.getReader();
+ TransactionEdit edit;
+ long seq = 0;
+ while ((edit = reader.next()) != null) {
+ System.out.println(String.format(" %d: %s", seq++, edit.toString()));
+ }
+ } catch (IOException ioe) {
+ System.err.println("ERROR reading log " + log.getName() + ": " + ioe.getMessage());
+ ioe.printStackTrace();
+ }
+ }
+}
[41/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersionSpecificFactory.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersionSpecificFactory.java b/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersionSpecificFactory.java
deleted file mode 100644
index 2bc51b5..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersionSpecificFactory.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import com.google.inject.Provider;
-import com.google.inject.ProvisionException;
-import org.apache.twill.internal.utils.Instances;
-
-/**
- * Common class factory behavior for classes which need specific implementations depending on HBase versions.
- * Specific factories can subclass this class and simply plug in the class names for their implementations.
- *
- * @param <T> Version specific class provided by this factory.
- */
-public abstract class HBaseVersionSpecificFactory<T> implements Provider<T> {
- @Override
- public T get() {
- T instance = null;
- try {
- switch (HBaseVersion.get()) {
- case HBASE_94:
- throw new ProvisionException("HBase 0.94 is no longer supported. Please upgrade to HBase 0.96 or newer.");
- case HBASE_96:
- instance = createInstance(getHBase96Classname());
- break;
- case HBASE_98:
- instance = createInstance(getHBase98Classname());
- break;
- case HBASE_10:
- instance = createInstance(getHBase10Classname());
- break;
- case HBASE_10_CDH:
- instance = createInstance(getHBase10CDHClassname());
- break;
- case HBASE_11:
- case HBASE_12_CDH:
- instance = createInstance(getHBase11Classname());
- break;
- case UNKNOWN:
- throw new ProvisionException("Unknown HBase version: " + HBaseVersion.getVersionString());
- }
- } catch (ClassNotFoundException cnfe) {
- throw new ProvisionException(cnfe.getMessage(), cnfe);
- }
- return instance;
- }
-
- protected T createInstance(String className) throws ClassNotFoundException {
- Class clz = Class.forName(className);
- return (T) Instances.newInstance(clz);
- }
-
- protected abstract String getHBase96Classname();
- protected abstract String getHBase98Classname();
- protected abstract String getHBase10Classname();
- protected abstract String getHBase10CDHClassname();
- protected abstract String getHBase11Classname();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/util/TxUtils.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/util/TxUtils.java b/tephra-core/src/main/java/co/cask/tephra/util/TxUtils.java
deleted file mode 100644
index cdc9fef..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/util/TxUtils.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import com.google.common.primitives.Longs;
-
-import java.util.Map;
-
-/**
- * Utility methods supporting transaction operations.
- */
-public class TxUtils {
-
- // Any cell with timestamp less than MAX_NON_TX_TIMESTAMP is assumed to be pre-existing data,
- // i.e. data written before table was converted into transactional table using Tephra.
- // Using 1.1 times current time to determine whether a timestamp is transactional timestamp or not is safe, and does
- // not produce any false positives or false negatives.
- //
- // To prove this, let's say the earliest transactional timestamp written by Tephra was in year 2000, and the oldest
- // that will be written is in year 2200.
- // 01-Jan-2000 GMT is 946684800000 milliseconds since epoch.
- // 31-Dec-2200 GMT is 7289654399000 milliseconds since epoch.
- //
- // Let's say that we enabled transactions on a table on 01-Jan-2000, then 1.1 * 946684800000 = 31-Dec-2002. Using
- // 31-Dec-2002, we can safely say from 01-Jan-2000 onwards, whether a cell timestamp is
- // non-transactional (<= 946684800000).
- // Note that transactional timestamp will be greater than 946684800000000000 (> year 31969) at this instant.
- //
- // On the other end, let's say we enabled transactions on a table on 31-Dec-2200,
- // then 1.1 * 7289654399000 = 07-Feb-2224. Again, we can use this time from 31-Dec-2200 onwards to say whether a
- // cell timestamp is transactional (<= 7289654399000).
- // Note that transactional timestamp will be greater than 7289654399000000000 (> year 232969) at this instant.
- private static final long MAX_NON_TX_TIMESTAMP = (long) (System.currentTimeMillis() * 1.1);
-
- /**
- * Returns the oldest visible timestamp for the given transaction, based on the TTLs configured for each column
- * family. If no TTL is set on any column family, the oldest visible timestamp will be {@code 0}.
- * @param ttlByFamily A map of column family name to TTL value (in milliseconds)
- * @param tx The current transaction
- * @return The oldest timestamp that will be visible for the given transaction and TTL configuration
- */
- public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx) {
- long maxTTL = getMaxTTL(ttlByFamily);
- // we know that data will not be cleaned up while this tx is running up to this point as janitor uses it
- return maxTTL < Long.MAX_VALUE ? tx.getVisibilityUpperBound() - maxTTL * TxConstants.MAX_TX_PER_MS : 0;
- }
-
- /**
- * Returns the oldest visible timestamp for the given transaction, based on the TTLs configured for each column
- * family. If no TTL is set on any column family, the oldest visible timestamp will be {@code 0}.
- * @param ttlByFamily A map of column family name to TTL value (in milliseconds)
- * @param tx The current transaction
- * @param readNonTxnData indicates that the timestamp returned should allow reading non-transactional data
- * @return The oldest timestamp that will be visible for the given transaction and TTL configuration
- */
- public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx, boolean readNonTxnData) {
- if (readNonTxnData) {
- long maxTTL = getMaxTTL(ttlByFamily);
- return maxTTL < Long.MAX_VALUE ? System.currentTimeMillis() - maxTTL : 0;
- }
-
- return getOldestVisibleTimestamp(ttlByFamily, tx);
- }
-
- /**
- * Returns the maximum timestamp to use for time-range operations, based on the given transaction.
- * @param tx The current transaction
- * @return The maximum timestamp (exclusive) to use for time-range operations
- */
- public static long getMaxVisibleTimestamp(Transaction tx) {
- // NOTE: +1 here because we want read up to writepointer inclusive, but timerange's end is exclusive
- // however, we also need to guard against overflow in the case write pointer is set to MAX_VALUE
- return tx.getWritePointer() < Long.MAX_VALUE ?
- tx.getWritePointer() + 1 : tx.getWritePointer();
- }
-
- /**
- * Creates a "dummy" transaction based on the given txVisibilityState's state. This is not a "real" transaction in
- * the sense that it has not been started, data should not be written with it, and it cannot be committed. However,
- * this can still be useful for filtering data according to the txVisibilityState's state. Instead of the actual
- * write pointer from the txVisibilityState, however, we use {@code Long.MAX_VALUE} to avoid mis-identifying any cells
- * as being written by this transaction (and therefore visible).
- */
- public static Transaction createDummyTransaction(TransactionVisibilityState txVisibilityState) {
- return new Transaction(txVisibilityState.getReadPointer(), Long.MAX_VALUE,
- Longs.toArray(txVisibilityState.getInvalid()),
- Longs.toArray(txVisibilityState.getInProgress().keySet()),
- TxUtils.getFirstShortInProgress(txVisibilityState.getInProgress()), TransactionType.SHORT);
- }
-
- /**
- * Returns the write pointer for the first "short" transaction that in the in-progress set, or
- * {@link Transaction#NO_TX_IN_PROGRESS} if none.
- */
- public static long getFirstShortInProgress(Map<Long, TransactionManager.InProgressTx> inProgress) {
- long firstShort = Transaction.NO_TX_IN_PROGRESS;
- for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
- if (!entry.getValue().isLongRunning()) {
- firstShort = entry.getKey();
- break;
- }
- }
- return firstShort;
- }
-
- /**
- * Returns the timestamp for calculating time to live for the given cell timestamp.
- * This takes into account pre-existing non-transactional cells while calculating the time.
- */
- public static long getTimestampForTTL(long cellTs) {
- return isPreExistingVersion(cellTs) ? cellTs * TxConstants.MAX_TX_PER_MS : cellTs;
- }
-
- /**
- * Returns the max TTL for the given TTL values. Returns Long.MAX_VALUE if any of the column families has no TTL set.
- */
- private static long getMaxTTL(Map<byte[], Long> ttlByFamily) {
- long maxTTL = 0;
- for (Long familyTTL : ttlByFamily.values()) {
- maxTTL = Math.max(familyTTL <= 0 ? Long.MAX_VALUE : familyTTL, maxTTL);
- }
- return maxTTL == 0 ? Long.MAX_VALUE : maxTTL;
- }
-
- /**
- * Returns true if version was written before table was converted into transactional table, false otherwise.
- */
- public static boolean isPreExistingVersion(long version) {
- return version < MAX_NON_TX_TIMESTAMP;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/visibility/DefaultFenceWait.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/visibility/DefaultFenceWait.java b/tephra-core/src/main/java/co/cask/tephra/visibility/DefaultFenceWait.java
deleted file mode 100644
index 3d5b25b..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/visibility/DefaultFenceWait.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.visibility;
-
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import com.google.common.base.Stopwatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Default implementation of {@link FenceWait}.
- */
-public class DefaultFenceWait implements FenceWait {
- private static final Logger LOG = LoggerFactory.getLogger(DefaultFenceWait.class);
-
- private final TransactionContext txContext;
-
- DefaultFenceWait(TransactionContext txContext) {
- this.txContext = txContext;
- }
-
- @Override
- public void await(long timeout, TimeUnit timeUnit)
- throws TransactionFailureException, InterruptedException, TimeoutException {
- Stopwatch stopwatch = new Stopwatch();
- stopwatch.start();
- long sleepTimeMicros = timeUnit.toMicros(timeout) / 10;
- // Have sleep time to be within 1 microsecond and 500 milliseconds
- sleepTimeMicros = Math.max(Math.min(sleepTimeMicros, 500 * 1000), 1);
- while (stopwatch.elapsedTime(timeUnit) < timeout) {
- txContext.start();
- try {
- txContext.finish();
- return;
- } catch (TransactionFailureException e) {
- LOG.error("Got exception waiting for fence. Sleeping for {} microseconds", sleepTimeMicros, e);
- txContext.abort();
- TimeUnit.MICROSECONDS.sleep(sleepTimeMicros);
- }
- }
- throw new TimeoutException("Timeout waiting for fence");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/visibility/FenceWait.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/visibility/FenceWait.java b/tephra-core/src/main/java/co/cask/tephra/visibility/FenceWait.java
deleted file mode 100644
index a5370b2..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/visibility/FenceWait.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.visibility;
-
-import co.cask.tephra.TransactionFailureException;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Used by a writer to wait on a fence so that changes are visible to all readers with in-progress transactions.
- */
-public interface FenceWait {
- /**
- * Waits until the fence is complete, or till the timeout specified. The fence wait transaction will get re-tried
- * several times until the timeout.
- * <p>
- *
- * If a fence wait times out then it means there are still some readers with in-progress transactions that have not
- * seen the change. In this case the wait will have to be retried using the same FenceWait object.
- *
- * @param timeout Maximum time to wait
- * @param timeUnit {@link TimeUnit} for timeout and sleepTime
- * @throws TransactionFailureException when not able to start fence wait transaction
- * @throws InterruptedException on any interrupt
- * @throws TimeoutException when timeout is reached
- */
- void await(long timeout, TimeUnit timeUnit)
- throws TransactionFailureException, InterruptedException, TimeoutException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/visibility/ReadFence.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/visibility/ReadFence.java b/tephra-core/src/main/java/co/cask/tephra/visibility/ReadFence.java
deleted file mode 100644
index f7b8850..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/visibility/ReadFence.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.visibility;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionAware;
-import com.google.common.primitives.Bytes;
-import com.google.common.primitives.Longs;
-
-import java.util.Collection;
-import java.util.Collections;
-
-/**
- * Implementation of {@link VisibilityFence} used by reader.
- */
-class ReadFence implements TransactionAware {
- private final byte[] fenceId;
- private Transaction tx;
-
- public ReadFence(byte[] fenceId) {
- this.fenceId = fenceId;
- }
-
- @Override
- public void startTx(Transaction tx) {
- this.tx = tx;
- }
-
- @Override
- public void updateTx(Transaction tx) {
- // Fences only need original transaction
- }
-
- @Override
- public Collection<byte[]> getTxChanges() {
- if (tx == null) {
- throw new IllegalStateException("Transaction has not started yet");
- }
- return Collections.singleton(Bytes.concat(fenceId, Longs.toByteArray(tx.getTransactionId())));
- }
-
- @Override
- public boolean commitTx() throws Exception {
- // Nothing to persist
- return true;
- }
-
- @Override
- public void postTxCommit() {
- tx = null;
- }
-
- @Override
- public boolean rollbackTx() throws Exception {
- // Nothing to rollback
- return true;
- }
-
- @Override
- public String getTransactionAwareName() {
- return getClass().getSimpleName();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/visibility/VisibilityFence.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/visibility/VisibilityFence.java b/tephra-core/src/main/java/co/cask/tephra/visibility/VisibilityFence.java
deleted file mode 100644
index beb92ed..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/visibility/VisibilityFence.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.visibility;
-
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.TransactionSystemClient;
-
-import java.util.concurrent.TimeoutException;
-
-/**
- * VisibilityFence is used to ensure that after a given point in time, all readers see an updated change
- * that got committed.
- * <p>
- *
- * Typically a reader will never conflict with a writer, since a reader only sees committed changes when its
- * transaction started. However to ensure that after a given point all readers are aware of a change,
- * we have to introduce a conflict between a reader and a writer that act on the same data concurrently.
- *<p>
- *
- * This is done by the reader indicating that it is interested in changes to a piece of data by using a fence
- * in its transaction. If there are no changes to the data when reader tries to commit the transaction
- * containing the fence, the commit succeeds.
- * <p>
- *
- * On the other hand, a writer updates the same data in a transaction. After the write transaction is committed,
- * the writer then waits on the fence to ensure that all in-progress readers are aware of this update.
- * When the wait on the fence returns successfully, it means that
- * any in-progress readers that have not seen the change will not be allowed to commit anymore. This will
- * force the readers to start a new transaction, and this ensures that the changes made by writer are visible
- * to the readers.
- *<p>
- *
- * In case an in-progress reader commits when the writer is waiting on the fence, then the wait method will retry
- * until the given timeout.
- * <p>
- *
- * Hence a successful await on a fence ensures that any reader (using the same fence) that successfully commits after
- * this point onwards would see the change.
- *
- * <p>
- * Sample reader code:
- * <pre>
- * <code>
- * TransactionAware fence = VisibilityFence.create(fenceId);
- * TransactionContext readTxContext = new TransactionContext(txClient, fence, table1, table2, ...);
- * readTxContext.start();
- *
- * // do operations using table1, table2, etc.
- *
- * // finally commit
- * try {
- * readTxContext.finish();
- * } catch (TransactionConflictException e) {
- * // handle conflict by aborting and starting over with a new transaction
- * }
- * </code>
- * </pre>
- *<p>
- *
- * Sample writer code:
- * <pre>
- * <code>
- * // start transaction
- * // write change
- * // commit transaction
- *
- * // Now wait on the fence (with the same fenceId as the readers) to ensure that all in-progress readers are
- * aware of this change
- * try {
- * FenceWait fenceWait = VisibilityFence.prepareWait(fenceId, txClient);
- * fenceWait.await(50000, 50, TimeUnit.MILLISECONDS);
- * } catch (TimeoutException e) {
- * // await timed out, the change may not be visible to all in-progress readers.
- * // Application has two options at this point:
- * // 1. Revert the write. Re-try the write and fence wait again.
- * // 2. Retry only the wait with the same fenceWait object (retry logic is not shown here).
- * }
- * </code>
- * </pre>
- *
- * fenceId in the above samples refers to any id that both the readers and writer know for a given
- * piece of data. Both readers and writer will have to use the same fenceId to synchronize on a given data.
- * Typically fenceId uniquely identifies the data in question.
- * For example, if the data is a table row, the fenceId can be composed of table name and row key.
- * If the data is a table cell, the fenceId can be composed of table name, row key, and column key.
- *<p>
- *
- * Note that in this implementation, any reader that starts a transaction after the write is committed, and
- * while this read transaction is in-progress, if a writer successfully starts and completes an await on the fence then
- * this reader will get a conflict while committing the fence even though this reader has seen the latest changes.
- * This is because today there is no way to determine the commit time of a transaction.
- */
-public final class VisibilityFence {
- private VisibilityFence() {
- // Cannot instantiate this class, all functionality is through static methods.
- }
-
- /**
- * Used by a reader to get a fence that can be added to its transaction context.
- *
- * @param fenceId uniquely identifies the data that this fence is used to synchronize.
- * If the data is a table cell then this id can be composed of the table name, row key
- * and column key for the data.
- * @return {@link TransactionAware} to be added to reader's transaction context.
- */
- public static TransactionAware create(byte[] fenceId) {
- return new ReadFence(fenceId);
- }
-
- /**
- * Used by a writer to wait on a fence so that changes are visible to all readers with in-progress transactions.
- *
- * @param fenceId uniquely identifies the data that this fence is used to synchronize.
- * If the data is a table cell then this id can be composed of the table name, row key
- * and column key for the data.
- * @return {@link FenceWait} object
- */
- public static FenceWait prepareWait(byte[] fenceId, TransactionSystemClient txClient)
- throws TransactionFailureException, InterruptedException, TimeoutException {
- return new DefaultFenceWait(new TransactionContext(txClient, new WriteFence(fenceId)));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/visibility/WriteFence.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/visibility/WriteFence.java b/tephra-core/src/main/java/co/cask/tephra/visibility/WriteFence.java
deleted file mode 100644
index a16264d..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/visibility/WriteFence.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.visibility;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionAware;
-import com.google.common.primitives.Bytes;
-import com.google.common.primitives.Longs;
-import com.google.common.primitives.UnsignedBytes;
-
-import java.util.Collection;
-import java.util.TreeSet;
-
-/**
- * Implementation used by {@link FenceWait} to wait for a {@link VisibilityFence}.
- */
-class WriteFence implements TransactionAware {
- private final byte[] fenceId;
- private Transaction tx;
- private Collection<byte[]> inProgressChanges;
-
- public WriteFence(byte[] fenceId) {
- this.fenceId = fenceId;
- }
-
- @Override
- public void startTx(Transaction tx) {
- this.tx = tx;
- if (inProgressChanges == null) {
- inProgressChanges = new TreeSet<>(UnsignedBytes.lexicographicalComparator());
- for (long inProgressTx : tx.getInProgress()) {
- inProgressChanges.add(Bytes.concat(fenceId, Longs.toByteArray(inProgressTx)));
- }
- }
- }
-
- @Override
- public void updateTx(Transaction tx) {
- // Fences only need original transaction
- }
-
- @Override
- public Collection<byte[]> getTxChanges() {
- if (inProgressChanges == null || tx == null) {
- throw new IllegalStateException("Transaction has not started yet");
- }
- return inProgressChanges;
- }
-
- @Override
- public boolean commitTx() throws Exception {
- // Nothing to persist
- return true;
- }
-
- @Override
- public void postTxCommit() {
- tx = null;
- }
-
- @Override
- public boolean rollbackTx() throws Exception {
- // Nothing to rollback
- return true;
- }
-
- @Override
- public String getTransactionAwareName() {
- return getClass().getSimpleName();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicACLData.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicACLData.java b/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicACLData.java
deleted file mode 100644
index ceaffad..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicACLData.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.zookeeper;
-
-import org.apache.twill.zookeeper.ACLData;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Stat;
-
-import java.util.List;
-
-/**
- * A straightforward implementation of {@link ACLData}.
- */
-final class BasicACLData implements ACLData {
-
- private final List<ACL> acl;
- private final Stat stat;
-
- BasicACLData(List<ACL> acl, Stat stat) {
- this.acl = acl;
- this.stat = stat;
- }
-
- @Override
- public List<ACL> getACL() {
- return acl;
- }
-
- @Override
- public Stat getStat() {
- return stat;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeChildren.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeChildren.java b/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeChildren.java
deleted file mode 100644
index ce81ade..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeChildren.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.zookeeper;
-
-import com.google.common.base.Objects;
-import org.apache.twill.zookeeper.NodeChildren;
-import org.apache.zookeeper.data.Stat;
-
-import java.util.List;
-
-/**
- * Implementation of the {@link NodeChildren}.
- */
-final class BasicNodeChildren implements NodeChildren {
-
- private final Stat stat;
- private final List<String> children;
-
- BasicNodeChildren(List<String> children, Stat stat) {
- this.stat = stat;
- this.children = children;
- }
-
- @Override
- public Stat getStat() {
- return stat;
- }
-
- @Override
- public List<String> getChildren() {
- return children;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || !(o instanceof NodeChildren)) {
- return false;
- }
-
- NodeChildren that = (NodeChildren) o;
- return stat.equals(that.getStat()) && children.equals(that.getChildren());
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(children, stat);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeData.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeData.java b/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeData.java
deleted file mode 100644
index b09ff4b..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/zookeeper/BasicNodeData.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.zookeeper;
-
-import com.google.common.base.Objects;
-import org.apache.twill.zookeeper.NodeData;
-import org.apache.zookeeper.data.Stat;
-
-import java.util.Arrays;
-
-/**
- * A straightforward implementation for {@link NodeData}.
- */
-final class BasicNodeData implements NodeData {
-
- private final byte[] data;
- private final Stat stat;
-
- BasicNodeData(byte[] data, Stat stat) {
- this.data = data;
- this.stat = stat;
- }
-
- @Override
- public Stat getStat() {
- return stat;
- }
-
- @Override
- public byte[] getData() {
- return data;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || !(o instanceof NodeData)) {
- return false;
- }
-
- BasicNodeData that = (BasicNodeData) o;
-
- return stat.equals(that.getStat()) && Arrays.equals(data, that.getData());
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(data, stat);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/zookeeper/TephraZKClientService.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/zookeeper/TephraZKClientService.java b/tephra-core/src/main/java/co/cask/tephra/zookeeper/TephraZKClientService.java
deleted file mode 100644
index 5f6f565..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/zookeeper/TephraZKClientService.java
+++ /dev/null
@@ -1,627 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.zookeeper;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Multimap;
-import com.google.common.util.concurrent.AbstractService;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import org.apache.twill.common.Cancellable;
-import org.apache.twill.common.Threads;
-import org.apache.twill.internal.zookeeper.SettableOperationFuture;
-import org.apache.twill.zookeeper.ACLData;
-import org.apache.twill.zookeeper.NodeChildren;
-import org.apache.twill.zookeeper.NodeData;
-import org.apache.twill.zookeeper.OperationFuture;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.apache.zookeeper.AsyncCallback;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import javax.annotation.Nullable;
-
-/**
- * The implementation of {@link ZKClientService}.
- */
-public class TephraZKClientService extends AbstractService implements ZKClientService, Watcher {
-
- private static final Logger LOG = LoggerFactory.getLogger(TephraZKClientService.class);
-
- private final String zkStr;
- private final int sessionTimeout;
- private final List<Watcher> connectionWatchers;
- private final Multimap<String, byte[]> authInfos;
- private final AtomicReference<ZooKeeper> zooKeeper;
- private final Runnable stopTask;
- private ExecutorService eventExecutor;
-
- /**
- * Create a new instance.
- * @param zkStr zookeper connection string
- * @param sessionTimeout timeout in milliseconds
- * @param connectionWatcher watcher to set
- * @param authInfos authorization bytes
- */
- public TephraZKClientService(String zkStr, int sessionTimeout,
- Watcher connectionWatcher, Multimap<String, byte[]> authInfos) {
- this.zkStr = zkStr;
- this.sessionTimeout = sessionTimeout;
- this.connectionWatchers = new CopyOnWriteArrayList<>();
- this.authInfos = copyAuthInfo(authInfos);
- addConnectionWatcher(connectionWatcher);
-
- this.zooKeeper = new AtomicReference<>();
- this.stopTask = createStopTask();
- }
-
- @Override
- public Long getSessionId() {
- ZooKeeper zk = zooKeeper.get();
- return zk == null ? null : zk.getSessionId();
- }
-
- @Override
- public String getConnectString() {
- return zkStr;
- }
-
- @Override
- public Cancellable addConnectionWatcher(final Watcher watcher) {
- if (watcher == null) {
- return new Cancellable() {
- @Override
- public void cancel() {
- // No-op
- }
- };
- }
-
- // Invocation of connection watchers are already done inside the event thread,
- // hence no need to wrap the watcher again.
- connectionWatchers.add(watcher);
- return new Cancellable() {
- @Override
- public void cancel() {
- connectionWatchers.remove(watcher);
- }
- };
- }
-
- @Override
- public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode) {
- return create(path, data, createMode, true);
- }
-
- @Override
- public OperationFuture<String> create(String path, @Nullable byte[] data,
- CreateMode createMode, boolean createParent) {
- return create(path, data, createMode, createParent, ZooDefs.Ids.OPEN_ACL_UNSAFE);
- }
-
- @Override
- public OperationFuture<String> create(String path, @Nullable byte[] data,
- CreateMode createMode, Iterable<ACL> acl) {
- return create(path, data, createMode, true, acl);
- }
-
- @Override
- public OperationFuture<Stat> exists(String path) {
- return exists(path, null);
- }
-
- @Override
- public OperationFuture<NodeChildren> getChildren(String path) {
- return getChildren(path, null);
- }
-
- @Override
- public OperationFuture<NodeData> getData(String path) {
- return getData(path, null);
- }
-
- @Override
- public OperationFuture<Stat> setData(String path, byte[] data) {
- return setData(path, data, -1);
- }
-
- @Override
- public OperationFuture<String> delete(String path) {
- return delete(path, -1);
- }
-
- @Override
- public OperationFuture<Stat> setACL(String path, Iterable<ACL> acl) {
- return setACL(path, acl, -1);
- }
-
- @Override
- public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode,
- boolean createParent, Iterable<ACL> acl) {
- return doCreate(path, data, createMode, createParent, ImmutableList.copyOf(acl), false);
- }
-
- private OperationFuture<String> doCreate(final String path,
- @Nullable final byte[] data,
- final CreateMode createMode,
- final boolean createParent,
- final List<ACL> acl,
- final boolean ignoreNodeExists) {
- final SettableOperationFuture<String> createFuture = SettableOperationFuture.create(path, eventExecutor);
- getZooKeeper().create(path, data, acl, createMode, Callbacks.STRING, createFuture);
- if (!createParent) {
- return createFuture;
- }
-
- // If create parent is request, return a different future
- final SettableOperationFuture<String> result = SettableOperationFuture.create(path, eventExecutor);
- // Watch for changes in the original future
- Futures.addCallback(createFuture, new FutureCallback<String>() {
- @Override
- public void onSuccess(String path) {
- // Propagate if creation was successful
- result.set(path);
- }
-
- @Override
- public void onFailure(Throwable t) {
- // See if the failure can be handled
- if (updateFailureResult(t, result, path, ignoreNodeExists)) {
- return;
- }
- // Create the parent node
- String parentPath = getParent(path);
- if (parentPath.isEmpty()) {
- result.setException(t);
- return;
- }
- // Watch for parent creation complete. Parent is created with the unsafe ACL.
- Futures.addCallback(doCreate(parentPath, null, CreateMode.PERSISTENT,
- true, ZooDefs.Ids.OPEN_ACL_UNSAFE, true), new FutureCallback<String>() {
- @Override
- public void onSuccess(String parentPath) {
- // Create the requested path again
- Futures.addCallback(
- doCreate(path, data, createMode, false, acl, ignoreNodeExists), new FutureCallback<String>() {
- @Override
- public void onSuccess(String pathResult) {
- result.set(pathResult);
- }
-
- @Override
- public void onFailure(Throwable t) {
- // handle the failure
- updateFailureResult(t, result, path, ignoreNodeExists);
- }
- });
- }
-
- @Override
- public void onFailure(Throwable t) {
- result.setException(t);
- }
- });
- }
-
- /**
- * Updates the result future based on the given {@link Throwable}.
- * @param t Cause of the failure
- * @param result Future to be updated
- * @param path Request path for the operation
- * @return {@code true} if it is a failure, {@code false} otherwise.
- */
- private boolean updateFailureResult(Throwable t, SettableOperationFuture<String> result,
- String path, boolean ignoreNodeExists) {
- // Propagate if there is error
- if (!(t instanceof KeeperException)) {
- result.setException(t);
- return true;
- }
- KeeperException.Code code = ((KeeperException) t).code();
- // Node already exists, simply return success if it allows for ignoring node exists (for parent node creation).
- if (ignoreNodeExists && code == KeeperException.Code.NODEEXISTS) {
- // The requested path could be used because it only applies to non-sequential node
- result.set(path);
- return false;
- }
- if (code != KeeperException.Code.NONODE) {
- result.setException(t);
- return true;
- }
- return false;
- }
-
- /**
- * Gets the parent of the given path.
- * @param path Path for computing its parent
- * @return Parent of the given path, or empty string if the given path is the root path already.
- */
- private String getParent(String path) {
- String parentPath = path.substring(0, path.lastIndexOf('/'));
- return (parentPath.isEmpty() && !"/".equals(path)) ? "/" : parentPath;
- }
- });
-
- return result;
- }
-
- @Override
- public OperationFuture<Stat> exists(String path, Watcher watcher) {
- SettableOperationFuture<Stat> result = SettableOperationFuture.create(path, eventExecutor);
- getZooKeeper().exists(path, wrapWatcher(watcher), Callbacks.STAT_NONODE, result);
- return result;
- }
-
- @Override
- public OperationFuture<NodeChildren> getChildren(String path, Watcher watcher) {
- SettableOperationFuture<NodeChildren> result = SettableOperationFuture.create(path, eventExecutor);
- getZooKeeper().getChildren(path, wrapWatcher(watcher), Callbacks.CHILDREN, result);
- return result;
- }
-
- @Override
- public OperationFuture<NodeData> getData(String path, Watcher watcher) {
- SettableOperationFuture<NodeData> result = SettableOperationFuture.create(path, eventExecutor);
- getZooKeeper().getData(path, wrapWatcher(watcher), Callbacks.DATA, result);
-
- return result;
- }
-
- @Override
- public OperationFuture<Stat> setData(String dataPath, byte[] data, int version) {
- SettableOperationFuture<Stat> result = SettableOperationFuture.create(dataPath, eventExecutor);
- getZooKeeper().setData(dataPath, data, version, Callbacks.STAT, result);
- return result;
- }
-
- @Override
- public OperationFuture<String> delete(String deletePath, int version) {
- SettableOperationFuture<String> result = SettableOperationFuture.create(deletePath, eventExecutor);
- getZooKeeper().delete(deletePath, version, Callbacks.VOID, result);
- return result;
- }
-
- @Override
- public OperationFuture<ACLData> getACL(String path) {
- SettableOperationFuture<ACLData> result = SettableOperationFuture.create(path, eventExecutor);
- getZooKeeper().getACL(path, new Stat(), Callbacks.ACL, result);
- return result;
- }
-
- @Override
- public OperationFuture<Stat> setACL(String path, Iterable<ACL> acl, int version) {
- SettableOperationFuture<Stat> result = SettableOperationFuture.create(path, eventExecutor);
- getZooKeeper().setACL(path, ImmutableList.copyOf(acl), version, Callbacks.STAT, result);
- return result;
- }
-
- @Override
- public Supplier<ZooKeeper> getZooKeeperSupplier() {
- return new Supplier<ZooKeeper>() {
- @Override
- public ZooKeeper get() {
- return getZooKeeper();
- }
- };
- }
-
- @Override
- protected void doStart() {
- // A single thread executor for all events
- ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS,
- new LinkedBlockingQueue<Runnable>(),
- Threads.createDaemonThreadFactory("zk-client-EventThread"));
- // Just discard the execution if the executor is closed
- executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
- eventExecutor = executor;
-
- try {
- zooKeeper.set(createZooKeeper());
- } catch (IOException e) {
- notifyFailed(e);
- }
- }
-
- @Override
- protected void doStop() {
- // Submit a task to the executor to make sure all pending events in the executor are fired before
- // transiting this Service into STOPPED state
- eventExecutor.submit(stopTask);
- eventExecutor.shutdown();
- }
-
- /**
- * @return Current {@link ZooKeeper} client.
- */
- private ZooKeeper getZooKeeper() {
- ZooKeeper zk = zooKeeper.get();
- Preconditions.checkArgument(zk != null, "Not connected to zooKeeper.");
- return zk;
- }
-
- /**
- * Wraps the given watcher to be called from the event executor.
- * @param watcher Watcher to be wrapped
- * @return The wrapped Watcher
- */
- private Watcher wrapWatcher(final Watcher watcher) {
- if (watcher == null) {
- return null;
- }
- return new Watcher() {
- @Override
- public void process(final WatchedEvent event) {
- if (eventExecutor.isShutdown()) {
- LOG.debug("Already shutdown. Discarding event: {}", event);
- return;
- }
- eventExecutor.execute(new Runnable() {
- @Override
- public void run() {
- try {
- watcher.process(event);
- } catch (Throwable t) {
- LOG.error("Watcher throws exception.", t);
- }
- }
- });
- }
- };
- }
-
- /**
- * Creates a deep copy of the given authInfos multimap.
- */
- private Multimap<String, byte[]> copyAuthInfo(Multimap<String, byte[]> authInfos) {
- Multimap<String, byte[]> result = ArrayListMultimap.create();
-
- for (Map.Entry<String, byte[]> entry : authInfos.entries()) {
- byte[] info = entry.getValue();
- result.put(entry.getKey(), info == null ? null : Arrays.copyOf(info, info.length));
- }
-
- return result;
- }
-
- @Override
- public void process(WatchedEvent event) {
- State state = state();
- if (state == State.TERMINATED || state == State.FAILED) {
- return;
- }
-
- try {
- if (event.getState() == Event.KeeperState.SyncConnected && state == State.STARTING) {
- LOG.debug("Connected to ZooKeeper: {}", zkStr);
- notifyStarted();
- return;
- }
- if (event.getState() == Event.KeeperState.Expired) {
- LOG.info("ZooKeeper session expired: {}", zkStr);
-
- // When connection expired, simply reconnect again
- if (state != State.RUNNING) {
- return;
- }
- eventExecutor.submit(new Runnable() {
- @Override
- public void run() {
- // Only reconnect if the current state is running
- if (state() != State.RUNNING) {
- return;
- }
- try {
- LOG.info("Reconnect to ZooKeeper due to expiration: {}", zkStr);
- closeZooKeeper(zooKeeper.getAndSet(createZooKeeper()));
- } catch (IOException e) {
- notifyFailed(e);
- }
- }
- });
- }
- } finally {
- if (event.getType() == Event.EventType.None) {
- for (Watcher connectionWatcher : connectionWatchers) {
- connectionWatcher.process(event);
- }
- }
- }
- }
-
- /**
- * Creates a new ZooKeeper connection.
- */
- private ZooKeeper createZooKeeper() throws IOException {
- ZooKeeper zk = new ZooKeeper(zkStr, sessionTimeout, wrapWatcher(this));
- for (Map.Entry<String, byte[]> authInfo : authInfos.entries()) {
- zk.addAuthInfo(authInfo.getKey(), authInfo.getValue());
- }
- return zk;
- }
-
- /**
- * Closes the given {@link ZooKeeper} if it is not null. If there is InterruptedException,
- * it will get logged.
- */
- private void closeZooKeeper(@Nullable ZooKeeper zk) {
- try {
- if (zk != null) {
- zk.close();
- }
- } catch (InterruptedException e) {
- LOG.warn("Interrupted when closing ZooKeeper", e);
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Creates a {@link Runnable} task that will get executed in the event executor for transiting this
- * Service into STOPPED state.
- */
- private Runnable createStopTask() {
- return new Runnable() {
- @Override
- public void run() {
- try {
- // Close the ZK connection in this task will make sure if there is ZK connection created
- // after doStop() was called but before this task has been executed is also closed.
- // It is possible to happen when the following sequence happens:
- //
- // 1. session expired, hence the expired event is triggered
- // 2. The reconnect task executed. With Service.state() == RUNNING, it creates a new ZK client
- // 3. Service.stop() gets called, Service.state() changed to STOPPING
- // 4. The new ZK client created from the reconnect thread update the zooKeeper with the new one
- closeZooKeeper(zooKeeper.getAndSet(null));
- notifyStopped();
- } catch (Exception e) {
- notifyFailed(e);
- }
- }
- };
- }
-
- /**
- * Collection of generic callbacks that simply reflect results into OperationFuture.
- */
- private static final class Callbacks {
- static final AsyncCallback.StringCallback STRING = new AsyncCallback.StringCallback() {
- @Override
- @SuppressWarnings("unchecked")
- public void processResult(int rc, String path, Object ctx, String name) {
- SettableOperationFuture<String> result = (SettableOperationFuture<String>) ctx;
- KeeperException.Code code = KeeperException.Code.get(rc);
- if (code == KeeperException.Code.OK) {
- result.set((name == null || name.isEmpty()) ? path : name);
- return;
- }
- result.setException(KeeperException.create(code, result.getRequestPath()));
- }
- };
-
- static final AsyncCallback.StatCallback STAT = new AsyncCallback.StatCallback() {
- @Override
- @SuppressWarnings("unchecked")
- public void processResult(int rc, String path, Object ctx, Stat stat) {
- SettableOperationFuture<Stat> result = (SettableOperationFuture<Stat>) ctx;
- KeeperException.Code code = KeeperException.Code.get(rc);
- if (code == KeeperException.Code.OK) {
- result.set(stat);
- return;
- }
- result.setException(KeeperException.create(code, result.getRequestPath()));
- }
- };
-
- /**
- * A stat callback that treats NONODE as success.
- */
- static final AsyncCallback.StatCallback STAT_NONODE = new AsyncCallback.StatCallback() {
- @Override
- @SuppressWarnings("unchecked")
- public void processResult(int rc, String path, Object ctx, Stat stat) {
- SettableOperationFuture<Stat> result = (SettableOperationFuture<Stat>) ctx;
- KeeperException.Code code = KeeperException.Code.get(rc);
- if (code == KeeperException.Code.OK || code == KeeperException.Code.NONODE) {
- result.set(stat);
- return;
- }
- result.setException(KeeperException.create(code, result.getRequestPath()));
- }
- };
-
- static final AsyncCallback.Children2Callback CHILDREN = new AsyncCallback.Children2Callback() {
- @Override
- @SuppressWarnings("unchecked")
- public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
- SettableOperationFuture<NodeChildren> result = (SettableOperationFuture<NodeChildren>) ctx;
- KeeperException.Code code = KeeperException.Code.get(rc);
- if (code == KeeperException.Code.OK) {
- result.set(new BasicNodeChildren(children, stat));
- return;
- }
- result.setException(KeeperException.create(code, result.getRequestPath()));
- }
- };
-
- static final AsyncCallback.DataCallback DATA = new AsyncCallback.DataCallback() {
- @Override
- @SuppressWarnings("unchecked")
- public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
- SettableOperationFuture<NodeData> result = (SettableOperationFuture<NodeData>) ctx;
- KeeperException.Code code = KeeperException.Code.get(rc);
- if (code == KeeperException.Code.OK) {
- result.set(new BasicNodeData(data, stat));
- return;
- }
- result.setException(KeeperException.create(code, result.getRequestPath()));
- }
- };
-
- static final AsyncCallback.VoidCallback VOID = new AsyncCallback.VoidCallback() {
- @Override
- @SuppressWarnings("unchecked")
- public void processResult(int rc, String path, Object ctx) {
- SettableOperationFuture<String> result = (SettableOperationFuture<String>) ctx;
- KeeperException.Code code = KeeperException.Code.get(rc);
- if (code == KeeperException.Code.OK) {
- result.set(result.getRequestPath());
- return;
- }
- // Otherwise, it is an error
- result.setException(KeeperException.create(code, result.getRequestPath()));
- }
- };
-
- static final AsyncCallback.ACLCallback ACL = new AsyncCallback.ACLCallback() {
- @Override
- @SuppressWarnings("unchecked")
- public void processResult(int rc, String path, Object ctx, List<ACL> acl, Stat stat) {
- SettableOperationFuture<ACLData> result = (SettableOperationFuture<ACLData>) ctx;
- KeeperException.Code code = KeeperException.Code.get(rc);
- if (code == KeeperException.Code.OK) {
- result.set(new BasicACLData(acl, stat));
- return;
- }
- result.setException(KeeperException.create(code, result.getRequestPath()));
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionAwareTable.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionAwareTable.java b/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionAwareTable.java
new file mode 100644
index 0000000..64fd7ed
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionAwareTable.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Objects;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Bytes;
+import com.google.common.primitives.UnsignedBytes;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Base class for all the common parts of the HBase version-specific {@code TransactionAwareHTable}
+ * implementations.
+ */
+public abstract class AbstractTransactionAwareTable implements TransactionAware {
+ protected final TransactionCodec txCodec;
+ // map of write pointers to change set associated with each
+ protected final Map<Long, Set<ActionChange>> changeSets;
+ protected final TxConstants.ConflictDetection conflictLevel;
+ protected Transaction tx;
+ protected boolean allowNonTransactional;
+
+ public AbstractTransactionAwareTable(TxConstants.ConflictDetection conflictLevel, boolean allowNonTransactional) {
+ this.conflictLevel = conflictLevel;
+ this.allowNonTransactional = allowNonTransactional;
+ this.txCodec = new TransactionCodec();
+ this.changeSets = Maps.newHashMap();
+ }
+
+ /**
+ * True if the instance allows non-transaction operations.
+ * @return
+ */
+ public boolean getAllowNonTransactional() {
+ return this.allowNonTransactional;
+ }
+
+ /**
+ * Set whether the instance allows non-transactional operations.
+ * @param allowNonTransactional
+ */
+ public void setAllowNonTransactional(boolean allowNonTransactional) {
+ this.allowNonTransactional = allowNonTransactional;
+ }
+
+ @Override
+ public void startTx(Transaction tx) {
+ this.tx = tx;
+ }
+
+ @Override
+ public void updateTx(Transaction tx) {
+ this.tx = tx;
+ }
+
+ @Override
+ public Collection<byte[]> getTxChanges() {
+ if (conflictLevel == TxConstants.ConflictDetection.NONE) {
+ return Collections.emptyList();
+ }
+
+ Collection<byte[]> txChanges = new TreeSet<byte[]>(UnsignedBytes.lexicographicalComparator());
+ for (Set<ActionChange> changeSet : changeSets.values()) {
+ for (ActionChange change : changeSet) {
+ txChanges.add(getChangeKey(change.getRow(), change.getFamily(), change.getQualifier()));
+ }
+ }
+ return txChanges;
+ }
+
+ public byte[] getChangeKey(byte[] row, byte[] family, byte[] qualifier) {
+ byte[] key;
+ switch (conflictLevel) {
+ case ROW:
+ key = Bytes.concat(getTableKey(), row);
+ break;
+ case COLUMN:
+ key = Bytes.concat(getTableKey(), row, family, qualifier);
+ break;
+ case NONE:
+ throw new IllegalStateException("NONE conflict detection does not support change keys");
+ default:
+ throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
+ }
+ return key;
+ }
+
+ @Override
+ public boolean commitTx() throws Exception {
+ return doCommit();
+ }
+
+ /**
+ * Commits any pending writes by flushing the wrapped {@code HTable} instance.
+ */
+ protected abstract boolean doCommit() throws IOException;
+
+ @Override
+ public void postTxCommit() {
+ tx = null;
+ changeSets.clear();
+ }
+
+ @Override
+ public String getTransactionAwareName() {
+ return new String(getTableKey(), Charsets.UTF_8);
+ }
+
+ /**
+ * Returns the table name to use as a key prefix for the transaction change set.
+ */
+ protected abstract byte[] getTableKey();
+
+ @Override
+ public boolean rollbackTx() throws Exception {
+ return doRollback();
+ }
+
+ /**
+ * Rolls back any persisted changes from the transaction by issuing offsetting deletes to the
+ * wrapped {@code HTable} instance. How this is handled will depend on the delete API exposed
+ * by the specific version of HBase.
+ */
+ protected abstract boolean doRollback() throws Exception;
+
+ protected void addToChangeSet(byte[] row, byte[] family, byte[] qualifier) {
+ long currentWritePointer = tx.getWritePointer();
+ Set<ActionChange> changeSet = changeSets.get(currentWritePointer);
+ if (changeSet == null) {
+ changeSet = Sets.newHashSet();
+ changeSets.put(currentWritePointer, changeSet);
+ }
+ switch (conflictLevel) {
+ case ROW:
+ case NONE:
+ // with ROW or NONE conflict detection, we still need to track changes per-family, since this
+ // is the granularity at which we will issue deletes for rollback
+ changeSet.add(new ActionChange(row, family));
+ break;
+ case COLUMN:
+ changeSet.add(new ActionChange(row, family, qualifier));
+ break;
+ default:
+ throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
+ }
+ }
+
+ /**
+ * Record of each transaction that causes a change. This reference is used to rollback
+ * any operation upon failure.
+ */
+ protected class ActionChange {
+ private final byte[] row;
+ private final byte[] family;
+ private final byte[] qualifier;
+
+ public ActionChange(byte[] row, byte[] family) {
+ this(row, family, null);
+ }
+
+ public ActionChange(byte[] row, byte[] family, byte[] qualifier) {
+ this.row = row;
+ this.family = family;
+ this.qualifier = qualifier;
+ }
+
+ public byte[] getRow() {
+ return row;
+ }
+
+ public byte[] getFamily() {
+ return family;
+ }
+
+ public byte[] getQualifier() {
+ return qualifier;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || o.getClass() != this.getClass()) {
+ return false;
+ }
+
+ if (o == this) {
+ return true;
+ }
+
+ ActionChange other = (ActionChange) o;
+ return Objects.equal(this.row, other.row) &&
+ Objects.equal(this.family, other.family) &&
+ Objects.equal(this.qualifier, other.qualifier);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode(row);
+ result = 31 * result + (family != null ? Arrays.hashCode(family) : 0);
+ result = 31 * result + (qualifier != null ? Arrays.hashCode(qualifier) : 0);
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionExecutor.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionExecutor.java b/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionExecutor.java
new file mode 100644
index 0000000..528085f
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/AbstractTransactionExecutor.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * Provides implementation of asynchronous methods of {@link TransactionExecutor} by delegating their execution
+ * to respective synchronous methods via provided {@link ExecutorService}.
+ */
+public abstract class AbstractTransactionExecutor implements TransactionExecutor {
+ private final ListeningExecutorService executorService;
+
+ protected AbstractTransactionExecutor(ExecutorService executorService) {
+ this.executorService = MoreExecutors.listeningDecorator(executorService);
+ }
+
+ @Override
+ public <I, O> O executeUnchecked(Function<I, O> function, I input) {
+ try {
+ return execute(function, input);
+ } catch (TransactionFailureException e) {
+ throw Throwables.propagate(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public <I> void executeUnchecked(Procedure<I> procedure, I input) {
+ try {
+ execute(procedure, input);
+ } catch (TransactionFailureException e) {
+ throw Throwables.propagate(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public <O> O executeUnchecked(Callable<O> callable) {
+ try {
+ return execute(callable);
+ } catch (TransactionFailureException e) {
+ throw Throwables.propagate(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void executeUnchecked(Subroutine subroutine) {
+ try {
+ execute(subroutine);
+ } catch (TransactionFailureException e) {
+ throw Throwables.propagate(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public <I, O> ListenableFuture<O> submit(final Function<I, O> function, final I input) {
+ return executorService.submit(new Callable<O>() {
+ @Override
+ public O call() throws Exception {
+ return execute(function, input);
+ }
+ });
+ }
+
+ @Override
+ public <I> ListenableFuture<?> submit(final Procedure<I> procedure, final I input) {
+ return executorService.submit(new Callable<Object>() {
+ @Override
+ public I call() throws Exception {
+ execute(procedure, input);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public <O> ListenableFuture<O> submit(final Callable<O> callable) {
+ return executorService.submit(new Callable<O>() {
+ @Override
+ public O call() throws Exception {
+ return execute(callable);
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<?> submit(final Subroutine subroutine) {
+ return executorService.submit(new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ execute(subroutine);
+ return null;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/ChangeId.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/ChangeId.java b/tephra-core/src/main/java/org/apache/tephra/ChangeId.java
new file mode 100644
index 0000000..0eb7191
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/ChangeId.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import java.util.Arrays;
+
+/**
+ * Represents a row key from a data set changed as part of a transaction.
+ */
+public final class ChangeId {
+ private final byte[] key;
+ private final int hash;
+
+ public ChangeId(byte[] bytes) {
+ key = bytes;
+ hash = Arrays.hashCode(bytes);
+ }
+
+ public byte[] getKey() {
+ return key;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o == null || o.getClass() != ChangeId.class) {
+ return false;
+ }
+ ChangeId other = (ChangeId) o;
+ return hash == other.hash && Arrays.equals(key, other.key);
+ }
+
+ @Override
+ public int hashCode() {
+ return hash;
+ }
+
+ @Override
+ public String toString() {
+ return toStringBinary(key, 0, key.length);
+ }
+
+ // Copy from Bytes.toStringBinary so that we don't need direct dependencies on Bytes.
+ private String toStringBinary(byte [] b, int off, int len) {
+ StringBuilder result = new StringBuilder();
+ for (int i = off; i < off + len; ++i) {
+ int ch = b[i] & 0xFF;
+ if ((ch >= '0' && ch <= '9')
+ || (ch >= 'A' && ch <= 'Z')
+ || (ch >= 'a' && ch <= 'z')
+ || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0) {
+ result.append((char) ch);
+ } else {
+ result.append(String.format("\\x%02X", ch));
+ }
+ }
+ return result.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/DefaultTransactionExecutor.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/DefaultTransactionExecutor.java b/tephra-core/src/main/java/org/apache/tephra/DefaultTransactionExecutor.java
new file mode 100644
index 0000000..c5e1cb3
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/DefaultTransactionExecutor.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utility class that encapsulates the transaction life cycle over a given set of
+ * transaction-aware datasets. The executor can be reused across multiple invocations
+ * of the execute() method. However, it is not thread-safe for concurrent execution.
+ * <p>
+ * Transaction execution will be retries according to specified in constructor {@link RetryStrategy}.
+ * By default {@link RetryOnConflictStrategy} is used with max 20 retries and 100 ms between retries.
+ * </p>
+ */
+public class DefaultTransactionExecutor extends AbstractTransactionExecutor {
+
+ private final Collection<TransactionAware> txAwares;
+ private final TransactionSystemClient txClient;
+ private final RetryStrategy retryStrategy;
+
+ /**
+ * Convenience constructor, has same affect as {@link #DefaultTransactionExecutor(TransactionSystemClient, Iterable)}
+ */
+ public DefaultTransactionExecutor(TransactionSystemClient txClient, TransactionAware... txAwares) {
+ this(txClient, Arrays.asList(txAwares));
+ }
+
+
+ public DefaultTransactionExecutor(TransactionSystemClient txClient,
+ Iterable<TransactionAware> txAwares,
+ RetryStrategy retryStrategy) {
+
+ super(MoreExecutors.sameThreadExecutor());
+ this.txAwares = ImmutableList.copyOf(txAwares);
+ this.txClient = txClient;
+ this.retryStrategy = retryStrategy;
+ }
+
+ /**
+ * Constructor for a transaction executor.
+ */
+ @Inject
+ public DefaultTransactionExecutor(TransactionSystemClient txClient, @Assisted Iterable<TransactionAware> txAwares) {
+ this(txClient, txAwares, RetryStrategies.retryOnConflict(20, 100));
+ }
+
+ @Override
+ public <I, O> O execute(Function<I, O> function, I input) throws TransactionFailureException, InterruptedException {
+ return executeWithRetry(function, input);
+ }
+
+ @Override
+ public <I> void execute(final Procedure<I> procedure, I input)
+ throws TransactionFailureException, InterruptedException {
+
+ execute(new Function<I, Void>() {
+ @Override
+ public Void apply(I input) throws Exception {
+ procedure.apply(input);
+ return null;
+ }
+ }, input);
+ }
+
+ @Override
+ public <O> O execute(final Callable<O> callable) throws TransactionFailureException, InterruptedException {
+ return execute(new Function<Void, O>() {
+ @Override
+ public O apply(Void input) throws Exception {
+ return callable.call();
+ }
+ }, null);
+ }
+
+ @Override
+ public void execute(final Subroutine subroutine) throws TransactionFailureException, InterruptedException {
+ execute(new Function<Void, Void>() {
+ @Override
+ public Void apply(Void input) throws Exception {
+ subroutine.apply();
+ return null;
+ }
+ }, null);
+ }
+
+ private <I, O> O executeWithRetry(Function<I, O> function, I input)
+ throws TransactionFailureException, InterruptedException {
+
+ int retries = 0;
+ while (true) {
+ try {
+ return executeOnce(function, input);
+ } catch (TransactionFailureException e) {
+ long delay = retryStrategy.nextRetry(e, ++retries);
+
+ if (delay < 0) {
+ throw e;
+ }
+
+ if (delay > 0) {
+ TimeUnit.MILLISECONDS.sleep(delay);
+ }
+ }
+ }
+
+ }
+
+ private <I, O> O executeOnce(Function<I, O> function, I input) throws TransactionFailureException {
+ TransactionContext txContext = new TransactionContext(txClient, txAwares);
+ txContext.start();
+ O o = null;
+ try {
+ o = function.apply(input);
+ } catch (Throwable e) {
+ txContext.abort(new TransactionFailureException("Transaction function failure for transaction. ", e));
+ // abort will throw
+ }
+ // will throw if smth goes wrong
+ txContext.finish();
+ return o;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/InvalidTruncateTimeException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/InvalidTruncateTimeException.java b/tephra-core/src/main/java/org/apache/tephra/InvalidTruncateTimeException.java
new file mode 100644
index 0000000..1cdbfb0
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/InvalidTruncateTimeException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Thrown when truncate invalid list is called with a time, and when there are in-progress transactions that
+ * were started before the given time.
+ */
+public class InvalidTruncateTimeException extends Exception {
+ public InvalidTruncateTimeException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/NoRetryStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/NoRetryStrategy.java b/tephra-core/src/main/java/org/apache/tephra/NoRetryStrategy.java
new file mode 100644
index 0000000..4dc245d
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/NoRetryStrategy.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Does no retries
+ */
+public class NoRetryStrategy implements RetryStrategy {
+ public static final RetryStrategy INSTANCE = new NoRetryStrategy();
+
+ private NoRetryStrategy() {}
+
+ @Override
+ public long nextRetry(TransactionFailureException reason, int failureCount) {
+ return -1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/RetryOnConflictStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/RetryOnConflictStrategy.java b/tephra-core/src/main/java/org/apache/tephra/RetryOnConflictStrategy.java
new file mode 100644
index 0000000..82f069f
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/RetryOnConflictStrategy.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Retries transaction execution when transaction fails with {@link TransactionConflictException}.
+ */
+public class RetryOnConflictStrategy implements RetryStrategy {
+ private final int maxRetries;
+ private final long retryDelay;
+
+ public RetryOnConflictStrategy(int maxRetries, long retryDelay) {
+ this.maxRetries = maxRetries;
+ this.retryDelay = retryDelay;
+ }
+
+ @Override
+ public long nextRetry(TransactionFailureException reason, int failureCount) {
+ if (reason instanceof TransactionConflictException) {
+ return failureCount > maxRetries ? -1 : retryDelay;
+ } else {
+ return -1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/RetryStrategies.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/RetryStrategies.java b/tephra-core/src/main/java/org/apache/tephra/RetryStrategies.java
new file mode 100644
index 0000000..9550d77
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/RetryStrategies.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Collection of {@link RetryStrategy}s.
+ */
+public final class RetryStrategies {
+ private RetryStrategies() {}
+
+ /**
+ * @param maxRetries max number of retries
+ * @param delayInMs delay between retries in milliseconds
+ * @return RetryStrategy that retries transaction execution when transaction fails with
+ * {@link TransactionConflictException}
+ */
+ public static RetryStrategy retryOnConflict(int maxRetries, long delayInMs) {
+ return new RetryOnConflictStrategy(maxRetries, delayInMs);
+ }
+
+ public static RetryStrategy noRetries() {
+ return NoRetryStrategy.INSTANCE;
+ }
+}
[15/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessorTest.java
deleted file mode 100644
index a5388ca..0000000
--- a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessorTest.java
+++ /dev/null
@@ -1,814 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98.coprocessor;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.HDFSTransactionStateStorage;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.snapshot.DefaultSnapshotCodec;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.executor.ExecutorService;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.ipc.RpcServerInterface;
-import org.apache.hadoop.hbase.master.TableLockManager;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
-import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
-import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
-import org.apache.hadoop.hbase.regionserver.FlushRequester;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
-import org.apache.hadoop.hbase.regionserver.Leases;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.zookeeper.KeeperException;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
- */
-public class TransactionProcessorTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
-
- // 8 versions, 1 hour apart, latest is current ts.
- private static final long[] V;
-
- static {
- long now = System.currentTimeMillis();
- V = new long[9];
- for (int i = 0; i < V.length; i++) {
- V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
- }
- }
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
- private static MiniDFSCluster dfsCluster;
- private static Configuration conf;
- private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
- private static TransactionVisibilityState txVisibilityState;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- Configuration hConf = new Configuration();
- String rootDir = tmpFolder.newFolder().getAbsolutePath();
- hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
- hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
-
- dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
- dfsCluster.waitActive();
- conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
-
- conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
- conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- String localTestDir = tmpFolder.newFolder().getAbsolutePath();
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
-
- // write an initial transaction snapshot
- TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(
- System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
- // this will set visibility upper bound to V[6]
- Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
- TransactionType.SHORT))),
- new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
- txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
- txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
- txSnapshot.getInProgress());
- HDFSTransactionStateStorage tmpStorage =
- new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
- tmpStorage.startAndWait();
- tmpStorage.writeSnapshot(txSnapshot);
- tmpStorage.stopAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- dfsCluster.shutdown();
- }
-
- @Test
- public void testDataJanitorRegionScanner() throws Exception {
- String tableName = "TestRegionScanner";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- for (int i = 1; i <= 8; i++) {
- for (int k = 1; k <= i; k++) {
- Put p = new Put(Bytes.toBytes(i));
- p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
- region.put(p);
- }
- }
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should only return the valid rows
- // do not use a filter here to test that cleanup works on flush
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
-
- // first returned value should be "4" with version "4"
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 4, new long[] {V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 5, new long[] {V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});
-
- results.clear();
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteFiltering() throws Exception {
- String tableName = "TestDeleteFiltering";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- byte[] row = Bytes.toBytes(1);
- for (int i = 4; i < V.length; i++) {
- Put p = new Put(row);
- p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
- region.put(p);
- }
-
- // delete from the third entry back
- // take that cell's timestamp + 1 to simulate a delete in a new tx
- long deleteTs = V[5] + 1;
- Delete d = new Delete(row, deleteTs);
- LOG.info("Issuing delete at timestamp " + deleteTs);
- // row deletes are not yet supported (TransactionAwareHTable normally handles this)
- d.deleteColumns(familyBytes, columnBytes);
- region.delete(d);
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, we should drop the deleted version, but not the others
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should return row with versions at: V[8], V[6].
- // V[7] is invalid and V[5] and prior are deleted.
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
- // should be only one row
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 1,
- new long[]{V[8], V[6], deleteTs},
- new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteMarkerCleanup() throws Exception {
- String tableName = "TestDeleteMarkerCleanup";
- byte[] familyBytes = Bytes.toBytes("f");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
-
- // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
- long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
- // deletes are performed after the writes, but still before the visibility upper bound
- long deleteTs = writeTs + 1;
- // write separate columns to confirm that delete markers survive across flushes
- byte[] row = Bytes.toBytes(100);
- Put p = new Put(row);
-
- LOG.info("Writing columns at timestamp " + writeTs);
- for (int i = 0; i < 5; i++) {
- byte[] iBytes = Bytes.toBytes(i);
- p.add(familyBytes, iBytes, writeTs, iBytes);
- }
- region.put(p);
- // read all back
- Scan scan = new Scan(row);
- RegionScanner regionScanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- for (int i = 0; i < 5; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a flush to clear the memstore
- LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // delete the odd entries
- for (int i = 0; i < 5; i++) {
- if (i % 2 == 1) {
- // deletes are performed as puts with empty values
- Put deletePut = new Put(row);
- deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
- region.put(deletePut);
- }
- }
-
- // read all back
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- LOG.info("Got cell " + cell);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force another flush on the delete markers
- // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
- LOG.info("After delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a major compaction
- LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
- region.compactStores(true);
-
- // perform a raw scan (no filter) to confirm that the delete markers are now gone
- scan = new Scan(row);
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
- } finally {
- region.close();
- }
- }
-
- /**
- * Test that we correctly preserve the timestamp set for column family delete markers. This is not
- * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
- * that we make it easy to interoperate with other systems.
- */
- @Test
- public void testFamilyDeleteTimestamp() throws Exception {
- String tableName = "TestFamilyDeleteTimestamp";
- byte[] family1Bytes = Bytes.toBytes("f1");
- byte[] columnBytes = Bytes.toBytes("c");
- byte[] rowBytes = Bytes.toBytes("row");
- byte[] valBytes = Bytes.toBytes("val");
- HRegion region = createRegion(tableName, family1Bytes, 0);
- try {
- region.initialize();
-
- long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
- Put p = new Put(rowBytes);
- p.add(family1Bytes, columnBytes, now - 10, valBytes);
- region.put(p);
-
- // issue a family delete with an explicit timestamp
- Delete delete = new Delete(rowBytes, now);
- delete.deleteFamily(family1Bytes, now - 5);
- region.delete(delete);
-
- // test that the delete marker preserved the timestamp
- Scan scan = new Scan();
- scan.setMaxVersions();
- RegionScanner scanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(2, results.size());
- // delete marker should appear first
- Cell cell = results.get(0);
- assertArrayEquals(new byte[0], cell.getQualifier());
- assertArrayEquals(new byte[0], cell.getValue());
- assertEquals(now - 5, cell.getTimestamp());
- // since this is an unfiltered scan against the region, the original put should be next
- cell = results.get(1);
- assertArrayEquals(valBytes, cell.getValue());
- assertEquals(now - 10, cell.getTimestamp());
- scanner.close();
-
-
- // with a filtered scan the original put should disappear
- scan = new Scan();
- scan.setMaxVersions();
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- scanner = region.getScanner(scan);
- results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(0, results.size());
- scanner.close();
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testPreExistingData() throws Exception {
- String tableName = "TestPreExistingData";
- byte[] familyBytes = Bytes.toBytes("f");
- long ttlMillis = TimeUnit.DAYS.toMillis(14);
- HRegion region = createRegion(tableName, familyBytes, ttlMillis);
- try {
- region.initialize();
-
- // timestamps for pre-existing, non-transactional data
- long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
- long older = now - ttlMillis / 2;
- long newer = now - ttlMillis / 3;
- // timestamps for transactional data
- long nowTx = txVisibilityState.getVisibilityUpperBound();
- long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
- long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(familyBytes, ttlMillis);
-
- List<Cell> cells = new ArrayList<>();
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
-
- // Write non-transactional and transactional data
- for (Cell c : cells) {
- region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
- }
-
- Scan rawScan = new Scan();
- rawScan.setMaxVersions();
-
- Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
- Scan txScan = new Scan();
- txScan.setMaxVersions();
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // read all back with raw scanner
- scanAndAssert(region, cells, rawScan);
-
- // read all back with transaction filter
- scanAndAssert(region, cells, txScan);
-
- // force a flush to clear the memstore
- region.flushcache();
- scanAndAssert(region, cells, txScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- scanAndAssert(region, cells, txScan);
-
- // Reduce TTL, this should make cells with timestamps older and olderTx expire
- long newTtl = ttlMillis / 2 - 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // Raw scan should still give all cells
- scanAndAssert(region, cells, rawScan);
- // However, tx scan should not return expired cells
- scanAndAssert(region, select(cells, 1, 3, 5), txScan);
-
- region.flushcache();
- scanAndAssert(region, cells, rawScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- // This time raw scan too should not return expired cells, as they would be dropped during major compaction
- scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
-
- // Reduce TTL again to 1 ms, this should expire all cells
- newTtl = 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // force a major compaction to remove expired cells
- region.compactStores(true);
- // This time raw scan should not return any cells, as all cells have expired.
- scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
- } finally {
- region.close();
- }
- }
-
- private List<Cell> select(List<Cell> cells, int... indexes) {
- List<Cell> newCells = new ArrayList<>();
- for (int i : indexes) {
- newCells.add(cells.get(i));
- }
- return newCells;
- }
-
- @SuppressWarnings("StatementWithEmptyBody")
- private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
- try (RegionScanner regionScanner = region.getScanner(scan)) {
- List<Cell> results = Lists.newArrayList();
- while (regionScanner.next(results)) { }
- assertEquals(expected, results);
- }
- }
-
- private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
- region.close();
- HTableDescriptor htd = region.getTableDesc();
- HColumnDescriptor cfd = new HColumnDescriptor(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- htd.addFamily(cfd);
- return HRegion.openHRegion(region.getRegionInfo(), htd, region.getLog(), conf,
- new MockRegionServerServices(conf, null), null);
- }
-
- private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
- HColumnDescriptor cfd = new HColumnDescriptor(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- htd.addFamily(cfd);
- htd.addCoprocessor(TransactionProcessor.class.getName());
- Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
- Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
- FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.mkdirs(tablePath));
- HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
- HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
- HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
- return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
- byte[][] values = new byte[versions.length][];
- for (int i = 0; i < versions.length; i++) {
- values[i] = Bytes.toBytes(versions[i]);
- }
- assertKeyValueMatches(results, index, versions, values);
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
- assertEquals(versions.length, results.size());
- assertEquals(values.length, results.size());
- for (int i = 0; i < versions.length; i++) {
- Cell kv = results.get(i);
- assertArrayEquals(Bytes.toBytes(index), kv.getRow());
- assertEquals(versions[i], kv.getTimestamp());
- assertArrayEquals(values[i], kv.getValue());
- }
- }
-
- @Test
- public void testTransactionStateCache() throws Exception {
- TransactionStateCache cache = new TransactionStateCache();
- cache.setConf(conf);
- cache.startAndWait();
- // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
- TransactionVisibilityState cachedSnapshot = cache.getLatestState();
- assertNotNull(cachedSnapshot);
- assertEquals(invalidSet, cachedSnapshot.getInvalid());
- cache.stopAndWait();
- }
-
- private static class MockRegionServerServices implements RegionServerServices {
- private final Configuration hConf;
- private final ZooKeeperWatcher zookeeper;
- private final Map<String, HRegion> regions = new HashMap<>();
- private boolean stopping = false;
- private final ConcurrentSkipListMap<byte[], Boolean> rit = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
- private HFileSystem hfs = null;
- private ServerName serverName = null;
- private RpcServerInterface rpcServer = null;
- private volatile boolean abortRequested;
-
-
- public MockRegionServerServices(Configuration hConf, ZooKeeperWatcher zookeeper) {
- this.hConf = hConf;
- this.zookeeper = zookeeper;
- }
-
- @Override
- public boolean isStopping() {
- return stopping;
- }
-
- @Override
- public HLog getWAL(HRegionInfo regionInfo) throws IOException {
- return null;
- }
-
- @Override
- public CompactionRequestor getCompactionRequester() {
- return null;
- }
-
- @Override
- public double getCompactionPressure() {
- return 0;
- }
-
- @Override
- public FlushRequester getFlushRequester() {
- return null;
- }
-
- @Override
- public RegionServerAccounting getRegionServerAccounting() {
- return null;
- }
-
- @Override
- public TableLockManager getTableLockManager() {
- return new TableLockManager.NullTableLockManager();
- }
-
- @Override
- public void postOpenDeployTasks(HRegion r, CatalogTracker ct) throws KeeperException, IOException {
- }
-
- @Override
- public RpcServerInterface getRpcServer() {
- return rpcServer;
- }
-
- @Override
- public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
- return rit;
- }
-
- @Override
- public FileSystem getFileSystem() {
- return hfs;
- }
-
- @Override
- public Leases getLeases() {
- return null;
- }
-
- @Override
- public ExecutorService getExecutorService() {
- return null;
- }
-
- @Override
- public CatalogTracker getCatalogTracker() {
- return null;
- }
-
- @Override
- public Map<String, HRegion> getRecoveringRegions() {
- return null;
- }
-
- @Override
- public void updateRegionFavoredNodesMapping(String encodedRegionName, List<HBaseProtos.ServerName> favoredNodes) {
- }
-
- @Override
- public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
- return new InetSocketAddress[0];
- }
-
- @Override
- public void addToOnlineRegions(HRegion r) {
- regions.put(r.getRegionNameAsString(), r);
- }
-
- @Override
- public boolean removeFromOnlineRegions(HRegion r, ServerName destination) {
- return regions.remove(r.getRegionInfo().getEncodedName()) != null;
- }
-
- @Override
- public HRegion getFromOnlineRegions(String encodedRegionName) {
- return regions.get(encodedRegionName);
- }
-
- @Override
- public List<HRegion> getOnlineRegions(TableName tableName) throws IOException {
- return null;
- }
-
- @Override
- public Configuration getConfiguration() {
- return hConf;
- }
-
- @Override
- public ZooKeeperWatcher getZooKeeper() {
- return zookeeper;
- }
-
- @Override
- public ServerName getServerName() {
- return serverName;
- }
-
- @Override
- public void abort(String why, Throwable e) {
- this.abortRequested = true;
- }
-
- @Override
- public boolean isAborted() {
- return abortRequested;
- }
-
- @Override
- public void stop(String why) {
- this.stopping = true;
- }
-
- @Override
- public boolean isStopped() {
- return stopping;
- }
-
- @Override
- public int getPriority(RequestHeader header, Message param) {
- return 0;
- }
-
- @Override
- public ServerNonceManager getNonceManager() {
- return null;
- }
-
- @Override
- public boolean reportRegionStateTransition(
- RegionServerStatusProtos.RegionStateTransition.TransitionCode transitionCode,
- long l, HRegionInfo... hRegionInfos) {
- return false;
- }
-
- @Override
- public boolean reportRegionStateTransition(
- RegionServerStatusProtos.RegionStateTransition.TransitionCode transitionCode,
- HRegionInfo... hRegionInfos) {
- return false;
- }
-
- @Override
- public boolean registerService(Service service) {
- return false;
- }
-
- @Override
- public HeapMemoryManager getHeapMemoryManager() {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java
deleted file mode 100644
index 0a93d16..0000000
--- a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase.AbstractTransactionVisibilityFilterTest;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * HBase 0.98 specific test for filtering logic applied when reading data transactionally.
- */
-public class TransactionVisibilityFilterTest extends AbstractTransactionVisibilityFilterTest {
- /**
- * Test filtering of KeyValues for in-progress and invalid transactions.
- * @throws Exception
- */
- @Test
- public void testFiltering() throws Exception {
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
- }
-
- @Test
- public void testSubFilter() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- @Test
- public void testSubFilterOverride() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- private void runFilteringTest(TxFilterFactory txFilterFactory,
- List<Filter.ReturnCode> assertCodes) throws Exception {
- /*
- * Start and stop some transactions. This will give us a transaction state something like the following
- * (numbers only reflect ordering, not actual transaction IDs):
- * 6 - in progress
- * 5 - committed
- * 4 - invalid
- * 3 - in-progress
- * 2 - committed
- * 1 - committed
- *
- * read ptr = 5
- * write ptr = 6
- */
-
- Transaction tx1 = txManager.startShort();
- assertTrue(txManager.canCommit(tx1, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx1));
-
- Transaction tx2 = txManager.startShort();
- assertTrue(txManager.canCommit(tx2, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx2));
-
- Transaction tx3 = txManager.startShort();
- Transaction tx4 = txManager.startShort();
- txManager.invalidate(tx4.getTransactionId());
-
- Transaction tx5 = txManager.startShort();
- assertTrue(txManager.canCommit(tx5, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx5));
-
- Transaction tx6 = txManager.startShort();
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- Filter filter = txFilterFactory.getTxFilter(tx6, ttls);
-
- assertEquals(assertCodes.get(5),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx6.getTransactionId())));
- assertEquals(assertCodes.get(4),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx5.getTransactionId())));
- assertEquals(assertCodes.get(3),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx4.getTransactionId())));
- assertEquals(assertCodes.get(2),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx3.getTransactionId())));
- assertEquals(assertCodes.get(1),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx2.getTransactionId())));
- assertEquals(assertCodes.get(0),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx1.getTransactionId())));
- }
-
- /**
- * Test filtering for TTL settings.
- * @throws Exception
- */
- @Test
- public void testTTLFiltering() throws Exception {
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(FAM, 10L);
- ttls.put(FAM2, 30L);
- ttls.put(FAM3, 0L);
-
- Transaction tx = txManager.startShort();
- long now = tx.getVisibilityUpperBound();
- Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 21 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 1001 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
-
- // Verify ttl for pre-existing, non-transactional data
- long preNow = now / TxConstants.MAX_TX_PER_MS;
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 11L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 1001L)));
- }
-
- protected KeyValue newKeyValue(String rowkey, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), FAM, COL, timestamp, Bytes.toBytes(value));
- }
-
- protected KeyValue newKeyValue(String rowkey, byte[] family, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), family, COL, timestamp, Bytes.toBytes(value));
- }
-
- private interface TxFilterFactory {
- Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs);
- }
-
- private class CustomTxFilter extends TransactionVisibilityFilter {
- public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
- @Nullable Filter cellFilter) {
- super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
- }
-
- @Override
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- switch (subFilterCode) {
- case INCLUDE:
- return ReturnCode.INCLUDE;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/HBase98ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/HBase98ConfigurationProviderTest.java b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/HBase98ConfigurationProviderTest.java
new file mode 100644
index 0000000..aa8ae33
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/HBase98ConfigurationProviderTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98;
+
+import org.apache.tephra.util.AbstractConfigurationProviderTest;
+import org.apache.tephra.util.HBaseVersion;
+
+/**
+ * Test for HBase 0.98 version specific behavior.
+ */
+public class HBase98ConfigurationProviderTest extends AbstractConfigurationProviderTest {
+ @Override
+ protected HBaseVersion.Version getExpectedVersion() {
+ return HBaseVersion.Version.HBASE_98;
+ }
+}
[42/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/rpc/ThriftRPCServer.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/rpc/ThriftRPCServer.java b/tephra-core/src/main/java/co/cask/tephra/rpc/ThriftRPCServer.java
deleted file mode 100644
index 432ab2f..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/rpc/ThriftRPCServer.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.rpc;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.AbstractExecutionThreadService;
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TThreadedSelectorServerWithFix;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TNonblockingServerSocket;
-import org.apache.twill.common.Threads;
-import org.apache.twill.internal.utils.Networks;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.net.InetSocketAddress;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @param <T> The type of service handler interface.
- * @param <I> The type of the thrift service.
- */
-public final class ThriftRPCServer<T extends RPCServiceHandler, I> extends AbstractExecutionThreadService {
-
- private static final Logger LOG = LoggerFactory.getLogger(ThriftRPCServer.class);
-
- private final String name;
- private final int ioThreads;
- private final int workerThreads;
- private final int maxReadBufferBytes;
- private final T serviceHandler;
- private final TProcessor processor;
-
- private InetSocketAddress bindAddress;
- private ExecutorService executor;
- private TServer server;
-
- /**
- * Creates a {@link Builder} for creating instance of {@link ThriftRPCServer}.
- * @param serviceType Class of the thrift service.
- * @param <I> Type of the thrift service.
- * @return A {@link Builder}.
- */
- public static <I> Builder<I> builder(Class<I> serviceType) {
- return new Builder<I>(serviceType);
- }
-
- /**
- * Builder for creating instance of ThriftRPCServer. By default, the instance created will bind to
- * random port and with 2 io threads and worker threads equals to min(2, number of cpu cores - 2).
- */
- public static final class Builder<I> {
- private final Class<I> serviceType;
- private String name;
- private InetSocketAddress bindAddress = new InetSocketAddress(0);
- private int ioThreads = 2;
- private int workerThreads = Runtime.getRuntime().availableProcessors() - 2;
- // 16Mb
- private int maxReadBufferBytes = 16 * 1024 * 1024;
-
- private Builder(Class<I> serviceType) {
- this.serviceType = serviceType;
- this.name = serviceType.getSimpleName();
- }
-
- public Builder<I> setName(String name) {
- this.name = name;
- return this;
- }
-
- public Builder<I> setHost(String host) {
- this.bindAddress = new InetSocketAddress(host, bindAddress.getPort());
- return this;
- }
-
- public Builder<I> setPort(int port) {
- this.bindAddress = new InetSocketAddress(bindAddress.getHostName(), port);
- return this;
- }
-
- public Builder<I> setIOThreads(int count) {
- this.ioThreads = count;
- return this;
- }
-
- public Builder<I> setWorkerThreads(int count) {
- this.workerThreads = count;
- return this;
- }
-
- public Builder<I> setMaxReadBufferBytes(int maxReadBufferBytes) {
- this.maxReadBufferBytes = maxReadBufferBytes;
- return this;
- }
-
- public <T extends RPCServiceHandler> ThriftRPCServer<T, I> build(T serviceHandler) {
- return new ThriftRPCServer<T, I>(bindAddress, ioThreads, workerThreads, maxReadBufferBytes,
- serviceHandler, serviceType, name);
- }
- }
-
- /**
- * Creates a ThriftRPCServer with the given paramters.
- *
- * @param bindAddress The socket address for the server to listen on. If {@code null}, it'll be binded to random
- * port on localhost.
- * @param ioThreads Number of io threads.
- * @param workerThreads Number of worker threads.
- * @param serviceHandler Handler for handling client requests.
- */
- @SuppressWarnings("unchecked")
- private ThriftRPCServer(InetSocketAddress bindAddress, int ioThreads,
- int workerThreads, int maxReadBufferBytes,
- T serviceHandler, Class<I> serviceType, String name) {
- Preconditions.checkArgument(ioThreads > 0, "IO threads must be > 0.");
- Preconditions.checkArgument(workerThreads > 0, "Worker threads must be > 0.");
-
- this.bindAddress = bindAddress;
- this.ioThreads = ioThreads;
- this.workerThreads = workerThreads;
- this.maxReadBufferBytes = maxReadBufferBytes;
- this.serviceHandler = serviceHandler;
- this.name = name;
- this.processor = createProcessor((Class<T>) serviceHandler.getClass(), serviceType);
- }
-
- public InetSocketAddress getBindAddress() {
- return bindAddress;
- }
-
- @Override
- protected void startUp() throws Exception {
- // Determines the address and port to listen on
- InetSocketAddress listenOn = bindAddress;
- if (listenOn == null || listenOn.getPort() <= 0) {
- int port = Networks.getRandomPort();
- if (listenOn == null) {
- listenOn = new InetSocketAddress("localhost", port);
- } else {
- listenOn = new InetSocketAddress(listenOn.getAddress(), port);
- }
- }
- bindAddress = listenOn;
-
- executor = new ThreadPoolExecutor(0, workerThreads,
- 60L, TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(),
- Threads.createDaemonThreadFactory(String.format("%s-rpc-%%d", name)),
- new ThreadPoolExecutor.CallerRunsPolicy());
- serviceHandler.init();
-
- TThreadedSelectorServerWithFix.Args args =
- new TThreadedSelectorServerWithFix.Args(new TNonblockingServerSocket(listenOn))
- .selectorThreads(ioThreads)
- .protocolFactory(new TBinaryProtocol.Factory())
- .transportFactory(new TFramedTransport.Factory())
- .processor(processor)
- .executorService(executor);
-
- // ENG-443 - Set the max read buffer size. This is important as this will
- // prevent the server from throwing OOME if telnetd to the port
- // it's running on.
- args.maxReadBufferBytes = maxReadBufferBytes;
- server = new TThreadedSelectorServerWithFix(args);
- LOG.info("Starting RPC server for {}", name);
- }
-
- @Override
- protected void shutDown() throws Exception {
- serviceHandler.destroy();
- executor.shutdownNow();
- LOG.info("RPC server for {} stopped.", name);
- }
-
- @Override
- protected void triggerShutdown() {
- LOG.info("Request to stop RPC server for {}", name);
- server.stop();
- }
-
- @Override
- protected void run() throws Exception {
- LOG.info("Running RPC server for {}", name);
- server.serve();
- LOG.info("Done running RPC server for {}", name);
- }
-
- @SuppressWarnings("unchecked")
- private TProcessor createProcessor(final Class<T> handlerType, Class<I> serviceType) {
- // Pick the Iface inner interface and the Processor class
- Class<? extends TProcessor> processorType = null;
- Class<?> ifaceType = null;
- for (Class<?> clz : serviceType.getDeclaredClasses()) {
- if (TProcessor.class.isAssignableFrom(clz)) {
- processorType = (Class<? extends TProcessor>) clz;
- } else if (clz.isInterface() && "Iface".equals(clz.getSimpleName())) {
- ifaceType = clz;
- }
- }
-
- Preconditions.checkArgument(processorType != null,
- "Missing TProcessor, %s is not a valid thrift service.", serviceType.getName());
- Preconditions.checkArgument(ifaceType != null,
- "Missing Iface, %s is not a valid thrift service.", serviceType.getName());
-
- // If handler already implements the Iface, simply delegate
- if (ifaceType.isAssignableFrom(handlerType)) {
- return createProxyProcessor(handlerType, processorType, ifaceType);
- }
-
- throw new IllegalArgumentException("Unsupported handler type.");
- }
-
- private TProcessor createProxyProcessor(final Class<T> handlerType,
- Class<? extends TProcessor> processorType, Class<?> ifaceType) {
-
- try {
- // Map from Iface method to handlerType method to save reflection lookup
- ImmutableMap.Builder<Method, Method> builder = ImmutableMap.builder();
- for (Method method : ifaceType.getMethods()) {
- Method handlerMethod = handlerType.getMethod(method.getName(), method.getParameterTypes());
- if (!handlerMethod.isAccessible()) {
- handlerMethod.setAccessible(true);
- }
- builder.put(method, handlerMethod);
- }
- final Map<Method, Method> methods = builder.build();
-
- Object proxy = Proxy.newProxyInstance(ifaceType.getClassLoader(),
- new Class[]{ifaceType}, new InvocationHandler() {
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
- try {
- return methods.get(method).invoke(serviceHandler, args);
- } catch (InvocationTargetException e) {
- if (e.getCause() != null) {
- throw e.getCause();
- } else {
- throw e;
- }
- }
- }
- });
-
- return processorType.getConstructor(ifaceType).newInstance(proxy);
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/rpc/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/rpc/package-info.java b/tephra-core/src/main/java/co/cask/tephra/rpc/package-info.java
deleted file mode 100644
index 2b5bccd..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/rpc/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * This package contains class for writing RPC server and client in simple manner.
- */
-package co.cask.tephra.rpc;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/ConfigModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/ConfigModule.java b/tephra-core/src/main/java/co/cask/tephra/runtime/ConfigModule.java
deleted file mode 100644
index 92df8cd..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/ConfigModule.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import com.google.inject.AbstractModule;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Provides Guice bindings for {@link Configuration}.
- */
-public final class ConfigModule extends AbstractModule {
-
- private final Configuration configuration;
-
- public ConfigModule(Configuration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- protected void configure() {
- bind(Configuration.class).toInstance(configuration);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/DiscoveryModules.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/DiscoveryModules.java b/tephra-core/src/main/java/co/cask/tephra/runtime/DiscoveryModules.java
deleted file mode 100644
index 76cbbdf..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/DiscoveryModules.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Module;
-import com.google.inject.PrivateModule;
-import com.google.inject.Provides;
-import com.google.inject.Singleton;
-import org.apache.twill.common.Cancellable;
-import org.apache.twill.discovery.Discoverable;
-import org.apache.twill.discovery.DiscoveryService;
-import org.apache.twill.discovery.DiscoveryServiceClient;
-import org.apache.twill.discovery.InMemoryDiscoveryService;
-import org.apache.twill.discovery.ServiceDiscovered;
-import org.apache.twill.discovery.ZKDiscoveryService;
-import org.apache.twill.zookeeper.ZKClientService;
-
-/**
- * Provides access to Google Guice modules for in-memory, single-node, and distributed operation for
- * {@link DiscoveryService} and {@link DiscoveryServiceClient}.
- */
-public final class DiscoveryModules {
-
- public Module getInMemoryModules() {
- return new InMemoryDiscoveryModule();
- }
-
- public Module getSingleNodeModules() {
- return new InMemoryDiscoveryModule();
- }
-
- public Module getDistributedModules() {
- return new ZKDiscoveryModule();
- }
-
- private static final class InMemoryDiscoveryModule extends AbstractModule {
-
- // ensuring to be singleton across JVM
- private static final InMemoryDiscoveryService IN_MEMORY_DISCOVERY_SERVICE = new InMemoryDiscoveryService();
-
- @Override
- protected void configure() {
- InMemoryDiscoveryService discovery = IN_MEMORY_DISCOVERY_SERVICE;
- bind(DiscoveryService.class).toInstance(discovery);
- bind(DiscoveryServiceClient.class).toInstance(discovery);
- }
- }
-
- private static final class ZKDiscoveryModule extends PrivateModule {
-
- @Override
- protected void configure() {
- expose(DiscoveryService.class);
- expose(DiscoveryServiceClient.class);
- }
-
- @Provides
- @Singleton
- private ZKDiscoveryService providesZKDiscoveryService(ZKClientService zkClient) {
- return new ZKDiscoveryService(zkClient);
- }
-
- @Provides
- @Singleton
- private DiscoveryService providesDiscoveryService(final ZKClientService zkClient,
- final ZKDiscoveryService delegate) {
- return new DiscoveryService() {
- @Override
- public Cancellable register(Discoverable discoverable) {
- if (!zkClient.isRunning()) {
- zkClient.startAndWait();
- }
- return delegate.register(discoverable);
- }
- };
- }
-
- @Provides
- @Singleton
- private DiscoveryServiceClient providesDiscoveryServiceClient(final ZKClientService zkClient,
- final ZKDiscoveryService delegate) {
- return new DiscoveryServiceClient() {
- @Override
- public ServiceDiscovered discover(String s) {
- if (!zkClient.isRunning()) {
- zkClient.startAndWait();
- }
- return delegate.discover(s);
- }
- };
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionClientModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionClientModule.java b/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionClientModule.java
deleted file mode 100644
index c6df2b3..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionClientModule.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.distributed.PooledClientProvider;
-import co.cask.tephra.distributed.ThreadLocalClientProvider;
-import co.cask.tephra.distributed.ThriftClientProvider;
-import com.google.inject.AbstractModule;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.discovery.DiscoveryServiceClient;
-
-/**
- * Provides Guice binding for {@link co.cask.tephra.distributed.ThriftClientProvider}.
- */
-public class TransactionClientModule extends AbstractModule {
-
- @Override
- protected void configure() {
- bind(ThriftClientProvider.class).toProvider(ThriftClientProviderSupplier.class);
- }
-
- /**
- * Provides implementation of {@link co.cask.tephra.distributed.ThriftClientProvider}
- * based on configuration.
- */
- @Singleton
- private static final class ThriftClientProviderSupplier implements Provider<ThriftClientProvider> {
-
- private final Configuration cConf;
- private DiscoveryServiceClient discoveryServiceClient;
-
- @Inject
- ThriftClientProviderSupplier(Configuration cConf) {
- this.cConf = cConf;
- }
-
- @Inject(optional = true)
- void setDiscoveryServiceClient(DiscoveryServiceClient discoveryServiceClient) {
- this.discoveryServiceClient = discoveryServiceClient;
- }
-
- @Override
- public ThriftClientProvider get() {
- // configure the client provider
- String provider = cConf.get(TxConstants.Service.CFG_DATA_TX_CLIENT_PROVIDER,
- TxConstants.Service.DEFAULT_DATA_TX_CLIENT_PROVIDER);
- ThriftClientProvider clientProvider;
- if ("pool".equals(provider)) {
- clientProvider = new PooledClientProvider(cConf, discoveryServiceClient);
- } else if ("thread-local".equals(provider)) {
- clientProvider = new ThreadLocalClientProvider(cConf, discoveryServiceClient);
- } else {
- String message = "Unknown Transaction Service Client Provider '" + provider + "'.";
- throw new IllegalArgumentException(message);
- }
- return clientProvider;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionDistributedModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionDistributedModule.java b/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionDistributedModule.java
deleted file mode 100644
index c88f742..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionDistributedModule.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import co.cask.tephra.DefaultTransactionExecutor;
-import co.cask.tephra.TransactionExecutor;
-import co.cask.tephra.TransactionExecutorFactory;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import co.cask.tephra.metrics.DefaultMetricsCollector;
-import co.cask.tephra.metrics.MetricsCollector;
-import co.cask.tephra.persist.HDFSTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import com.google.inject.AbstractModule;
-import com.google.inject.Singleton;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import com.google.inject.name.Names;
-
-/**
- * Guice bindings for running in distributed mode on a cluster.
- */
-final class TransactionDistributedModule extends AbstractModule {
-
- @Override
- protected void configure() {
- bind(SnapshotCodecProvider.class).in(Singleton.class);
- bind(TransactionStateStorage.class).annotatedWith(Names.named("persist"))
- .to(HDFSTransactionStateStorage.class).in(Singleton.class);
- bind(TransactionStateStorage.class).toProvider(TransactionStateStorageProvider.class).in(Singleton.class);
-
- bind(TransactionManager.class).in(Singleton.class);
- bind(TransactionSystemClient.class).to(TransactionServiceClient.class).in(Singleton.class);
- bind(MetricsCollector.class).to(DefaultMetricsCollector.class).in(Singleton.class);
-
- install(new FactoryModuleBuilder()
- .implement(TransactionExecutor.class, DefaultTransactionExecutor.class)
- .build(TransactionExecutorFactory.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionInMemoryModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionInMemoryModule.java b/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionInMemoryModule.java
deleted file mode 100644
index d9780e1..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionInMemoryModule.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import co.cask.tephra.DefaultTransactionExecutor;
-import co.cask.tephra.TransactionExecutor;
-import co.cask.tephra.TransactionExecutorFactory;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.metrics.MetricsCollector;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.NoOpTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Singleton;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-
-/**
- * Guice bindings for running completely in-memory (no persistence). This should only be used for
- * test classes, as the transaction state cannot be recovered in the case of a failure.
- */
-public class TransactionInMemoryModule extends AbstractModule {
- public TransactionInMemoryModule() {
- }
-
- @Override
- protected void configure() {
- bind(SnapshotCodecProvider.class).in(Singleton.class);
- bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class).in(Singleton.class);
- bind(TransactionManager.class).in(Singleton.class);
- bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
- // no metrics output for in-memory
- bind(MetricsCollector.class).to(TxMetricsCollector.class);
-
- install(new FactoryModuleBuilder()
- .implement(TransactionExecutor.class, DefaultTransactionExecutor.class)
- .build(TransactionExecutorFactory.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionLocalModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionLocalModule.java b/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionLocalModule.java
deleted file mode 100644
index 720c84c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionLocalModule.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import co.cask.tephra.DefaultTransactionExecutor;
-import co.cask.tephra.TransactionExecutor;
-import co.cask.tephra.TransactionExecutorFactory;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.metrics.DefaultMetricsCollector;
-import co.cask.tephra.metrics.MetricsCollector;
-import co.cask.tephra.persist.LocalFileTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import com.google.inject.AbstractModule;
-import com.google.inject.Singleton;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import com.google.inject.name.Names;
-
-/**
- * Guice bindings for running in single-node mode (persistence to local disk and in-memory client).
- */
-final class TransactionLocalModule extends AbstractModule {
-
- @Override
- protected void configure() {
- bind(SnapshotCodecProvider.class).in(Singleton.class);
- bind(TransactionStateStorage.class).annotatedWith(Names.named("persist"))
- .to(LocalFileTransactionStateStorage.class).in(Singleton.class);
- bind(TransactionStateStorage.class).toProvider(TransactionStateStorageProvider.class).in(Singleton.class);
-
- bind(TransactionManager.class).in(Singleton.class);
- bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
- bind(MetricsCollector.class).to(DefaultMetricsCollector.class);
-
- install(new FactoryModuleBuilder()
- .implement(TransactionExecutor.class, DefaultTransactionExecutor.class)
- .build(TransactionExecutorFactory.class));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionModules.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionModules.java b/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionModules.java
deleted file mode 100644
index 551fb3a..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionModules.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import com.google.inject.Module;
-
-/**
- * Provides access to Google Guice modules for in-memory, single-node, and distributed operation.
- */
-public class TransactionModules {
- public TransactionModules() {
- }
-
- public Module getInMemoryModules() {
- return new TransactionInMemoryModule();
- }
-
- public Module getSingleNodeModules() {
- return new TransactionLocalModule();
- }
-
- public Module getDistributedModules() {
- return new TransactionDistributedModule();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionStateStorageProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionStateStorageProvider.java b/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionStateStorageProvider.java
deleted file mode 100644
index 777354c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/TransactionStateStorageProvider.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.runtime;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.persist.NoOpTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-import com.google.inject.name.Names;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A provider for {@link TransactionStateStorage} that provides different
- * {@link TransactionStateStorage} implementation based on configuration.
- */
-@Singleton
-public final class TransactionStateStorageProvider implements Provider<TransactionStateStorage> {
-
- private final Configuration cConf;
- private final Injector injector;
-
- @Inject
- TransactionStateStorageProvider(Configuration cConf, Injector injector) {
- this.cConf = cConf;
- this.injector = injector;
- }
-
- @Override
- public TransactionStateStorage get() {
- if (cConf.getBoolean(TxConstants.Manager.CFG_DO_PERSIST, true)) {
- return injector.getInstance(Key.get(TransactionStateStorage.class, Names.named("persist")));
- } else {
- return injector.getInstance(NoOpTransactionStateStorage.class);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/runtime/ZKModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/runtime/ZKModule.java b/tephra-core/src/main/java/co/cask/tephra/runtime/ZKModule.java
deleted file mode 100644
index ebc1df7..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/runtime/ZKModule.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.runtime;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.zookeeper.TephraZKClientService;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.google.inject.Singleton;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.zookeeper.RetryStrategies;
-import org.apache.twill.zookeeper.ZKClient;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.apache.twill.zookeeper.ZKClientServices;
-import org.apache.twill.zookeeper.ZKClients;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Provides Guice binding to {@link ZKClient} and {@link ZKClientService}.
- */
-public class ZKModule extends AbstractModule {
-
- @Override
- protected void configure() {
- /**
- * ZKClientService is provided by the provider method
- * {@link #provideZKClientService(org.apache.hadoop.conf.Configuration)}.
- */
- bind(ZKClient.class).to(ZKClientService.class);
- }
-
- @Provides
- @Singleton
- private ZKClientService provideZKClientService(Configuration conf) {
- String zkStr = conf.get(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM);
- if (zkStr == null) {
- // Default to HBase one.
- zkStr = conf.get(TxConstants.HBase.ZOOKEEPER_QUORUM);
- }
-
- int timeOut = conf.getInt(TxConstants.HBase.ZK_SESSION_TIMEOUT, TxConstants.HBase.DEFAULT_ZK_SESSION_TIMEOUT);
- ZKClientService zkClientService = new TephraZKClientService(zkStr, timeOut, null,
- ArrayListMultimap.<String, byte[]>create());
- return ZKClientServices.delegate(
- ZKClients.reWatchOnExpire(
- ZKClients.retryOnFailure(zkClientService, RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
- )
- )
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryDecoder.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryDecoder.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryDecoder.java
deleted file mode 100644
index 8203494..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryDecoder.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * An decoder to help read snapshots in binary format.
- */
-public final class BinaryDecoder {
-
- private final InputStream input;
-
- /**
- * @param input Stream to read from.
- */
- public BinaryDecoder(InputStream input) {
- this.input = input;
- }
-
- /**
- * Read one int from the input.
- * @return the read number
- * @throws java.io.IOException If there is IO error.
- * @throws java.io.EOFException If end of file reached.
- */
- public int readInt() throws IOException {
- int val = 0;
- int shift = 0;
- int b = readByte();
- while (b > 0x7f) {
- val ^= (b & 0x7f) << shift;
- shift += 7;
- b = readByte();
- }
- val ^= b << shift;
- return (val >>> 1) ^ -(val & 1);
- }
-
- /**
- * Read one long int from the input.
- * @return the read number
- * @throws java.io.IOException If there is IO error.
- * @throws java.io.EOFException If end of file reached.
- */
- public long readLong() throws IOException {
- long val = 0;
- int shift = 0;
- int b = readByte();
- while (b > 0x7f) {
- val ^= (long) (b & 0x7f) << shift;
- shift += 7;
- b = readByte();
- }
- val ^= (long) b << shift;
- return (val >>> 1) ^ -(val & 1);
- }
-
- /**
- * Read a byte sequence. First read an int to indicate how many bytes to read, then that many bytes.
- * @return the read bytes as a byte array
- * @throws java.io.IOException If there is IO error.
- * @throws java.io.EOFException If end of file reached.
- */
- public byte[] readBytes() throws IOException {
- int toRead = readInt();
- byte[] bytes = new byte[toRead];
- while (toRead > 0) {
- int byteRead = input.read(bytes, bytes.length - toRead, toRead);
- if (byteRead == -1) {
- throw new EOFException();
- }
- toRead -= byteRead;
- }
- return bytes;
- }
-
- /**
- * Reads a single byte value.
- *
- * @return The byte value read.
- * @throws java.io.IOException If there is IO error.
- * @throws java.io.EOFException If end of file reached.
- */
- private int readByte() throws IOException {
- int b = input.read();
- if (b == -1) {
- throw new EOFException();
- }
- return b;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryEncoder.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryEncoder.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryEncoder.java
deleted file mode 100644
index bc1bce0..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/BinaryEncoder.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * An encoder to help encode snapshots in binary format.
- */
-public final class BinaryEncoder {
-
- private final OutputStream output;
-
- /**
- * @param output stream to write to
- */
- public BinaryEncoder(OutputStream output) {
- this.output = output;
- }
-
- /**
- * write a single int value.
- * @throws java.io.IOException If there is IO error.
- */
- public BinaryEncoder writeInt(int i) throws IOException {
- // Compute the zig-zag value. First double the value and flip the bit if the input is negative.
- int val = (i << 1) ^ (i >> 31);
-
- if ((val & ~0x7f) != 0) {
- output.write(0x80 | val & 0x7f);
- val >>>= 7;
- while (val > 0x7f) {
- output.write(0x80 | val & 0x7f);
- val >>>= 7;
- }
- }
- output.write(val);
-
- return this;
- }
-
- /**
- * write a single long int value.
- * @throws java.io.IOException If there is IO error.
- */
- public BinaryEncoder writeLong(long l) throws IOException {
- // Compute the zig-zag value. First double the value and flip the bit if the input is negative.
- long val = (l << 1) ^ (l >> 63);
-
- if ((val & ~0x7f) != 0) {
- output.write((int) (0x80 | val & 0x7f));
- val >>>= 7;
- while (val > 0x7f) {
- output.write((int) (0x80 | val & 0x7f));
- val >>>= 7;
- }
- }
- output.write((int) val);
-
- return this;
- }
-
- /**
- * write a sequence of bytes. First writes the number of bytes as an int, then the bytes themselves.
- * @throws java.io.IOException If there is IO error.
- */
- public BinaryEncoder writeBytes(byte[] bytes) throws IOException {
- writeLong(bytes.length);
- output.write(bytes, 0, bytes.length);
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/DefaultSnapshotCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/DefaultSnapshotCodec.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/DefaultSnapshotCodec.java
deleted file mode 100644
index d09d79d..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/DefaultSnapshotCodec.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.TreeMap;
-
-/**
- * Handles serialization/deserialization of a {@link co.cask.tephra.persist.TransactionSnapshot}
- * and its elements to {@code byte[]}.
- * @deprecated This codec is now deprecated and is replaced by {@link SnapshotCodecV2}.
- */
-@Deprecated
-public class DefaultSnapshotCodec implements SnapshotCodec {
-
- private static final Logger LOG = LoggerFactory.getLogger(DefaultSnapshotCodec.class);
-
- @Override
- public int getVersion() {
- return 1;
- }
-
- @Override
- public void encode(OutputStream out, TransactionSnapshot snapshot) {
- try {
- BinaryEncoder encoder = new BinaryEncoder(out);
-
- encoder.writeLong(snapshot.getTimestamp());
- encoder.writeLong(snapshot.getReadPointer());
- encoder.writeLong(snapshot.getWritePointer());
- encodeInvalid(encoder, snapshot.getInvalid());
- encodeInProgress(encoder, snapshot.getInProgress());
- encodeChangeSets(encoder, snapshot.getCommittingChangeSets());
- encodeChangeSets(encoder, snapshot.getCommittedChangeSets());
-
- } catch (IOException e) {
- LOG.error("Unable to serialize transaction state: ", e);
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public TransactionSnapshot decode(InputStream in) {
- BinaryDecoder decoder = new BinaryDecoder(in);
-
- try {
- TransactionVisibilityState minTxSnapshot = decodeTransactionVisibilityState(in);
- NavigableMap<Long, Set<ChangeId>> committing = decodeChangeSets(decoder);
- NavigableMap<Long, Set<ChangeId>> committed = decodeChangeSets(decoder);
- return new TransactionSnapshot(minTxSnapshot.getTimestamp(), minTxSnapshot.getReadPointer(),
- minTxSnapshot.getWritePointer(), minTxSnapshot.getInvalid(),
- minTxSnapshot.getInProgress(), committing, committed);
- } catch (IOException e) {
- LOG.error("Unable to deserialize transaction state: ", e);
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public TransactionVisibilityState decodeTransactionVisibilityState(InputStream in) {
- BinaryDecoder decoder = new BinaryDecoder(in);
- try {
- long timestamp = decoder.readLong();
- long readPointer = decoder.readLong();
- long writePointer = decoder.readLong();
- Collection<Long> invalid = decodeInvalid(decoder);
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress = decodeInProgress(decoder);
- return new TransactionSnapshot(timestamp, readPointer, writePointer, invalid, inProgress);
- } catch (IOException e) {
- LOG.error("Unable to deserialize transaction state: ", e);
- throw Throwables.propagate(e);
- }
- }
-
- private void encodeInvalid(BinaryEncoder encoder, Collection<Long> invalid) throws IOException {
- if (!invalid.isEmpty()) {
- encoder.writeInt(invalid.size());
- for (long invalidTx : invalid) {
- encoder.writeLong(invalidTx);
- }
- }
- encoder.writeInt(0); // zero denotes end of list as per AVRO spec
- }
-
- private Collection<Long> decodeInvalid(BinaryDecoder decoder) throws IOException {
- int size = decoder.readInt();
- Collection<Long> invalid = Lists.newArrayListWithCapacity(size);
- while (size != 0) { // zero denotes end of list as per AVRO spec
- for (int remaining = size; remaining > 0; --remaining) {
- invalid.add(decoder.readLong());
- }
- size = decoder.readInt();
- }
- return invalid;
- }
-
- protected void encodeInProgress(BinaryEncoder encoder, Map<Long, TransactionManager.InProgressTx> inProgress)
- throws IOException {
-
- if (!inProgress.isEmpty()) {
- encoder.writeInt(inProgress.size());
- for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
- encoder.writeLong(entry.getKey()); // tx id
- encoder.writeLong(entry.getValue().getExpiration());
- encoder.writeLong(entry.getValue().getVisibilityUpperBound());
- }
- }
- encoder.writeInt(0); // zero denotes end of list as per AVRO spec
- }
-
- protected NavigableMap<Long, TransactionManager.InProgressTx> decodeInProgress(BinaryDecoder decoder)
- throws IOException {
-
- int size = decoder.readInt();
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
- while (size != 0) { // zero denotes end of list as per AVRO spec
- for (int remaining = size; remaining > 0; --remaining) {
- long txId = decoder.readLong();
- long expiration = decoder.readLong();
- long visibilityUpperBound = decoder.readLong();
- inProgress.put(txId,
- new TransactionManager.InProgressTx(visibilityUpperBound, expiration));
- }
- size = decoder.readInt();
- }
- return inProgress;
- }
-
- private void encodeChangeSets(BinaryEncoder encoder, Map<Long, Set<ChangeId>> changes) throws IOException {
- if (!changes.isEmpty()) {
- encoder.writeInt(changes.size());
- for (Map.Entry<Long, Set<ChangeId>> entry : changes.entrySet()) {
- encoder.writeLong(entry.getKey());
- encodeChanges(encoder, entry.getValue());
- }
- }
- encoder.writeInt(0); // zero denotes end of list as per AVRO spec
- }
-
- private NavigableMap<Long, Set<ChangeId>> decodeChangeSets(BinaryDecoder decoder) throws IOException {
- int size = decoder.readInt();
- NavigableMap<Long, Set<ChangeId>> changeSets = new TreeMap<Long, Set<ChangeId>>();
- while (size != 0) { // zero denotes end of list as per AVRO spec
- for (int remaining = size; remaining > 0; --remaining) {
- changeSets.put(decoder.readLong(), decodeChanges(decoder));
- }
- size = decoder.readInt();
- }
- return changeSets;
- }
-
- private void encodeChanges(BinaryEncoder encoder, Set<ChangeId> changes) throws IOException {
- if (!changes.isEmpty()) {
- encoder.writeInt(changes.size());
- for (ChangeId change : changes) {
- encoder.writeBytes(change.getKey());
- }
- }
- encoder.writeInt(0); // zero denotes end of list as per AVRO spec
- }
-
- private Set<ChangeId> decodeChanges(BinaryDecoder decoder) throws IOException {
- int size = decoder.readInt();
- HashSet<ChangeId> changes = Sets.newHashSetWithExpectedSize(size);
- while (size != 0) { // zero denotes end of list as per AVRO spec
- for (int remaining = size; remaining > 0; --remaining) {
- changes.add(new ChangeId(decoder.readBytes()));
- }
- size = decoder.readInt();
- }
- // todo is there an immutable hash set?
- return changes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodec.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodec.java
deleted file mode 100644
index 8e9624e..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodec.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionVisibilityState;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-
-/**
- * Interface to decode and encode a transaction snapshot. Each codec implements one version of the encoding.
- * It need not include the version when encoding the snapshot.
- */
-public interface SnapshotCodec {
-
- /**
- * @return the version of the encoding implemented by the codec.
- */
- int getVersion();
-
- /**
- * Encode a transaction snapshot into an output stream.
- * @param out the output stream to write to
- * @param snapshot the snapshot to encode
- */
- void encode(OutputStream out, TransactionSnapshot snapshot);
-
- /**
- * Decode a transaction snapshot from an input stream.
- * @param in the input stream to read from
- * @return the decoded snapshot
- */
- TransactionSnapshot decode(InputStream in);
-
- /**
- * Decode transaction visibility state from an input stream.
- * @param in the input stream to read from
- * @return {@link TransactionVisibilityState}
- */
- TransactionVisibilityState decodeTransactionVisibilityState(InputStream in);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecProvider.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecProvider.java
deleted file mode 100644
index 28d829c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecProvider.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.inject.Inject;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.List;
-import java.util.SortedMap;
-import javax.annotation.Nonnull;
-
-/**
- * Maintains the codecs for all known versions of the transaction snapshot encoding.
- */
-public class SnapshotCodecProvider implements SnapshotCodec {
-
- private static final Logger LOG = LoggerFactory.getLogger(SnapshotCodecProvider.class);
-
- private final SortedMap<Integer, SnapshotCodec> codecs = Maps.newTreeMap();
-
- @Inject
- public SnapshotCodecProvider(Configuration configuration) {
- initialize(configuration);
- }
-
- /**
- * Register all codec specified in the configuration with this provider.
- * There can only be one codec for a given version.
- */
- private void initialize(Configuration configuration) {
- String[] codecClassNames = configuration.getTrimmedStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- List<Class> codecClasses = Lists.newArrayList();
- if (codecClassNames != null) {
- for (String clsName : codecClassNames) {
- try {
- codecClasses.add(Class.forName(clsName));
- } catch (ClassNotFoundException cnfe) {
- LOG.warn("Unable to load class configured for " + TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES
- + ": " + clsName, cnfe);
- }
- }
- }
-
- if (codecClasses.size() == 0) {
- codecClasses.addAll(Arrays.asList(TxConstants.Persist.DEFAULT_TX_SNAPHOT_CODEC_CLASSES));
- }
- for (Class<?> codecClass : codecClasses) {
- try {
- SnapshotCodec codec = (SnapshotCodec) (codecClass.newInstance());
- codecs.put(codec.getVersion(), codec);
- LOG.debug("Using snapshot codec {} for snapshots of version {}", codecClass.getName(), codec.getVersion());
- } catch (Exception e) {
- LOG.warn("Error instantiating snapshot codec {}. Skipping.", codecClass.getName(), e);
- }
- }
- }
-
- /**
- * Retrieve the codec for a particular version of the encoding.
- * @param version the version of interest
- * @return the corresponding codec
- * @throws java.lang.IllegalArgumentException if the version is not known
- */
- @Nonnull
- @VisibleForTesting
- SnapshotCodec getCodecForVersion(int version) {
- SnapshotCodec codec = codecs.get(version);
- if (codec == null) {
- throw new IllegalArgumentException(String.format("Version %d of snapshot encoding is not supported", version));
- }
- return codec;
- }
-
- /**
- * Retrieve the current snapshot codec, that is, the codec with the highest known version.
- * @return the current codec
- * @throws java.lang.IllegalStateException if no codecs are registered
- */
- private SnapshotCodec getCurrentCodec() {
- if (codecs.isEmpty()) {
- throw new IllegalStateException(String.format("No codecs are registered."));
- }
- return codecs.get(codecs.lastKey());
- }
-
- // Return the appropriate codec for the version in InputStream
- private SnapshotCodec getCodec(InputStream in) {
- BinaryDecoder decoder = new BinaryDecoder(in);
- int persistedVersion;
- try {
- persistedVersion = decoder.readInt();
- } catch (IOException e) {
- LOG.error("Unable to read transaction state version: ", e);
- throw Throwables.propagate(e);
- }
- return getCodecForVersion(persistedVersion);
- }
-
- @Override
- public int getVersion() {
- return getCurrentCodec().getVersion();
- }
-
- @Override
- public TransactionSnapshot decode(InputStream in) {
- return getCodec(in).decode(in);
- }
-
- @Override
- public TransactionVisibilityState decodeTransactionVisibilityState(InputStream in) {
- return getCodec(in).decodeTransactionVisibilityState(in);
- }
-
- @Override
- public void encode(OutputStream out, TransactionSnapshot snapshot) {
- SnapshotCodec codec = getCurrentCodec();
- try {
- new BinaryEncoder(out).writeInt(codec.getVersion());
- } catch (IOException e) {
- LOG.error("Unable to write transaction state version: ", e);
- throw Throwables.propagate(e);
- }
- codec.encode(out, snapshot);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV2.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV2.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV2.java
deleted file mode 100644
index 5d07ba5..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV2.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import com.google.common.collect.Maps;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-/**
- * Handles serialization/deserialization of a {@link co.cask.tephra.persist.TransactionSnapshot}
- * and its elements to {@code byte[]}.
- */
-public class SnapshotCodecV2 extends DefaultSnapshotCodec {
- @Override
- public int getVersion() {
- return 2;
- }
-
- @Override
- protected void encodeInProgress(BinaryEncoder encoder, Map<Long, TransactionManager.InProgressTx> inProgress)
- throws IOException {
-
- if (!inProgress.isEmpty()) {
- encoder.writeInt(inProgress.size());
- for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
- encoder.writeLong(entry.getKey()); // tx id
- encoder.writeLong(entry.getValue().getExpiration());
- encoder.writeLong(entry.getValue().getVisibilityUpperBound());
- encoder.writeInt(entry.getValue().getType().ordinal());
- }
- }
- encoder.writeInt(0); // zero denotes end of list as per AVRO spec
- }
-
- @Override
- protected NavigableMap<Long, TransactionManager.InProgressTx> decodeInProgress(BinaryDecoder decoder)
- throws IOException {
-
- int size = decoder.readInt();
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
- while (size != 0) { // zero denotes end of list as per AVRO spec
- for (int remaining = size; remaining > 0; --remaining) {
- long txId = decoder.readLong();
- long expiration = decoder.readLong();
- long visibilityUpperBound = decoder.readLong();
- int txTypeIdx = decoder.readInt();
- TransactionType txType;
- try {
- txType = TransactionType.values()[txTypeIdx];
- } catch (ArrayIndexOutOfBoundsException e) {
- throw new IOException("Type enum ordinal value is out of range: " + txTypeIdx);
- }
- inProgress.put(txId,
- new TransactionManager.InProgressTx(visibilityUpperBound, expiration, txType,
- new LongArrayList()));
- }
- size = decoder.readInt();
- }
- return inProgress;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV3.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV3.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV3.java
deleted file mode 100644
index b11821a..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV3.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-/**
- * Handles serialization/deserialization of a {@link co.cask.tephra.persist.TransactionSnapshot}
- * and its elements to {@code byte[]}.
- *
- * <p>The serialization/deserialization of this codec is the same as that performed by {@link SnapshotCodecV2},
- * but a new version number is used to allow easy migration from projects using deprecated codecs with
- * conflicting version numbers.</p>
- */
-public class SnapshotCodecV3 extends SnapshotCodecV2 {
- @Override
- public int getVersion() {
- return 3;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV4.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV4.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV4.java
deleted file mode 100644
index 41d30f2..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/SnapshotCodecV4.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.snapshot;
-
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import com.google.common.collect.Maps;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-/**
- * Handles serialization/deserialization of a {@link co.cask.tephra.persist.TransactionSnapshot}
- * and its elements to {@code byte[]}.
- *
- */
-public class SnapshotCodecV4 extends SnapshotCodecV2 {
- @Override
- public int getVersion() {
- return 4;
- }
-
- @Override
- protected void encodeInProgress(BinaryEncoder encoder, Map<Long, TransactionManager.InProgressTx> inProgress)
- throws IOException {
-
- if (!inProgress.isEmpty()) {
- encoder.writeInt(inProgress.size());
- for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
- encoder.writeLong(entry.getKey()); // tx id
- encoder.writeLong(entry.getValue().getExpiration());
- encoder.writeLong(entry.getValue().getVisibilityUpperBound());
- encoder.writeInt(entry.getValue().getType().ordinal());
- // write checkpoint tx IDs
- LongArrayList checkpointPointers = entry.getValue().getCheckpointWritePointers();
- if (checkpointPointers != null && !checkpointPointers.isEmpty()) {
- encoder.writeInt(checkpointPointers.size());
- for (int i = 0; i < checkpointPointers.size(); i++) {
- encoder.writeLong(checkpointPointers.getLong(i));
- }
- }
- encoder.writeInt(0);
- }
- }
- encoder.writeInt(0); // zero denotes end of list as per AVRO spec
- }
-
- @Override
- protected NavigableMap<Long, TransactionManager.InProgressTx> decodeInProgress(BinaryDecoder decoder)
- throws IOException {
-
- int size = decoder.readInt();
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
- while (size != 0) { // zero denotes end of list as per AVRO spec
- for (int remaining = size; remaining > 0; --remaining) {
- long txId = decoder.readLong();
- long expiration = decoder.readLong();
- long visibilityUpperBound = decoder.readLong();
- int txTypeIdx = decoder.readInt();
- TransactionType txType;
- try {
- txType = TransactionType.values()[txTypeIdx];
- } catch (ArrayIndexOutOfBoundsException e) {
- throw new IOException("Type enum ordinal value is out of range: " + txTypeIdx);
- }
- // read checkpoint tx IDs
- int checkpointPointerSize = decoder.readInt();
- LongArrayList checkpointPointers = new LongArrayList(checkpointPointerSize);
- while (checkpointPointerSize != 0) {
- for (int checkpointRemaining = checkpointPointerSize; checkpointRemaining > 0; --checkpointRemaining) {
- checkpointPointers.add(decoder.readLong());
- }
- checkpointPointerSize = decoder.readInt();
- }
- inProgress.put(txId,
- new TransactionManager.InProgressTx(visibilityUpperBound, expiration, txType, checkpointPointers));
- }
- size = decoder.readInt();
- }
- return inProgress;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/snapshot/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/snapshot/package-info.java b/tephra-core/src/main/java/co/cask/tephra/snapshot/package-info.java
deleted file mode 100644
index e2d044c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/snapshot/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains interfaces and implementations for encoding and decoding transaction snapshots.
- */
-package co.cask.tephra.snapshot;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationFactory.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationFactory.java b/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationFactory.java
deleted file mode 100644
index e830524..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationFactory.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import com.google.inject.Provider;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Provides {@code org.apache.hadoop.conf.Configuration} instances, constructed by the correct version used
- * for the runtime.
- */
-public class ConfigurationFactory implements Provider<Configuration> {
- private static class ConfigurationProviderFactory extends HBaseVersionSpecificFactory<ConfigurationProvider> {
- @Override
- protected String getHBase96Classname() {
- return "co.cask.tephra.hbase96.HBase96ConfigurationProvider";
- }
-
- @Override
- protected String getHBase98Classname() {
- return "co.cask.tephra.hbase98.HBase98ConfigurationProvider";
- }
-
- @Override
- protected String getHBase10Classname() {
- return "co.cask.tephra.hbase10.HBase10ConfigurationProvider";
- }
-
- @Override
- protected String getHBase11Classname() {
- return "co.cask.tephra.hbase11.HBase11ConfigurationProvider";
- }
-
- @Override
- protected String getHBase10CDHClassname() {
- return "co.cask.tephra.hbase10cdh.HBase10ConfigurationProvider";
- }
- }
-
- private final ConfigurationProvider provider = new ConfigurationProviderFactory().get();
-
- /**
- * Returns a new {@link org.apache.hadoop.conf.Configuration} instance from the HBase version-specific factory.
- */
- @Override
- public Configuration get() {
- return provider.get();
- }
-
- /**
- * Returns a new {@link org.apache.hadoop.conf.Configuration} instance from the HBase version-specific factory.
- *
- * @param baseConf additional configuration properties to merge on to the classpath configuration
- * @return the merged configuration
- */
- public Configuration get(Configuration baseConf) {
- return provider.get(baseConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationProvider.java b/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationProvider.java
deleted file mode 100644
index e88acf8..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/util/ConfigurationProvider.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import com.google.inject.Provider;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Provides {@code Configuration} instances, constructed by the HBase version on which we are running.
- */
-public abstract class ConfigurationProvider implements Provider<Configuration> {
- @Override
- public abstract Configuration get();
-
- public abstract Configuration get(Configuration baseConf);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersion.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersion.java b/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersion.java
deleted file mode 100644
index 84311c9..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/util/HBaseVersion.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.util;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.Method;
-import java.text.ParseException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Detects the currently loaded HBase version. It is assumed that only one HBase version is loaded at a time,
- * since using more than one HBase version within the same process will require classloader isolation anyway.
- */
-public class HBaseVersion {
- private static final String HBASE_94_VERSION = "0.94";
- private static final String HBASE_96_VERSION = "0.96";
- private static final String HBASE_98_VERSION = "0.98";
- private static final String HBASE_10_VERSION = "1.0";
- private static final String HBASE_11_VERSION = "1.1";
- private static final String HBASE_12_VERSION = "1.2";
- private static final String CDH_CLASSIFIER = "cdh";
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseVersion.class);
-
- /**
- * Represents the major version of the HBase library that is currently loaded.
- */
- public enum Version {
- HBASE_94("0.94"),
- HBASE_96("0.96"),
- HBASE_98("0.98"),
- HBASE_10("1.0"),
- HBASE_10_CDH("1.0-cdh"),
- HBASE_11("1.1"),
- HBASE_12_CDH("1.2-cdh"),
- UNKNOWN("unknown");
-
- final String majorVersion;
-
- Version(String majorVersion) {
- this.majorVersion = majorVersion;
- }
-
- public String getMajorVersion() {
- return majorVersion;
- }
- }
-
- private static Version currentVersion;
- private static String versionString;
- static {
- try {
- Class versionInfoClass = Class.forName("org.apache.hadoop.hbase.util.VersionInfo");
- Method versionMethod = versionInfoClass.getMethod("getVersion");
- versionString = (String) versionMethod.invoke(null);
- if (versionString.startsWith(HBASE_94_VERSION)) {
- currentVersion = Version.HBASE_94;
- } else if (versionString.startsWith(HBASE_96_VERSION)) {
- currentVersion = Version.HBASE_96;
- } else if (versionString.startsWith(HBASE_98_VERSION)) {
- currentVersion = Version.HBASE_98;
- } else if (versionString.startsWith(HBASE_10_VERSION)) {
- VersionNumber ver = VersionNumber.create(versionString);
- if (ver.getClassifier() != null && ver.getClassifier().startsWith(CDH_CLASSIFIER)) {
- currentVersion = Version.HBASE_10_CDH;
- } else {
- currentVersion = Version.HBASE_10;
- }
- } else if (versionString.startsWith(HBASE_11_VERSION)) {
- currentVersion = Version.HBASE_11;
- } else if (versionString.startsWith(HBASE_12_VERSION)) {
- VersionNumber ver = VersionNumber.create(versionString);
- if (ver.getClassifier() != null && ver.getClassifier().startsWith(CDH_CLASSIFIER)) {
- currentVersion = Version.HBASE_12_CDH;
- } else {
- // CDH 5.7 comes with HBase version 1.2.0-CDH5.7.0. However currently there is no
- // other hadoop distribution that uses HBase 1.2, so the version is set here to UNKNOWN.
- currentVersion = Version.UNKNOWN;
- }
- } else {
- currentVersion = Version.UNKNOWN;
- }
- } catch (Throwable e) {
- // must be a class loading exception, HBase is not there
- LOG.error("Unable to determine HBase version from string '{}', are HBase classes available?", versionString);
- LOG.error("Exception was: ", e);
- currentVersion = Version.UNKNOWN;
- }
- }
-
- /**
- * Returns the major version of the currently loaded HBase library.
- */
- public static Version get() {
- return currentVersion;
- }
-
- /**
- * Returns the full version string for the currently loaded HBase library.
- */
- public static String getVersionString() {
- return versionString;
- }
-
- /**
- * Prints out the HBase {@link Version} enum value for the current version of HBase on the classpath.
- */
- public static void main(String[] args) {
- boolean verbose = args.length == 1 && "-v".equals(args[0]);
- Version version = HBaseVersion.get();
- System.out.println(version.getMajorVersion());
- if (verbose) {
- System.out.println("versionString=" + getVersionString());
- }
- }
-
- /**
- * Utility class to parse apart version number components. The version string provided is expected to be in
- * the format: major[.minor[.patch[.last]][-classifier][-SNAPSHOT]
- *
- * <p>Only the major version number is actually required.</p>
- */
- public static class VersionNumber {
- private static final Pattern PATTERN =
- Pattern.compile("(\\d+)(\\.(\\d+))?(\\.(\\d+))?(\\.(\\d+))?(\\-(?!SNAPSHOT)([^\\-]+))?(\\-SNAPSHOT)?");
-
- private Integer major;
- private Integer minor;
- private Integer patch;
- private Integer last;
- private String classifier;
- private boolean snapshot;
-
- private VersionNumber(Integer major, Integer minor, Integer patch, Integer last,
- String classifier, boolean snapshot) {
- this.major = major;
- this.minor = minor;
- this.patch = patch;
- this.last = last;
- this.classifier = classifier;
- this.snapshot = snapshot;
- }
-
- public Integer getMajor() {
- return major;
- }
-
- public Integer getMinor() {
- return minor;
- }
-
- public Integer getPatch() {
- return patch;
- }
-
- public Integer getLast() {
- return last;
- }
-
- public String getClassifier() {
- return classifier;
- }
-
- public boolean isSnapshot() {
- return snapshot;
- }
-
- public static VersionNumber create(String versionString) throws ParseException {
- Matcher matcher = PATTERN.matcher(versionString);
- if (matcher.matches()) {
- String majorString = matcher.group(1);
- String minorString = matcher.group(3);
- String patchString = matcher.group(5);
- String last = matcher.group(7);
- String classifier = matcher.group(9);
- String snapshotString = matcher.group(10);
- return new VersionNumber(new Integer(majorString),
- minorString != null ? new Integer(minorString) : null,
- patchString != null ? new Integer(patchString) : null,
- last != null ? new Integer(last) : null,
- classifier,
- "-SNAPSHOT".equals(snapshotString));
- }
- throw new ParseException(
- "Input string did not match expected pattern: major[.minor[.patch]][-classifier][-SNAPSHOT]", 0);
- }
- }
-}
[24/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/TransactionAwareHTable.java b/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/TransactionAwareHTable.java
deleted file mode 100644
index a4d3140..0000000
--- a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/TransactionAwareHTable.java
+++ /dev/null
@@ -1,615 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase96;
-
-import co.cask.tephra.AbstractTransactionAwareTable;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TxConstants;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.Batch;
-import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-
-/**
- * A Transaction Aware HTable implementation for HBase 0.96. Operations are committed as usual,
- * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
- * was started.
- */
-public class TransactionAwareHTable extends AbstractTransactionAwareTable
- implements HTableInterface, TransactionAware {
-
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
- private final HTableInterface hTable;
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- */
- public TransactionAwareHTable(HTableInterface hTable) {
- this(hTable, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
- this(hTable, conflictLevel, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
- this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
- boolean allowNonTransactional) {
- super(conflictLevel, allowNonTransactional);
- this.hTable = hTable;
- }
-
- /* AbstractTransactionAwareTable implementation */
-
- @Override
- protected byte[] getTableKey() {
- return getTableName();
- }
-
- @Override
- protected boolean doCommit() throws IOException {
- hTable.flushCommits();
- return true;
- }
-
- @Override
- protected boolean doRollback() throws Exception {
- try {
- // pre-size arraylist of deletes
- int size = 0;
- for (Set<ActionChange> cs : changeSets.values()) {
- size += cs.size();
- }
- List<Delete> rollbackDeletes = new ArrayList<>(size);
- for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
- long transactionTimestamp = entry.getKey();
- for (ActionChange change : entry.getValue()) {
- byte[] row = change.getRow();
- byte[] family = change.getFamily();
- byte[] qualifier = change.getQualifier();
- Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- switch (conflictLevel) {
- case ROW:
- case NONE:
- // issue family delete for the tx write pointer
- rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
- break;
- case COLUMN:
- if (family != null && qualifier == null) {
- rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
- } else if (family != null && qualifier != null) {
- rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
- }
- break;
- default:
- throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
- }
- rollbackDeletes.add(rollbackDelete);
- }
- }
- hTable.delete(rollbackDeletes);
- return true;
- } finally {
- try {
- hTable.flushCommits();
- } catch (Exception e) {
- LOG.error("Could not flush HTable commits", e);
- }
- tx = null;
- changeSets.clear();
- }
- }
-
- /* HTableInterface implementation */
-
- @Override
- public byte[] getTableName() {
- return hTable.getTableName();
- }
-
- @Override
- public TableName getName() {
- return hTable.getName();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hTable.getConfiguration();
- }
-
- @Override
- public HTableDescriptor getTableDescriptor() throws IOException {
- return hTable.getTableDescriptor();
- }
-
- @Override
- public boolean exists(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.exists(transactionalizeAction(get));
- }
-
- @Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.exists(transactionalizedGets);
- }
-
- @Override
- public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batch(transactionalizeActions(actions), results);
- }
-
- @Override
- public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batch(transactionalizeActions(actions));
- }
-
- @Override
- public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
- IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batchCallback(transactionalizeActions(actions), results, callback);
- }
-
- @Override
- public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
- InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batchCallback(transactionalizeActions(actions), callback);
- }
-
- @Override
- public Result get(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.get(transactionalizeAction(get));
- }
-
- @Override
- public Result[] get(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.get(transactionalizedGets);
- }
-
- @Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- if (allowNonTransactional) {
- return hTable.getRowOrBefore(row, family);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public ResultScanner getScanner(Scan scan) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addFamily(family);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addColumn(family, qualifier);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public void put(Put put) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Put txPut = transactionalizeAction(put);
- hTable.put(txPut);
- }
-
- @Override
- public void put(List<Put> puts) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
- for (Put put : puts) {
- Put txPut = transactionalizeAction(put);
- transactionalizedPuts.add(txPut);
- }
- hTable.put(transactionalizedPuts);
- }
-
- @Override
- public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndPut(row, family, qualifier, value, put);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public void delete(Delete delete) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.delete(transactionalizeAction(delete));
- }
-
- @Override
- public void delete(List<Delete> deletes) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
- for (Delete delete : deletes) {
- Delete txDelete = transactionalizeAction(delete);
- transactionalizedDeletes.add(txDelete);
- }
- hTable.delete(transactionalizedDeletes);
- }
-
- @Override
- public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndDelete(row, family, qualifier, value, delete);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public void mutateRow(RowMutations rm) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- RowMutations transactionalMutations = new RowMutations();
- for (Mutation mutation : rm.getMutations()) {
- if (mutation instanceof Put) {
- transactionalMutations.add(transactionalizeAction((Put) mutation));
- } else if (mutation instanceof Delete) {
- transactionalMutations.add(transactionalizeAction((Delete) mutation));
- }
- }
- hTable.mutateRow(transactionalMutations);
- }
-
- @Override
- public Result append(Append append) throws IOException {
- if (allowNonTransactional) {
- return hTable.append(append);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public Result increment(Increment increment) throws IOException {
- if (allowNonTransactional) {
- return hTable.increment(increment);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean isAutoFlush() {
- return hTable.isAutoFlush();
- }
-
- @Override
- public void flushCommits() throws IOException {
- hTable.flushCommits();
- }
-
- @Override
- public void close() throws IOException {
- hTable.close();
- }
-
- @Override
- public CoprocessorRpcChannel coprocessorService(byte[] row) {
- return hTable.coprocessorService(row);
- }
-
- @Override
- public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable)
- throws ServiceException, Throwable {
- return hTable.coprocessorService(service, startKey, endKey, callable);
- }
-
- @Override
- public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable, Batch.Callback<R> callback)
- throws ServiceException, Throwable {
- hTable.coprocessorService(service, startKey, endKey, callable, callback);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush) {
- setAutoFlushTo(autoFlush);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- hTable.setAutoFlush(autoFlush, clearBufferOnFail);
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- hTable.setAutoFlushTo(autoFlush);
- }
-
- @Override
- public long getWriteBufferSize() {
- return hTable.getWriteBufferSize();
- }
-
- @Override
- public void setWriteBufferSize(long writeBufferSize) throws IOException {
- hTable.setWriteBufferSize(writeBufferSize);
- }
-
- // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
-
- private Get transactionalizeAction(Get get) throws IOException {
- addToOperation(get, tx);
- return get;
- }
-
- private Scan transactionalizeAction(Scan scan) throws IOException {
- addToOperation(scan, tx);
- return scan;
- }
-
- private Put transactionalizeAction(Put put) throws IOException {
- Put txPut = new Put(put.getRow(), tx.getWritePointer());
- Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
- if (!familyMap.isEmpty()) {
- for (Map.Entry<byte[], List<Cell>> family : familyMap) {
- List<Cell> familyValues = family.getValue();
- if (!familyValues.isEmpty()) {
- for (Cell value : familyValues) {
- txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
- addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
- txPut.setAttribute(entry.getKey(), entry.getValue());
- }
- txPut.setDurability(put.getDurability());
- addToOperation(txPut, tx);
- return txPut;
- }
-
- private Delete transactionalizeAction(Delete delete) throws IOException {
- long transactionTimestamp = tx.getWritePointer();
-
- byte[] deleteRow = delete.getRow();
- Delete txDelete = new Delete(deleteRow, transactionTimestamp);
-
- Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
- if (familyToDelete.isEmpty()) {
- // perform a row delete if we are using row-level conflict detection
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // Row delete leaves delete markers in all column families of the table
- // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
- for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
- // no need to identify individual columns deleted
- addToChangeSet(deleteRow, columnDescriptor.getName(), null);
- }
- } else {
- Result result = get(new Get(delete.getRow()));
- // Delete everything
- NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
- for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
- }
- }
- }
- } else {
- for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
- byte[] family = familyEntry.getKey();
- List<Cell> entries = familyEntry.getValue();
- boolean isFamilyDelete = false;
- if (entries.size() == 1) {
- Cell cell = entries.get(0);
- isFamilyDelete = CellUtil.isDeleteFamily(cell);
- }
- if (isFamilyDelete) {
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // no need to identify individual columns deleted
- // Older versions of HBase 0.96 lack HBASE-10964, so family deletes do not correctly
- // inherit the common Delete timestamp, so must explicitly set the timestamp here.
- txDelete.deleteFamily(family, transactionTimestamp);
- addToChangeSet(deleteRow, family, null);
- } else {
- Result result = get(new Get(delete.getRow()).addFamily(family));
- // Delete entire family
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, family, column.getKey());
- }
- }
- } else {
- for (Cell value : entries) {
- txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
- addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- txDelete.setAttribute(entry.getKey(), entry.getValue());
- }
- txDelete.setDurability(delete.getDurability());
- return txDelete;
- }
-
- private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
- List<Row> transactionalizedActions = new ArrayList<>(actions.size());
- for (Row action : actions) {
- if (action instanceof Get) {
- transactionalizedActions.add(transactionalizeAction((Get) action));
- } else if (action instanceof Put) {
- transactionalizedActions.add(transactionalizeAction((Put) action));
- } else if (action instanceof Delete) {
- transactionalizedActions.add(transactionalizeAction((Delete) action));
- } else {
- transactionalizedActions.add(action);
- }
- }
- return transactionalizedActions;
- }
-
- public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
- op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilter.java
deleted file mode 100644
index 8bbf929..0000000
--- a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilter.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
- * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
- * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
- * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
- * Please see TEPHRA-169 for more details.
- */
-public class CellSkipFilter extends FilterBase {
- private final Filter filter;
- // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
- private KeyValue skipColumn = null;
-
- public CellSkipFilter(Filter filter) {
- this.filter = filter;
- }
-
- /**
- * Determines whether the current cell should be skipped. The cell will be skipped
- * if the previous keyvalue had the same key as the current cell. This means filter already responded
- * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
- * @param cell the {@link Cell} to be tested for skipping
- * @return true is current cell should be skipped, false otherwise
- */
- private boolean skipCellVersion(Cell cell) {
- return skipColumn != null
- && skipColumn.matchingRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
- && skipColumn.matchingColumn(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (skipCellVersion(cell)) {
- return ReturnCode.NEXT_COL;
- }
-
- ReturnCode code = filter.filterKeyValue(cell);
- if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
- // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
- skipColumn = KeyValue.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
- cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(),
- cell.getQualifierLength());
- } else {
- skipColumn = null;
- }
- return code;
- }
-
- @Override
- public boolean filterRow() throws IOException {
- return filter.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- return filter.transformCell(cell);
- }
-
- @Override
- public void reset() throws IOException {
- filter.reset();
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- return filter.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- return filter.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- filter.filterRowCells(kvs);
- }
-
- @Override
- public boolean hasFilterRow() {
- return filter.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- return filter.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- return filter.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- return filter.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return filter.toByteArray();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionFilters.java b/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionFilters.java
deleted file mode 100644
index 22b0ae7..0000000
--- a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionFilters.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96.coprocessor;
-
-import co.cask.tephra.Transaction;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Factory class for providing {@link Filter} instances.
- */
-public class TransactionFilters {
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
- }
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessor.java
deleted file mode 100644
index 8351d2e..0000000
--- a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessor.java
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCodec;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreScanner;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-
-/**
- * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
- * for transactions:
- * <ul>
- * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
- * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
- * or expired due to TTL.</li>
- * </ul>
- *
- * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
- * or on all user tables by adding the following to hbase-site.xml:
- * {@code
- * <property>
- * <name>hbase.coprocessor.region.classes</name>
- * <value>co.cask.tephra.hbase96.coprocessor.TransactionProcessor</value>
- * </property>
- * }
- * </p>
- *
- * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
- * as an attribute:
- * {@code
- * Transaction t = ...;
- * Get get = new Get(...);
- * TransactionCodec codec = new TransactionCodec();
- * codec.addToOperation(get, t);
- * }
- * </p>
- */
-public class TransactionProcessor extends BaseRegionObserver {
- private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
-
- private TransactionStateCache cache;
- private final TransactionCodec txCodec;
- protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
- protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
-
- public TransactionProcessor() {
- this.txCodec = new TransactionCodec();
- }
-
- /* RegionObserver implementation */
-
- @Override
- public void start(CoprocessorEnvironment e) throws IOException {
- if (e instanceof RegionCoprocessorEnvironment) {
- RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
- Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
- this.cache = cacheSupplier.get();
-
- HTableDescriptor tableDesc = env.getRegion().getTableDesc();
- for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
- String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
- long ttl = 0;
- if (columnTTL != null) {
- try {
- ttl = Long.parseLong(columnTTL);
- LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
- } catch (NumberFormatException nfe) {
- LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
- ", value = " + columnTTL);
- }
- }
- ttlByFamily.put(columnDesc.getName(), ttl);
- }
-
- this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
- TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
- this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
- if (readNonTxnData) {
- LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
- }
- }
- }
-
- protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
- return new TransactionStateCacheSupplier(env.getConfiguration());
- }
-
- @Override
- public void stop(CoprocessorEnvironment e) throws IOException {
- // nothing to do
- }
-
- @Override
- public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
- throws IOException {
- Transaction tx = getFromOperation(get);
- if (tx != null) {
- projectFamilyDeletes(get);
- get.setMaxVersions();
- get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
- get.setFilter(newFilter);
- }
- }
-
- @Override
- public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
- Durability durability) throws IOException {
- // Translate deletes into our own delete tombstones
- // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
- // us to rollback the changes (by a real delete) if the transaction fails
-
- // Deletes that are part of a transaction rollback do not need special handling.
- // They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
- return;
- }
-
- // Other deletes are client-initiated and need to be translated into our own tombstones
- // TODO: this should delegate to the DeleteStrategy implementation.
- Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
- for (byte[] family : delete.getFamilyCellMap().keySet()) {
- List<Cell> familyCells = delete.getFamilyCellMap().get(family);
- if (isFamilyDelete(familyCells)) {
- deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- } else {
- int cellSize = familyCells.size();
- for (int i = 0; i < cellSize; i++) {
- Cell cell = familyCells.get(i);
- deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
- }
- e.getEnvironment().getRegion().put(deleteMarkers);
- // skip normal delete handling
- e.bypass();
- }
-
- private boolean isFamilyDelete(List<Cell> familyCells) {
- return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
- }
-
- @Override
- public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
- throws IOException {
- Transaction tx = getFromOperation(scan);
- if (tx != null) {
- projectFamilyDeletes(scan);
- scan.setMaxVersions();
- scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
- scan.setFilter(newFilter);
- }
- return s;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any scan operation.
- * @param scan The original scan request
- * @return The modified scan request with the family delete qualifiers represented
- */
- private Scan projectFamilyDeletes(Scan scan) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return scan;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any get operation.
- * @param get The original get request
- * @return The modified get request with the family delete qualifiers represented
- */
- private Get projectFamilyDeletes(Get get) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return get;
- }
-
- @Override
- public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- KeyValueScanner memstoreScanner, InternalScanner scanner)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
- Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
- HConstants.OLDEST_TIMESTAMP);
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
- CompactionRequest request)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners, scanType,
- earliestPutTs);
- }
-
- protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
- TransactionVisibilityState snapshot, Store store,
- List<? extends KeyValueScanner> scanners, ScanType type,
- long earliestPutTs) throws IOException {
- if (snapshot == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
- ", no current transaction state found, defaulting to normal " + action + " scanner");
- }
- return null;
- }
-
- // construct a dummy transaction from the latest snapshot
- Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
- Scan scan = new Scan();
- // need to see all versions, since we filter out excludes and applications may rely on multiple versions
- scan.setMaxVersions();
- scan.setFilter(
- new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
- snapshot.getInvalid(),
- getTransactionFilter(dummyTx, type, null)));
-
- return new StoreScanner(store, store.getScanInfo(), scan, scanners,
- type, store.getSmallestReadPoint(), earliestPutTs);
- }
-
- private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
- byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
- if (encoded != null) {
- return txCodec.decode(encoded);
- }
- return null;
- }
-
- /**
- * Derived classes can override this method to customize the filter used to return data visible for the current
- * transaction.
- *
- * @param tx the current transaction to apply
- * @param type the type of scan operation being performed
- */
- protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
- return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
- }
-
- /**
- * Filter used to include cells visible to in-progress transactions on flush and commit.
- */
- static class IncludeInProgressFilter extends FilterBase {
- private final long visibilityUpperBound;
- private final Set<Long> invalidIds;
- private final Filter txFilter;
-
- public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
- this.visibilityUpperBound = upperBound;
- this.invalidIds = Sets.newHashSet(invalids);
- this.txFilter = transactionFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- // include all cells visible to in-progress transactions, except for those already marked as invalid
- long ts = cell.getTimestamp();
- if (ts > visibilityUpperBound) {
- // include everything that could still be in-progress except invalids
- if (invalidIds.contains(ts)) {
- return ReturnCode.SKIP;
- }
- return ReturnCode.INCLUDE;
- }
- return txFilter.filterKeyValue(cell);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java
deleted file mode 100644
index 1266b76..0000000
--- a/tephra-hbase-compat-0.96/src/main/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Applies filtering of data based on transactional visibility (HBase 0.96+ specific version).
- * Note: this is intended for server-side use only, as additional properties need to be set on
- * any {@code Scan} or {@code Get} operation performed.
- */
-public class TransactionVisibilityFilter extends FilterBase {
- private final Transaction tx;
- // oldest visible timestamp by column family, used to apply TTL when reading
- private final Map<byte[], Long> oldestTsByFamily;
- // if false, empty values will be interpreted as deletes
- private final boolean allowEmptyValues;
- // whether or not we can remove delete markers
- // these can only be safely removed when we are traversing all storefiles
- private final boolean clearDeletes;
- // optional sub-filter to apply to visible cells
- private final Filter cellFilter;
-
- // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
- private byte[] currentFamily = new byte[0];
- private long currentOldestTs;
-
- private DeleteTracker deleteTracker = new DeleteTracker();
-
- /**
- * Creates a new {@link Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- this(tx, ttlByFamily, allowEmptyValues, scanType, null);
- }
-
- /**
- * Creates a new {@link Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- this.tx = tx;
- this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
- long familyTTL = ttlEntry.getValue();
- oldestTsByFamily.put(ttlEntry.getKey(),
- familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
- }
- this.allowEmptyValues = allowEmptyValues;
- this.clearDeletes =
- scanType == ScanType.COMPACT_DROP_DELETES ||
- (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
- this.cellFilter = cellFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (!CellUtil.matchingFamily(cell, currentFamily)) {
- // column family changed
- currentFamily = CellUtil.cloneFamily(cell);
- Long familyOldestTs = oldestTsByFamily.get(currentFamily);
- currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
- deleteTracker.reset();
- }
- // need to apply TTL for the column family here
- long kvTimestamp = cell.getTimestamp();
- if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
- // passed TTL for this column, seek to next
- return ReturnCode.NEXT_COL;
- } else if (tx.isVisible(kvTimestamp)) {
- // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
- // cell is visible
- // visibility SNAPSHOT_ALL needs all matches
- return runSubFilter(ReturnCode.INCLUDE, cell);
- }
- if (DeleteTracker.isFamilyDelete(cell)) {
- deleteTracker.addFamilyDelete(cell);
- if (clearDeletes) {
- return ReturnCode.NEXT_COL;
- } else {
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // check if masked by family delete
- if (deleteTracker.isDeleted(cell)) {
- return ReturnCode.NEXT_COL;
- }
- // check for column delete
- if (isColumnDelete(cell)) {
- if (clearDeletes) {
- // skip "deleted" cell
- return ReturnCode.NEXT_COL;
- } else {
- // keep the marker but skip any remaining versions
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- } else {
- return ReturnCode.SKIP;
- }
- }
-
- private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
- if (cellFilter != null) {
- ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
- return determineReturnCode(txFilterCode, subFilterCode);
- }
- return txFilterCode;
- }
-
- /**
- * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
- * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
- * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
- * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
- *
- * @param txFilterCode return code from TransactionVisibilityFilter
- * @param subFilterCode return code from sub-filter
- * @return final return code
- */
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- // Return the more restrictive of the two filter responses
- switch (subFilterCode) {
- case INCLUDE:
- return txFilterCode;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
-
- @Override
- public boolean filterRow() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRow();
- }
- return super.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- // Convert Tephra deletes back into HBase deletes
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
- if (DeleteTracker.isFamilyDelete(cell)) {
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
- KeyValue.Type.DeleteFamily);
- } else if (isColumnDelete(cell)) {
- // Note: in some cases KeyValue.Type.Delete is used in Delete object,
- // and in some other cases KeyValue.Type.DeleteColumn is used.
- // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
- // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
- // work in both cases.
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
- cell.getTimestamp(), KeyValue.Type.DeleteColumn);
- }
- }
- return cell;
- }
-
- @Override
- public void reset() throws IOException {
- deleteTracker.reset();
- if (cellFilter != null) {
- cellFilter.reset();
- }
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRowKey(buffer, offset, length);
- }
- return super.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterAllRemaining();
- }
- return super.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- if (cellFilter != null) {
- cellFilter.filterRowCells(kvs);
- } else {
- super.filterRowCells(kvs);
- }
- }
-
- @Override
- public boolean hasFilterRow() {
- if (cellFilter != null) {
- return cellFilter.hasFilterRow();
- }
- return super.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextKeyHint(currentKV);
- }
- return super.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextCellHint(currentKV);
- }
- return super.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- if (cellFilter != null) {
- return cellFilter.isFamilyEssential(name);
- }
- return super.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return super.toByteArray();
- }
-
- private boolean isColumnDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
- }
-
- private static final class DeleteTracker {
- private long familyDeleteTs;
-
- public static boolean isFamilyDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
- CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
- CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
- }
-
- public void addFamilyDelete(Cell delete) {
- this.familyDeleteTs = delete.getTimestamp();
- }
-
- public boolean isDeleted(Cell cell) {
- return cell.getTimestamp() <= familyDeleteTs;
- }
-
- public void reset() {
- this.familyDeleteTs = 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/HBase96ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/HBase96ConfigurationProvider.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/HBase96ConfigurationProvider.java
new file mode 100644
index 0000000..5bd3559
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/HBase96ConfigurationProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.tephra.util.ConfigurationProvider;
+
+/**
+ * HBase 0.96 version of {@link ConfigurationProvider}.
+ */
+public class HBase96ConfigurationProvider extends ConfigurationProvider {
+ @Override
+ public Configuration get() {
+ return HBaseConfiguration.create();
+ }
+
+ @Override
+ public Configuration get(Configuration baseConf) {
+ return HBaseConfiguration.create(baseConf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/SecondaryIndexTable.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/SecondaryIndexTable.java
new file mode 100644
index 0000000..7a22fcd
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/SecondaryIndexTable.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase96;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.apache.tephra.distributed.TransactionServiceClient;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A Transactional SecondaryIndexTable.
+ */
+public class SecondaryIndexTable {
+ private byte[] secondaryIndex;
+ private TransactionAwareHTable transactionAwareHTable;
+ private TransactionAwareHTable secondaryIndexTable;
+ private TransactionContext transactionContext;
+ private final TableName secondaryIndexTableName;
+ private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
+ private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
+ private static final byte[] DELIMITER = new byte[] {0};
+
+ public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
+ byte[] secondaryIndex) {
+ secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
+ HTable secondaryIndexHTable = null;
+ HBaseAdmin hBaseAdmin = null;
+ try {
+ hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
+ if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
+ hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
+ }
+ secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ } finally {
+ try {
+ hBaseAdmin.close();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ }
+
+ this.secondaryIndex = secondaryIndex;
+ this.transactionAwareHTable = new TransactionAwareHTable(hTable);
+ this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
+ this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
+ secondaryIndexTable);
+ }
+
+ public Result get(Get get) throws IOException {
+ return get(Collections.singletonList(get))[0];
+ }
+
+ public Result[] get(List<Get> gets) throws IOException {
+ try {
+ transactionContext.start();
+ Result[] result = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return result;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public Result[] getByIndex(byte[] value) throws IOException {
+ try {
+ transactionContext.start();
+ Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
+ scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
+ ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
+
+ ArrayList<Get> gets = new ArrayList<Get>();
+ for (Result result : indexScanner) {
+ for (Cell cell : result.listCells()) {
+ gets.add(new Get(cell.getValue()));
+ }
+ }
+ Result[] results = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return results;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public void put(Put put) throws IOException {
+ put(Collections.singletonList(put));
+ }
+
+
+ public void put(List<Put> puts) throws IOException {
+ try {
+ transactionContext.start();
+ ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
+ for (Put put : puts) {
+ List<Put> indexPuts = new ArrayList<Put>();
+ Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
+ for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
+ for (KeyValue value : family.getValue()) {
+ if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
+ secondaryIndex, 0, secondaryIndex.length)) {
+ byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
+ Bytes.add(value.getValue(), DELIMITER,
+ value.getRow()));
+ Put indexPut = new Put(secondaryRow);
+ indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
+ indexPuts.add(indexPut);
+ }
+ }
+ }
+ secondaryIndexPuts.addAll(indexPuts);
+ }
+ transactionAwareHTable.put(puts);
+ secondaryIndexTable.put(secondaryIndexPuts);
+ transactionContext.finish();
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ }
+}
[50/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionAwareTable.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionAwareTable.java b/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionAwareTable.java
deleted file mode 100644
index 11a5b55..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionAwareTable.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Objects;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.common.primitives.Bytes;
-import com.google.common.primitives.UnsignedBytes;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Base class for all the common parts of the HBase version-specific {@code TransactionAwareHTable}
- * implementations.
- */
-public abstract class AbstractTransactionAwareTable implements TransactionAware {
- protected final TransactionCodec txCodec;
- // map of write pointers to change set associated with each
- protected final Map<Long, Set<ActionChange>> changeSets;
- protected final TxConstants.ConflictDetection conflictLevel;
- protected Transaction tx;
- protected boolean allowNonTransactional;
-
- public AbstractTransactionAwareTable(TxConstants.ConflictDetection conflictLevel, boolean allowNonTransactional) {
- this.conflictLevel = conflictLevel;
- this.allowNonTransactional = allowNonTransactional;
- this.txCodec = new TransactionCodec();
- this.changeSets = Maps.newHashMap();
- }
-
- /**
- * True if the instance allows non-transaction operations.
- * @return
- */
- public boolean getAllowNonTransactional() {
- return this.allowNonTransactional;
- }
-
- /**
- * Set whether the instance allows non-transactional operations.
- * @param allowNonTransactional
- */
- public void setAllowNonTransactional(boolean allowNonTransactional) {
- this.allowNonTransactional = allowNonTransactional;
- }
-
- @Override
- public void startTx(Transaction tx) {
- this.tx = tx;
- }
-
- @Override
- public void updateTx(Transaction tx) {
- this.tx = tx;
- }
-
- @Override
- public Collection<byte[]> getTxChanges() {
- if (conflictLevel == TxConstants.ConflictDetection.NONE) {
- return Collections.emptyList();
- }
-
- Collection<byte[]> txChanges = new TreeSet<byte[]>(UnsignedBytes.lexicographicalComparator());
- for (Set<ActionChange> changeSet : changeSets.values()) {
- for (ActionChange change : changeSet) {
- txChanges.add(getChangeKey(change.getRow(), change.getFamily(), change.getQualifier()));
- }
- }
- return txChanges;
- }
-
- public byte[] getChangeKey(byte[] row, byte[] family, byte[] qualifier) {
- byte[] key;
- switch (conflictLevel) {
- case ROW:
- key = Bytes.concat(getTableKey(), row);
- break;
- case COLUMN:
- key = Bytes.concat(getTableKey(), row, family, qualifier);
- break;
- case NONE:
- throw new IllegalStateException("NONE conflict detection does not support change keys");
- default:
- throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
- }
- return key;
- }
-
- @Override
- public boolean commitTx() throws Exception {
- return doCommit();
- }
-
- /**
- * Commits any pending writes by flushing the wrapped {@code HTable} instance.
- */
- protected abstract boolean doCommit() throws IOException;
-
- @Override
- public void postTxCommit() {
- tx = null;
- changeSets.clear();
- }
-
- @Override
- public String getTransactionAwareName() {
- return new String(getTableKey(), Charsets.UTF_8);
- }
-
- /**
- * Returns the table name to use as a key prefix for the transaction change set.
- */
- protected abstract byte[] getTableKey();
-
- @Override
- public boolean rollbackTx() throws Exception {
- return doRollback();
- }
-
- /**
- * Rolls back any persisted changes from the transaction by issuing offsetting deletes to the
- * wrapped {@code HTable} instance. How this is handled will depend on the delete API exposed
- * by the specific version of HBase.
- */
- protected abstract boolean doRollback() throws Exception;
-
- protected void addToChangeSet(byte[] row, byte[] family, byte[] qualifier) {
- long currentWritePointer = tx.getWritePointer();
- Set<ActionChange> changeSet = changeSets.get(currentWritePointer);
- if (changeSet == null) {
- changeSet = Sets.newHashSet();
- changeSets.put(currentWritePointer, changeSet);
- }
- switch (conflictLevel) {
- case ROW:
- case NONE:
- // with ROW or NONE conflict detection, we still need to track changes per-family, since this
- // is the granularity at which we will issue deletes for rollback
- changeSet.add(new ActionChange(row, family));
- break;
- case COLUMN:
- changeSet.add(new ActionChange(row, family, qualifier));
- break;
- default:
- throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
- }
- }
-
- /**
- * Record of each transaction that causes a change. This reference is used to rollback
- * any operation upon failure.
- */
- protected class ActionChange {
- private final byte[] row;
- private final byte[] family;
- private final byte[] qualifier;
-
- public ActionChange(byte[] row, byte[] family) {
- this(row, family, null);
- }
-
- public ActionChange(byte[] row, byte[] family, byte[] qualifier) {
- this.row = row;
- this.family = family;
- this.qualifier = qualifier;
- }
-
- public byte[] getRow() {
- return row;
- }
-
- public byte[] getFamily() {
- return family;
- }
-
- public byte[] getQualifier() {
- return qualifier;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == null || o.getClass() != this.getClass()) {
- return false;
- }
-
- if (o == this) {
- return true;
- }
-
- ActionChange other = (ActionChange) o;
- return Objects.equal(this.row, other.row) &&
- Objects.equal(this.family, other.family) &&
- Objects.equal(this.qualifier, other.qualifier);
- }
-
- @Override
- public int hashCode() {
- int result = Arrays.hashCode(row);
- result = 31 * result + (family != null ? Arrays.hashCode(family) : 0);
- result = 31 * result + (qualifier != null ? Arrays.hashCode(qualifier) : 0);
- return result;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionExecutor.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionExecutor.java b/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionExecutor.java
deleted file mode 100644
index f2a28a8..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/AbstractTransactionExecutor.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-
-/**
- * Provides implementation of asynchronous methods of {@link TransactionExecutor} by delegating their execution
- * to respective synchronous methods via provided {@link ExecutorService}.
- */
-public abstract class AbstractTransactionExecutor implements TransactionExecutor {
- private final ListeningExecutorService executorService;
-
- protected AbstractTransactionExecutor(ExecutorService executorService) {
- this.executorService = MoreExecutors.listeningDecorator(executorService);
- }
-
- @Override
- public <I, O> O executeUnchecked(Function<I, O> function, I input) {
- try {
- return execute(function, input);
- } catch (TransactionFailureException e) {
- throw Throwables.propagate(e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public <I> void executeUnchecked(Procedure<I> procedure, I input) {
- try {
- execute(procedure, input);
- } catch (TransactionFailureException e) {
- throw Throwables.propagate(e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public <O> O executeUnchecked(Callable<O> callable) {
- try {
- return execute(callable);
- } catch (TransactionFailureException e) {
- throw Throwables.propagate(e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public void executeUnchecked(Subroutine subroutine) {
- try {
- execute(subroutine);
- } catch (TransactionFailureException e) {
- throw Throwables.propagate(e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw Throwables.propagate(e);
- }
- }
-
- @Override
- public <I, O> ListenableFuture<O> submit(final Function<I, O> function, final I input) {
- return executorService.submit(new Callable<O>() {
- @Override
- public O call() throws Exception {
- return execute(function, input);
- }
- });
- }
-
- @Override
- public <I> ListenableFuture<?> submit(final Procedure<I> procedure, final I input) {
- return executorService.submit(new Callable<Object>() {
- @Override
- public I call() throws Exception {
- execute(procedure, input);
- return null;
- }
- });
- }
-
- @Override
- public <O> ListenableFuture<O> submit(final Callable<O> callable) {
- return executorService.submit(new Callable<O>() {
- @Override
- public O call() throws Exception {
- return execute(callable);
- }
- });
- }
-
- @Override
- public ListenableFuture<?> submit(final Subroutine subroutine) {
- return executorService.submit(new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- execute(subroutine);
- return null;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/ChangeId.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/ChangeId.java b/tephra-core/src/main/java/co/cask/tephra/ChangeId.java
deleted file mode 100644
index 8413256..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/ChangeId.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import java.util.Arrays;
-
-/**
- * Represents a row key from a data set changed as part of a transaction.
- */
-public final class ChangeId {
- private final byte[] key;
- private final int hash;
-
- public ChangeId(byte[] bytes) {
- key = bytes;
- hash = Arrays.hashCode(bytes);
- }
-
- public byte[] getKey() {
- return key;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
- if (o == null || o.getClass() != ChangeId.class) {
- return false;
- }
- ChangeId other = (ChangeId) o;
- return hash == other.hash && Arrays.equals(key, other.key);
- }
-
- @Override
- public int hashCode() {
- return hash;
- }
-
- @Override
- public String toString() {
- return toStringBinary(key, 0, key.length);
- }
-
- // Copy from Bytes.toStringBinary so that we don't need direct dependencies on Bytes.
- private String toStringBinary(byte [] b, int off, int len) {
- StringBuilder result = new StringBuilder();
- for (int i = off; i < off + len; ++i) {
- int ch = b[i] & 0xFF;
- if ((ch >= '0' && ch <= '9')
- || (ch >= 'A' && ch <= 'Z')
- || (ch >= 'a' && ch <= 'z')
- || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0) {
- result.append((char) ch);
- } else {
- result.append(String.format("\\x%02X", ch));
- }
- }
- return result.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/DefaultTransactionExecutor.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/DefaultTransactionExecutor.java b/tephra-core/src/main/java/co/cask/tephra/DefaultTransactionExecutor.java
deleted file mode 100644
index f85dd6b..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/DefaultTransactionExecutor.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Utility class that encapsulates the transaction life cycle over a given set of
- * transaction-aware datasets. The executor can be reused across multiple invocations
- * of the execute() method. However, it is not thread-safe for concurrent execution.
- * <p>
- * Transaction execution will be retries according to specified in constructor {@link RetryStrategy}.
- * By default {@link RetryOnConflictStrategy} is used with max 20 retries and 100 ms between retries.
- * </p>
- */
-public class DefaultTransactionExecutor extends AbstractTransactionExecutor {
-
- private final Collection<TransactionAware> txAwares;
- private final TransactionSystemClient txClient;
- private final RetryStrategy retryStrategy;
-
- /**
- * Convenience constructor, has same affect as {@link #DefaultTransactionExecutor(TransactionSystemClient, Iterable)}
- */
- public DefaultTransactionExecutor(TransactionSystemClient txClient, TransactionAware... txAwares) {
- this(txClient, Arrays.asList(txAwares));
- }
-
-
- public DefaultTransactionExecutor(TransactionSystemClient txClient,
- Iterable<TransactionAware> txAwares,
- RetryStrategy retryStrategy) {
-
- super(MoreExecutors.sameThreadExecutor());
- this.txAwares = ImmutableList.copyOf(txAwares);
- this.txClient = txClient;
- this.retryStrategy = retryStrategy;
- }
-
- /**
- * Constructor for a transaction executor.
- */
- @Inject
- public DefaultTransactionExecutor(TransactionSystemClient txClient, @Assisted Iterable<TransactionAware> txAwares) {
- this(txClient, txAwares, RetryStrategies.retryOnConflict(20, 100));
- }
-
- @Override
- public <I, O> O execute(Function<I, O> function, I input) throws TransactionFailureException, InterruptedException {
- return executeWithRetry(function, input);
- }
-
- @Override
- public <I> void execute(final Procedure<I> procedure, I input)
- throws TransactionFailureException, InterruptedException {
-
- execute(new Function<I, Void>() {
- @Override
- public Void apply(I input) throws Exception {
- procedure.apply(input);
- return null;
- }
- }, input);
- }
-
- @Override
- public <O> O execute(final Callable<O> callable) throws TransactionFailureException, InterruptedException {
- return execute(new Function<Void, O>() {
- @Override
- public O apply(Void input) throws Exception {
- return callable.call();
- }
- }, null);
- }
-
- @Override
- public void execute(final Subroutine subroutine) throws TransactionFailureException, InterruptedException {
- execute(new Function<Void, Void>() {
- @Override
- public Void apply(Void input) throws Exception {
- subroutine.apply();
- return null;
- }
- }, null);
- }
-
- private <I, O> O executeWithRetry(Function<I, O> function, I input)
- throws TransactionFailureException, InterruptedException {
-
- int retries = 0;
- while (true) {
- try {
- return executeOnce(function, input);
- } catch (TransactionFailureException e) {
- long delay = retryStrategy.nextRetry(e, ++retries);
-
- if (delay < 0) {
- throw e;
- }
-
- if (delay > 0) {
- TimeUnit.MILLISECONDS.sleep(delay);
- }
- }
- }
-
- }
-
- private <I, O> O executeOnce(Function<I, O> function, I input) throws TransactionFailureException {
- TransactionContext txContext = new TransactionContext(txClient, txAwares);
- txContext.start();
- O o = null;
- try {
- o = function.apply(input);
- } catch (Throwable e) {
- txContext.abort(new TransactionFailureException("Transaction function failure for transaction. ", e));
- // abort will throw
- }
- // will throw if smth goes wrong
- txContext.finish();
- return o;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/InvalidTruncateTimeException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/InvalidTruncateTimeException.java b/tephra-core/src/main/java/co/cask/tephra/InvalidTruncateTimeException.java
deleted file mode 100644
index b4576ee..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/InvalidTruncateTimeException.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Thrown when truncate invalid list is called with a time, and when there are in-progress transactions that
- * were started before the given time.
- */
-public class InvalidTruncateTimeException extends Exception {
- public InvalidTruncateTimeException(String s) {
- super(s);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/NoRetryStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/NoRetryStrategy.java b/tephra-core/src/main/java/co/cask/tephra/NoRetryStrategy.java
deleted file mode 100644
index 30e9ceb..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/NoRetryStrategy.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Does no retries
- */
-public class NoRetryStrategy implements RetryStrategy {
- public static final RetryStrategy INSTANCE = new NoRetryStrategy();
-
- private NoRetryStrategy() {}
-
- @Override
- public long nextRetry(TransactionFailureException reason, int failureCount) {
- return -1;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/RetryOnConflictStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/RetryOnConflictStrategy.java b/tephra-core/src/main/java/co/cask/tephra/RetryOnConflictStrategy.java
deleted file mode 100644
index 04b9f75..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/RetryOnConflictStrategy.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Retries transaction execution when transaction fails with {@link TransactionConflictException}.
- */
-public class RetryOnConflictStrategy implements RetryStrategy {
- private final int maxRetries;
- private final long retryDelay;
-
- public RetryOnConflictStrategy(int maxRetries, long retryDelay) {
- this.maxRetries = maxRetries;
- this.retryDelay = retryDelay;
- }
-
- @Override
- public long nextRetry(TransactionFailureException reason, int failureCount) {
- if (reason instanceof TransactionConflictException) {
- return failureCount > maxRetries ? -1 : retryDelay;
- } else {
- return -1;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/RetryStrategies.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/RetryStrategies.java b/tephra-core/src/main/java/co/cask/tephra/RetryStrategies.java
deleted file mode 100644
index d1de0e5..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/RetryStrategies.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Collection of {@link RetryStrategy}s.
- */
-public final class RetryStrategies {
- private RetryStrategies() {}
-
- /**
- * @param maxRetries max number of retries
- * @param delayInMs delay between retries in milliseconds
- * @return RetryStrategy that retries transaction execution when transaction fails with
- * {@link TransactionConflictException}
- */
- public static RetryStrategy retryOnConflict(int maxRetries, long delayInMs) {
- return new RetryOnConflictStrategy(maxRetries, delayInMs);
- }
-
- public static RetryStrategy noRetries() {
- return NoRetryStrategy.INSTANCE;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/RetryStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/RetryStrategy.java b/tephra-core/src/main/java/co/cask/tephra/RetryStrategy.java
deleted file mode 100644
index e651d0e..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/RetryStrategy.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Retry strategy for failed transactions
- */
-public interface RetryStrategy {
- /**
- * Returns the number of milliseconds to wait before retrying the operation.
- *
- * @param reason Reason for transaction failure.
- * @param failureCount Number of times that the request has been failed.
- * @return Number of milliseconds to wait before retrying the operation. Returning {@code 0} means
- * retry it immediately, while negative means abort the operation.
- */
- long nextRetry(TransactionFailureException reason, int failureCount);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionAdmin.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionAdmin.java b/tephra-core/src/main/java/co/cask/tephra/TransactionAdmin.java
deleted file mode 100644
index 25ccbfb..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionAdmin.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.zookeeper.ZKClientService;
-
-import java.io.PrintStream;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Allows calling some methods on {@link TransactionManager} from command line.
- */
-public class TransactionAdmin {
- private static final String OPT_TRUNCATE_INVALID_TX = "--truncate-invalid-tx";
- private static final String OPT_TRUNCATE_INVALID_TX_BEFORE = "--truncate-invalid-tx-before";
- private static final String OPT_GET_INVALID_TX_SIZE = "--get-invalid-tx-size";
-
- private final PrintStream out;
- private final PrintStream err;
-
- public static void main(String[] args) {
- TransactionAdmin txAdmin = new TransactionAdmin(System.out, System.err);
- int status = txAdmin.doMain(args, new Configuration());
- System.exit(status);
- }
-
- public TransactionAdmin(PrintStream out, PrintStream err) {
- this.out = out;
- this.err = err;
- }
-
- @VisibleForTesting
- int doMain(String[] args, Configuration conf) {
- if (args.length < 1) {
- printUsage();
- return 1;
- }
-
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- new TransactionModules().getDistributedModules(),
- new TransactionClientModule()
- );
-
- ZKClientService zkClient = injector.getInstance(ZKClientService.class);
- zkClient.startAndWait();
-
- try {
- TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
- String option = args[0];
-
- if (option.equals(OPT_TRUNCATE_INVALID_TX)) {
- if (args.length != 2) {
- printUsage();
- return 1;
- }
- Set<Long> txIds;
- try {
- txIds = parseTxIds(args[1]);
- } catch (NumberFormatException e) {
- err.println("NumberFormatException: " + e.getMessage());
- return 1;
- }
- if (!txIds.isEmpty()) {
- out.println("Invalid list size before truncation: " + txClient.getInvalidSize());
- txClient.truncateInvalidTx(txIds);
- out.println("Invalid list size after truncation: " + txClient.getInvalidSize());
- }
- } else if (option.equals(OPT_TRUNCATE_INVALID_TX_BEFORE)) {
- if (args.length != 2) {
- printUsage();
- return 1;
- }
- try {
- long time = Long.parseLong(args[1]);
- out.println("Invalid list size before truncation: " + txClient.getInvalidSize());
- txClient.truncateInvalidTxBefore(time);
- out.println("Invalid list size after truncation: " + txClient.getInvalidSize());
- } catch (InvalidTruncateTimeException e) {
- err.println(e.getMessage());
- return 1;
- } catch (NumberFormatException e) {
- err.println("NumberFormatException: " + e.getMessage());
- return 1;
- }
- } else if (option.equals(OPT_GET_INVALID_TX_SIZE)) {
- if (args.length != 1) {
- printUsage();
- return 1;
- }
- out.println("Invalid list size: " + txClient.getInvalidSize());
- } else {
- printUsage();
- return 1;
- }
- } finally {
- zkClient.stopAndWait();
- }
- return 0;
- }
-
- private Set<Long> parseTxIds(String option) throws NumberFormatException {
- Set<Long> txIds = Sets.newHashSet();
- for (String str : Splitter.on(',').split(option)) {
- txIds.add(Long.parseLong(str));
- }
- return txIds;
- }
-
- private void printUsage() {
- String programName = TransactionAdmin.class.getSimpleName();
- String spaces = " ";
- List<String> options = Lists.newArrayList();
- options.add(join("Usage: "));
- options.add(join(spaces, programName, OPT_TRUNCATE_INVALID_TX, "<tx1,tx2,...>"));
- options.add(join(spaces, programName, OPT_TRUNCATE_INVALID_TX_BEFORE, "<time in secs>"));
- options.add(join(spaces, programName, OPT_GET_INVALID_TX_SIZE));
-
- String usage = Joiner.on(System.getProperty("line.separator")).join(options);
- err.println(usage);
- }
-
- private static String join(String... args) {
- return Joiner.on(" ").join(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionCodec.java b/tephra-core/src/main/java/co/cask/tephra/TransactionCodec.java
deleted file mode 100644
index 403b1ec..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionCodec.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.distributed.TransactionConverterUtils;
-import co.cask.tephra.distributed.thrift.TTransaction;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-
-import java.io.IOException;
-
-/**
- * Handles serialization and deserialization of {@link co.cask.tephra.Transaction} instances to and from {@code byte[]}.
- */
-public class TransactionCodec {
-
- public TransactionCodec() {
- }
-
- public byte[] encode(Transaction tx) throws IOException {
- TTransaction thriftTx = TransactionConverterUtils.wrap(tx);
- TSerializer serializer = new TSerializer();
- try {
- return serializer.serialize(thriftTx);
- } catch (TException te) {
- throw new IOException(te);
- }
- }
-
- public Transaction decode(byte[] encoded) throws IOException {
- TTransaction thriftTx = new TTransaction();
- TDeserializer deserializer = new TDeserializer();
- try {
- deserializer.deserialize(thriftTx, encoded);
- return TransactionConverterUtils.unwrap(thriftTx);
- } catch (TException te) {
- throw new IOException(te);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionContext.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionContext.java b/tephra-core/src/main/java/co/cask/tephra/TransactionContext.java
deleted file mode 100644
index 5b31ebe..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionContext.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import javax.annotation.Nullable;
-
-/**
- * Utility class that encapsulates the transaction life cycle over a given set of
- * transaction-aware datasets. It is not thread-safe for concurrent execution.
- */
-public class TransactionContext {
-
- private static final Logger LOG = LoggerFactory.getLogger(TransactionContext.class);
-
- private final Collection<TransactionAware> txAwares;
- private final TransactionSystemClient txClient;
-
- private Transaction currentTx;
-
- public TransactionContext(TransactionSystemClient txClient, TransactionAware... txAwares) {
- this(txClient, ImmutableList.copyOf(txAwares));
- }
-
- public TransactionContext(TransactionSystemClient txClient, Iterable<TransactionAware> txAwares) {
- // Use a set to avoid adding the same TransactionAware twice and to make removal faster.
- // Use a linked hash set so that insertion order is preserved (same behavior as when it was using a List).
- this.txAwares = Sets.newLinkedHashSet(txAwares);
- this.txClient = txClient;
- }
-
- /**
- * Adds a new transaction-aware to participate in the transaction.
- * @param txAware the new transaction-aware
- */
- public boolean addTransactionAware(TransactionAware txAware) {
- // If the txAware is newly added, call startTx as well if there is an active transaction
- boolean added = txAwares.add(txAware);
- if (added && currentTx != null) {
- txAware.startTx(currentTx);
- }
- return added;
- }
-
- /**
- * Removes a {@link TransactionAware} and withdraws from participation in the transaction.
- * Withdrawal is only allowed if there is no active transaction.
- *
- * @param txAware the {@link TransactionAware} to be removed
- * @return true if the given {@link TransactionAware} is removed; false otherwise.
- * @throws IllegalStateException if there is an active transaction going on with this TransactionContext.
- */
- public boolean removeTransactionAware(TransactionAware txAware) {
- Preconditions.checkState(currentTx == null, "Cannot remove TransactionAware while there is an active transaction.");
- return txAwares.remove(txAware);
- }
-
- /**
- * Starts a new transaction. Calling this will initiate a new transaction using the {@link TransactionSystemClient},
- * and pass the returned transaction to {@link TransactionAware#startTx(Transaction)} for each registered
- * TransactionAware. If an exception is encountered, the transaction will be aborted and a
- * {@code TransactionFailureException} wrapping the root cause will be thrown.
- *
- * @throws TransactionFailureException if an exception occurs starting the transaction with any registered
- * TransactionAware
- */
- public void start() throws TransactionFailureException {
- currentTx = txClient.startShort();
- for (TransactionAware txAware : txAwares) {
- try {
- txAware.startTx(currentTx);
- } catch (Throwable e) {
- String message = String.format("Unable to start transaction-aware '%s' for transaction %d. ",
- txAware.getTransactionAwareName(), currentTx.getTransactionId());
- LOG.warn(message, e);
- txClient.abort(currentTx);
- throw new TransactionFailureException(message, e);
- }
- }
- }
-
- /**
- * Commits the current transaction. This will: check for any conflicts, based on the change set aggregated from
- * all registered {@link TransactionAware} instances; flush any pending writes from the {@code TransactionAware}s;
- * commit the current transaction with the {@link TransactionSystemClient}; and clear the current transaction state.
- *
- * @throws TransactionConflictException if a conflict is detected with a recently committed transaction
- * @throws TransactionFailureException if an error occurs while committing
- */
- public void finish() throws TransactionFailureException {
- Preconditions.checkState(currentTx != null, "Cannot finish tx that has not been started");
- // each of these steps will abort and rollback the tx in case if errors, and throw an exception
- checkForConflicts();
- persist();
- commit();
- postCommit();
- currentTx = null;
- }
-
- /**
- * Aborts the given transaction, and rolls back all data set changes. If rollback fails,
- * the transaction is invalidated. If an exception is caught during rollback, the exception
- * is rethrown wrapped in a TransactionFailureException, after all remaining TransactionAwares have
- * completed rollback.
- *
- * @throws TransactionFailureException for any exception that is encountered.
- */
- public void abort() throws TransactionFailureException {
- abort(null);
- }
-
- /**
- * Checkpoints the current transaction by flushing any pending writes for the registered {@link TransactionAware}
- * instances, and obtaining a new current write pointer for the transaction. By performing a checkpoint,
- * the client can ensure that all previous writes were flushed and are visible. By default, the current write
- * pointer for the transaction is also visible. The current write pointer can be excluded from read
- * operations by calling {@link Transaction#setVisibility(Transaction.VisibilityLevel)} with the visibility level set
- * to {@link Transaction.VisibilityLevel#SNAPSHOT_EXCLUDE_CURRENT} on the {@link Transaction} instance created
- * by the checkpoint call, which can be retrieved by calling {@link #getCurrentTransaction()}.
- *
- * After the checkpoint operation is performed, the updated
- * {@link Transaction} instance will be passed to {@link TransactionAware#startTx(Transaction)} for each
- * registered {@code TransactionAware} instance.
- *
- * @throws TransactionFailureException if an error occurs while performing the checkpoint
- */
- public void checkpoint() throws TransactionFailureException {
- Preconditions.checkState(currentTx != null, "Cannot checkpoint tx that has not been started");
- persist();
- try {
- currentTx = txClient.checkpoint(currentTx);
- // update the current transaction with all TransactionAwares
- for (TransactionAware txAware : txAwares) {
- txAware.updateTx(currentTx);
- }
- } catch (TransactionNotInProgressException e) {
- String message = String.format("Transaction %d is not in progress.", currentTx.getTransactionId());
- LOG.warn(message, e);
- abort(new TransactionFailureException(message, e));
- // abort will throw that exception
- } catch (Throwable e) {
- String message = String.format("Exception from checkpoint for transaction %d.", currentTx.getTransactionId());
- LOG.warn(message, e);
- abort(new TransactionFailureException(message, e));
- // abort will throw that exception
- }
- }
-
- /**
- * Returns the current transaction or null if no transaction is currently in progress.
- */
- @Nullable
- public Transaction getCurrentTransaction() {
- return currentTx;
- }
-
- // CHECKSTYLE IGNORE "@throws" FOR 11 LINES
- /**
- * Aborts the given transaction, and rolls back all data set changes. If rollback fails,
- * the transaction is invalidated. If an exception is caught during rollback, the exception
- * is rethrown wrapped into a TransactionFailureException, after all remaining TransactionAwares have
- * completed rollback. If an existing exception is passed in, that exception is thrown in either
- * case, whether the rollback is successful or not. In other words, this method always throws the
- * first exception that it encounters.
- * @param cause the original exception that caused the abort
- * @throws TransactionFailureException for any exception that is encountered.
- */
- public void abort(TransactionFailureException cause) throws TransactionFailureException {
- if (currentTx == null) {
- // might be called by some generic exception handler even though already aborted/finished - we allow that
- return;
- }
- try {
- boolean success = true;
- for (TransactionAware txAware : txAwares) {
- try {
- if (!txAware.rollbackTx()) {
- success = false;
- }
- } catch (Throwable e) {
- String message = String.format("Unable to roll back changes in transaction-aware '%s' for transaction %d. ",
- txAware.getTransactionAwareName(), currentTx.getTransactionId());
- LOG.warn(message, e);
- if (cause == null) {
- cause = new TransactionFailureException(message, e);
- }
- success = false;
- }
- }
- if (success) {
- txClient.abort(currentTx);
- } else {
- txClient.invalidate(currentTx.getTransactionId());
- }
- if (cause != null) {
- throw cause;
- }
- } finally {
- currentTx = null;
- }
- }
-
- private void checkForConflicts() throws TransactionFailureException {
- Collection<byte[]> changes = Lists.newArrayList();
- for (TransactionAware txAware : txAwares) {
- try {
- changes.addAll(txAware.getTxChanges());
- } catch (Throwable e) {
- String message = String.format("Unable to retrieve changes from transaction-aware '%s' for transaction %d. ",
- txAware.getTransactionAwareName(), currentTx.getTransactionId());
- LOG.warn(message, e);
- abort(new TransactionFailureException(message, e));
- // abort will throw that exception
- }
- }
-
- boolean canCommit = false;
- try {
- canCommit = txClient.canCommit(currentTx, changes);
- } catch (TransactionNotInProgressException e) {
- String message = String.format("Transaction %d is not in progress.", currentTx.getTransactionId());
- LOG.warn(message, e);
- abort(new TransactionFailureException(message, e));
- // abort will throw that exception
- } catch (Throwable e) {
- String message = String.format("Exception from canCommit for transaction %d.", currentTx.getTransactionId());
- LOG.warn(message, e);
- abort(new TransactionFailureException(message, e));
- // abort will throw that exception
- }
- if (!canCommit) {
- String message = String.format("Conflict detected for transaction %d.", currentTx.getTransactionId());
- abort(new TransactionConflictException(message));
- // abort will throw
- }
- }
-
- private void persist() throws TransactionFailureException {
- for (TransactionAware txAware : txAwares) {
- boolean success;
- Throwable cause = null;
- try {
- success = txAware.commitTx();
- } catch (Throwable e) {
- success = false;
- cause = e;
- }
- if (!success) {
- String message = String.format("Unable to persist changes of transaction-aware '%s' for transaction %d. ",
- txAware.getTransactionAwareName(), currentTx.getTransactionId());
- if (cause == null) {
- LOG.warn(message);
- } else {
- LOG.warn(message, cause);
- }
- abort(new TransactionFailureException(message, cause));
- // abort will throw that exception
- }
- }
- }
-
- private void commit() throws TransactionFailureException {
- boolean commitSuccess = false;
- try {
- commitSuccess = txClient.commit(currentTx);
- } catch (TransactionNotInProgressException e) {
- String message = String.format("Transaction %d is not in progress.", currentTx.getTransactionId());
- LOG.warn(message, e);
- abort(new TransactionFailureException(message, e));
- // abort will throw that exception
- } catch (Throwable e) {
- String message = String.format("Exception from commit for transaction %d.", currentTx.getTransactionId());
- LOG.warn(message, e);
- abort(new TransactionFailureException(message, e));
- // abort will throw that exception
- }
- if (!commitSuccess) {
- String message = String.format("Conflict detected for transaction %d.", currentTx.getTransactionId());
- abort(new TransactionConflictException(message));
- // abort will throw
- }
- }
-
- private void postCommit() throws TransactionFailureException {
- TransactionFailureException cause = null;
- for (TransactionAware txAware : txAwares) {
- try {
- txAware.postTxCommit();
- } catch (Throwable e) {
- String message = String.format("Unable to perform post-commit in transaction-aware '%s' for transaction %d. ",
- txAware.getTransactionAwareName(), currentTx.getTransactionId());
- LOG.warn(message, e);
- cause = new TransactionFailureException(message, e);
- }
- }
- if (cause != null) {
- throw cause;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionCouldNotTakeSnapshotException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionCouldNotTakeSnapshotException.java b/tephra-core/src/main/java/co/cask/tephra/TransactionCouldNotTakeSnapshotException.java
deleted file mode 100644
index a98be14..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionCouldNotTakeSnapshotException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Throw when taking a snapshot fails.
- */
-public class TransactionCouldNotTakeSnapshotException extends Exception {
- public TransactionCouldNotTakeSnapshotException(String message) {
- super(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionExecutor.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionExecutor.java b/tephra-core/src/main/java/co/cask/tephra/TransactionExecutor.java
deleted file mode 100644
index a1cb188..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionExecutor.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import com.google.common.util.concurrent.ListenableFuture;
-
-import java.util.concurrent.Callable;
-
-/**
- * Utility that wraps the execution of a function into the context of a transaction.
- */
-// todo: implementations should throw different from TransactionFailureException in case of user code error?
-// todo: accept only Callable? Executors util has a way to convert everything to Callable...
-public interface TransactionExecutor {
-
- /**
- * A function is a class with a single method that takes an argument and returns a result.
- * @param <I> the type of the argument
- * @param <O> the type of the result
- */
- public interface Function<I, O> {
- O apply(I input) throws Exception;
- }
-
- /**
- * A procedure is a class with a single void method that takes an argument.
- * @param <I> the type of the argument
- */
- public interface Procedure<I> {
- void apply(I input) throws Exception;
- }
-
- /**
- * A subroutine is a class with a single void method without arguments.
- */
- public interface Subroutine {
- void apply() throws Exception;
- }
-
- // Due to a bug in checkstyle, it would emit false positives here of the form
- // "Unable to get class information for @throws tag '<exn>' (...)".
- // This comment disables that check up to the corresponding ON comments below
-
- // CHECKSTYLE OFF: @throws
-
- /**
- * Execute a function under transactional semantics. A transaction is started and all datasets
- * are initialized with the transaction. Then the passed function is executed, the transaction
- * is committed, and the function return value is returned as the return value of this method.
- * If any exception is caught, the transaction is aborted and the original exception is rethrown,
- * wrapped into a TransactionFailureException. If the transaction fails due to a write conflict,
- * a TransactionConflictException is thrown.
- * @param function the function to execute
- * @param input the input parameter for the function
- * @param <I> the input type of the function
- * @param <O> the result type of the function
- * @return the function's return value
- * @throws TransactionConflictException if there is a write conflict with another transaction.
- * @throws TransactionFailureException if any exception is caught, be it from the function or from the datasets.
- */
- <I, O> O execute(Function<I, O> function, I input) throws TransactionFailureException, InterruptedException;
-
- // CHECKSTYLE ON
-
- /**
- * Like {@link #execute(Function, Object)} but without a return value.
- */
- <I> void execute(Procedure<I> procedure, I input) throws TransactionFailureException, InterruptedException;
-
- /**
- * Like {@link #execute(Function, Object)} but the callable has no argument.
- */
- <O> O execute(Callable<O> callable) throws TransactionFailureException, InterruptedException;
-
- /**
- * Like {@link #execute(Function, Object)} but without argument or return value.
- */
- void execute(Subroutine subroutine) throws TransactionFailureException, InterruptedException;
-
- /**
- * Same as {@link #execute(Function, Object)} but
- * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
- */
- <I, O> O executeUnchecked(Function<I, O> function, I input);
-
- /**
- * Same as {@link #execute(Procedure, Object)} but
- * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
- */
- <I> void executeUnchecked(Procedure<I> procedure, I input);
-
- /**
- * Same as {@link #execute(Callable)} but
- * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
- */
- <O> O executeUnchecked(Callable<O> callable);
-
- /**
- * Same as {@link #execute(Subroutine)} but
- * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
- */
- void executeUnchecked(Subroutine subroutine);
-
- /**
- * Same as {@link #execute(Function, Object)} but executes asynchronously
- */
- <I, O> ListenableFuture<O> submit(Function<I, O> function, I input);
-
- /**
- * Same as {@link #execute(Procedure, Object)} but executes asynchronously
- */
- <I> ListenableFuture<?> submit(Procedure<I> procedure, I input);
-
- /**
- * Same as {@link #execute(Callable)} but executes asynchronously
- */
- <O> ListenableFuture<O> submit(Callable<O> callable);
-
- /**
- * Same as {@link #execute(Subroutine)} but executes asynchronously
- */
- ListenableFuture<?> submit(Subroutine subroutine);
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionExecutorFactory.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionExecutorFactory.java b/tephra-core/src/main/java/co/cask/tephra/TransactionExecutorFactory.java
deleted file mode 100644
index 4f30478..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionExecutorFactory.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * A factory for transaction executors.
- */
-public interface TransactionExecutorFactory {
-
- TransactionExecutor createExecutor(Iterable<TransactionAware> txAwares);
-
-}
[16/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/TransactionAwareHTableTest.java b/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/TransactionAwareHTableTest.java
deleted file mode 100644
index 2e330f8..0000000
--- a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/TransactionAwareHTableTest.java
+++ /dev/null
@@ -1,1552 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase98;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionConflictException;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase98.coprocessor.TransactionProcessor;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.LongComparator;
-import org.apache.hadoop.hbase.filter.ValueFilter;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Tests for TransactionAwareHTables.
- */
-public class TransactionAwareHTableTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
-
- private static HBaseTestingUtility testUtil;
- private static HBaseAdmin hBaseAdmin;
- private static TransactionStateStorage txStateStorage;
- private static TransactionManager txManager;
- private static Configuration conf;
- private TransactionContext transactionContext;
- private TransactionAwareHTable transactionAwareHTable;
- private HTable hTable;
-
- private static final class TestBytes {
- private static final byte[] table = Bytes.toBytes("testtable");
- private static final byte[] family = Bytes.toBytes("f1");
- private static final byte[] family2 = Bytes.toBytes("f2");
- private static final byte[] qualifier = Bytes.toBytes("col1");
- private static final byte[] qualifier2 = Bytes.toBytes("col2");
- private static final byte[] row = Bytes.toBytes("row");
- private static final byte[] row2 = Bytes.toBytes("row2");
- private static final byte[] row3 = Bytes.toBytes("row3");
- private static final byte[] row4 = Bytes.toBytes("row4");
- private static final byte[] value = Bytes.toBytes("value");
- private static final byte[] value2 = Bytes.toBytes("value2");
- private static final byte[] value3 = Bytes.toBytes("value3");
- }
-
- private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
-
- public static class TestRegionObserver extends BaseRegionObserver {
- @Override
- public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Put put, final WALEdit edit,
- final Durability durability) throws IOException {
- if (put.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Put should preserve attributes");
- }
- if (put.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
-
- @Override
- public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Delete delete, final WALEdit edit,
- final Durability durability) throws IOException {
- if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Delete should preserve attributes");
- }
- if (delete.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
- }
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- testUtil = new HBaseTestingUtility();
- conf = testUtil.getConfiguration();
-
- // Tune down the connection thread pool size
- conf.setInt("hbase.hconnection.threads.core", 5);
- conf.setInt("hbase.hconnection.threads.max", 10);
- // Tunn down handler threads in regionserver
- conf.setInt("hbase.regionserver.handler.count", 10);
-
- // Set to random port
- conf.setInt("hbase.master.port", 0);
- conf.setInt("hbase.master.info.port", 0);
- conf.setInt("hbase.regionserver.port", 0);
- conf.setInt("hbase.regionserver.info.port", 0);
-
- testUtil.startMiniCluster();
- hBaseAdmin = testUtil.getHBaseAdmin();
- txStateStorage = new InMemoryTransactionStateStorage();
- txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
- txManager.startAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- testUtil.shutdownMiniCluster();
- hBaseAdmin.close();
- }
-
- @Before
- public void setupBeforeTest() throws Exception {
- hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
- transactionAwareHTable = new TransactionAwareHTable(hTable);
- transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
- }
-
- @After
- public void shutdownAfterTest() throws IOException {
- hBaseAdmin.disableTable(TestBytes.table);
- hBaseAdmin.deleteTable(TestBytes.table);
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
- return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
- List<String> coprocessors) throws Exception {
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
- for (byte[] family : columnFamilies) {
- HColumnDescriptor columnDesc = new HColumnDescriptor(family);
- columnDesc.setMaxVersions(Integer.MAX_VALUE);
- columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
- desc.addFamily(columnDesc);
- }
- if (existingData) {
- desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
- }
- // Divide individually to prevent any overflow
- int priority = Coprocessor.PRIORITY_USER;
- desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
- // order in list is the same order that coprocessors will be invoked
- for (String coprocessor : coprocessors) {
- desc.addCoprocessor(coprocessor, null, ++priority, null);
- }
- hBaseAdmin.createTable(desc);
- testUtil.waitTableAvailable(tableName, 5000);
- return new HTable(testUtil.getConfiguration(), tableName);
- }
-
- /**
- * Test transactional put and get requests.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
-
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- /**
- * Test aborted put requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
-
- transactionContext.abort();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(value, null);
- }
-
- /**
- * Test transactional delete operations.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
- new byte[][]{TestBytes.family, TestBytes.family2});
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
-
- // test column delete
- // load 10 rows
- txContext.start();
- int rowCount = 10;
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
- }
- txTable.put(p);
- }
- txContext.finish();
-
- // verify loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertFalse(r.isEmpty());
- for (int j = 0; j < 10; j++) {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
- }
- }
- txContext.finish();
-
- // delete odds columns from odd rows and even columns from even rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- if (i % 2 == j % 2) {
- LOG.info("Deleting row={}, column={}", i, j);
- d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
- }
- }
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted columns
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertEquals(5, r.size());
- for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
- int col = Bytes.toInt(entry.getKey());
- LOG.info("Got row={}, col={}", i, col);
- // each row should only have the opposite mod (odd=even, even=odd)
- assertNotEquals(i % 2, col % 2);
- assertArrayEquals(TestBytes.value, entry.getValue());
- }
- }
- txContext.finish();
-
- // test family delete
- // load 10 rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("famrow" + i));
- p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
- txTable.put(p);
- }
- txContext.finish();
-
- // verify all loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(2, r.size());
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- txContext.finish();
-
- // delete family1 for even rows, family2 for odd rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("famrow" + i));
- d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted families
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(1, r.size());
- if (i % 2 == 0) {
- assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- } else {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- }
- txContext.finish();
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test that put and delete attributes are preserved
- *
- * @throws Exception
- */
- @Test
- public void testAttributesPreserved() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
- new byte[][]{TestBytes.family, TestBytes.family2}, false,
- Lists.newArrayList(TestRegionObserver.class.getName()));
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- // set an attribute on the put, TestRegionObserver will verify it still exists
- put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete, TestRegionObserver will verify it still exists
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test aborted transactional delete requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionalDelete() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
-
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row);
- transactionAwareHTable.delete(delete);
- transactionContext.abort();
-
- transactionContext.start();
- result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
- String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
- HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- // Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- // Start a tx, delete a column family and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
- txContext.abort();
-
- // Above operations should have no effect on the row, since they were aborted
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
- }
- }
-
- @Test
- public void testDeleteRollback() throws Exception {
- testDeleteRollback(TxConstants.ConflictDetection.ROW);
- testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
- testDeleteRollback(TxConstants.ConflictDetection.NONE);
- }
-
- @Test
- public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- txContext.start();
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- Result result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
-
- //Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- //Start a tx and scan all the col families to make sure none of them have delete markers
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
- }
- }
-
- @Test
- public void testRowDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][]{TestBytes.family, TestBytes.family2});
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
- try {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Test 1: full row delete
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // delete entire row
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.finish();
-
- // verify row is now empty
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertTrue(result.isEmpty());
-
- // verify row is empty for explicit column retrieval
- result = txTable.get(new Get(TestBytes.row)
- .addColumn(TestBytes.family, TestBytes.qualifier)
- .addFamily(TestBytes.family2));
- assertTrue(result.isEmpty());
-
- // verify row is empty for scan
- ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
- assertNull(scanner.next());
- scanner.close();
-
- // verify row is empty for scan with explicit column
- scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
- assertNull(scanner.next());
- scanner.close();
- txContext.finish();
-
- // write swapped values to one column per family
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
- txContext.finish();
-
- // verify new values appear
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertFalse(result.isEmpty());
- assertEquals(2, result.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
-
- scanner = txTable.getScanner(new Scan(TestBytes.row));
- Result result1 = scanner.next();
- assertNotNull(result1);
- assertFalse(result1.isEmpty());
- assertEquals(2, result1.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- scanner.close();
- txContext.finish();
-
- // Test 2: delete of first column family
- txContext.start();
- txTable.put(new Put(TestBytes.row2)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
- txContext.finish();
-
- txContext.start();
- Result fam1Result = txTable.get(new Get(TestBytes.row2));
- assertFalse(fam1Result.isEmpty());
- assertEquals(2, fam1Result.size());
- assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 3: delete of second column family
- txContext.start();
- txTable.put(new Put(TestBytes.row3)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
- txContext.finish();
-
- txContext.start();
- Result fam2Result = txTable.get(new Get(TestBytes.row3));
- assertFalse(fam2Result.isEmpty());
- assertEquals(2, fam2Result.size());
- assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 4: delete specific rows in a range
- txContext.start();
- for (int i = 0; i < 10; i++) {
- txTable.put(new Put(Bytes.toBytes("z" + i))
- .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
- .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
- }
- txContext.finish();
-
- txContext.start();
- // delete odd rows
- for (int i = 1; i < 10; i += 2) {
- txTable.delete(new Delete(Bytes.toBytes("z" + i)));
- }
- txContext.finish();
-
- txContext.start();
- int cnt = 0;
- ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
- Result res;
- while ((res = zScanner.next()) != null) {
- assertFalse(res.isEmpty());
- assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
- cnt += 2;
- }
-
- // Test 5: delete prior writes in the same transaction
- txContext.start();
- txTable.put(new Put(TestBytes.row4)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txTable.delete(new Delete(TestBytes.row4));
- txContext.finish();
-
- txContext.start();
- Result row4Result = txTable.get(new Get(TestBytes.row4));
- assertTrue(row4Result.isEmpty());
- txContext.finish();
- } finally {
- txTable.close();
- }
- }
-
- /**
- * Expect an exception since a transaction hasn't been started.
- *
- * @throws Exception
- */
- @Test(expected = IOException.class)
- public void testTransactionlessFailure() throws Exception {
- transactionAwareHTable.get(new Get(TestBytes.row));
- }
-
- /**
- * Tests that each transaction can see its own persisted writes, while not seeing writes from other
- * in-progress transactions.
- */
- @Test
- public void testReadYourWrites() throws Exception {
- // In-progress tx1: started before our main transaction
- HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
- TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
-
- // In-progress tx2: started while our main transaction is running
- HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
- TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
-
- // create an in-progress write that should be ignored
- byte[] col2 = Bytes.toBytes("col2");
- inprogressTxContext1.start();
- Put putCol2 = new Put(TestBytes.row);
- byte[] valueCol2 = Bytes.toBytes("writing in progress");
- putCol2.add(TestBytes.family, col2, valueCol2);
- txHTable1.put(putCol2);
-
- // start a tx and write a value to test reading in same tx
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] value = Bytes.toBytes("writing");
- put.add(TestBytes.family, TestBytes.qualifier, value);
- transactionAwareHTable.put(put);
-
- // test that a write from a tx started after the first is not visible
- inprogressTxContext2.start();
- Put put2 = new Put(TestBytes.row);
- byte[] value2 = Bytes.toBytes("writing2");
- put2.add(TestBytes.family, TestBytes.qualifier, value2);
- txHTable2.put(put2);
-
- Get get = new Get(TestBytes.row);
- Result row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
- Assert.assertNotNull(col1Value);
- Assert.assertArrayEquals(value, col1Value);
- // write from in-progress transaction should not be visible
- byte[] col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- // commit in-progress transaction, should still not be visible
- inprogressTxContext1.finish();
-
- get = new Get(TestBytes.row);
- row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- transactionContext.finish();
-
- inprogressTxContext2.abort();
- }
-
- @Test
- public void testRowLevelConflictDetection() throws Exception {
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- byte[] row1 = Bytes.toBytes("row1");
- byte[] row2 = Bytes.toBytes("row2");
- byte[] col1 = Bytes.toBytes("c1");
- byte[] col2 = Bytes.toBytes("c2");
- byte[] val1 = Bytes.toBytes("val1");
- byte[] val2 = Bytes.toBytes("val2");
-
- // test that concurrent writing to different rows succeeds
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
-
- // should be no conflicts
- txContext1.finish();
- txContext2.finish();
-
- transactionContext.start();
- Result res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
-
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // test that writing to different columns in the same row fails
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
-
- txContext2.start();
- txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row-level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- // should now be val2
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
-
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- // col2 should not be visible due to conflict
- assertNull(cell);
- transactionContext.finish();
-
- // test that writing to the same column in the same row fails
- txContext1.start();
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row and column level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- assertNotNull(cell);
- // should now be val1
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // verify change set that is being reported only on rows
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- Collection<byte[]> changeSet = txTable1.getTxChanges();
- assertNotNull(changeSet);
- assertEquals(2, changeSet.size());
- assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
- assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
- txContext1.finish();
- }
-
- @Test
- public void testNoneLevelConflictDetection() throws Exception {
- InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
-
- // overlapping writes to the same row + column should not conflict
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // changes should not be visible yet
- txContext2.start();
- Result row = txTable2.get(new Get(TestBytes.row));
- assertTrue(row.isEmpty());
-
- txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // both commits should succeed
- txContext1.finish();
- txContext2.finish();
-
- txContext1.start();
- row = txTable1.get(new Get(TestBytes.row));
- assertFalse(row.isEmpty());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
- txContext1.finish();
-
- // transaction abort should still rollback changes
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext1.abort();
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row2 = txTable2.get(new Get(TestBytes.row2));
- assertTrue(row2.isEmpty());
- txContext2.finish();
-
- // transaction invalidate should still make changes invisible
-
- txContext1.start();
- Transaction tx1 = txContext1.getCurrentTransaction();
- txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- assertNotNull(tx1);
- txClient.invalidate(tx1.getWritePointer());
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row3 = txTable2.get(new Get(TestBytes.row3));
- assertTrue(row3.isEmpty());
- txContext2.finish();
- }
-
- @Test
- public void testCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionContext.checkpoint();
- Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
- assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
- long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
- assertEquals(1, checkpointPtrs.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
-
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
- assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
- long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
- assertEquals(2, checkpointPtrs2.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
- assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
-
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // by default, all rows should be visible with Read-Your-Writes
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
-
- // when disabling current write pointer, only the previous checkpoints should be visible
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- Get get = new Get(TestBytes.row);
- verifyRow(transactionAwareHTable, get, TestBytes.value);
- get = new Get(TestBytes.row2);
- verifyRow(transactionAwareHTable, get, TestBytes.value2);
- get = new Get(TestBytes.row3);
- verifyRow(transactionAwareHTable, get, null);
-
- // test scan results excluding current write pointer
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
-
- Result row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row2, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNull(row);
- scanner.close();
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
-
- // commit transaction, verify writes are visible
- transactionContext.finish();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
- transactionContext.finish();
- }
-
- @Test
- public void testInProgressCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // check that writes are still not visible to other clients
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- txContext2.start();
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- txContext2.finish();
- txTable2.close();
-
- transactionContext.finish();
-
- // verify writes are visible after commit
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointRollback() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- transactionContext.abort();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, null);
- verifyRow(transactionAwareHTable, TestBytes.row2, null);
- verifyRow(transactionAwareHTable, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointInvalidate() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
- txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
-
- // check that writes are not visible
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
- txContext2.start();
- Transaction newTx = txContext2.getCurrentTransaction();
-
- // all 3 writes pointers from the previous transaction should now be excluded
- assertTrue(newTx.isExcluded(origTx.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
-
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- verifyRow(txTable2, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = txTable2.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- txContext2.finish();
- }
-
- @Test
- public void testExistingData() throws Exception {
- byte[] val11 = Bytes.toBytes("val11");
- byte[] val12 = Bytes.toBytes("val12");
- byte[] val21 = Bytes.toBytes("val21");
- byte[] val22 = Bytes.toBytes("val22");
- byte[] val31 = Bytes.toBytes("val31");
- byte[] val111 = Bytes.toBytes("val111");
-
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
- Collections.<String>emptyList()));
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Add some pre-existing, non-transactional data
- HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
- HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.flushCommits();
-
- // Add transactional data
- txContext.start();
- txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
- txContext.finish();
-
- txContext.start();
- // test get
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
-
- // test scan
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
-
- // test update and delete
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
- txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
-
- txContext.start();
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
- txContext.finish();
-
- // test scan
- txContext.start();
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
- }
-
- private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
- verifyRow(table, new Get(rowkey), expectedValue);
- }
-
- private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
- verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
- }
-
- private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
- Result result = table.get(get);
- if (expectedValues == null) {
- assertTrue(result.isEmpty());
- } else {
- assertFalse(result.isEmpty());
- byte[] family = TestBytes.family;
- byte[] col = TestBytes.qualifier;
- if (get.hasFamilies()) {
- family = get.getFamilyMap().keySet().iterator().next();
- col = get.getFamilyMap().get(family).first();
- }
- Iterator<Cell> it = result.getColumnCells(family, col).iterator();
- for (byte[] expectedValue : expectedValues) {
- Assert.assertTrue(it.hasNext());
- assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
- }
- }
- }
-
- private Cell[] getRow(HTableInterface table, Get get) throws Exception {
- Result result = table.get(get);
- return result.rawCells();
- }
-
- private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
- List<Cell> actualCells = new ArrayList<>();
- try (ResultScanner scanner = table.getScanner(scan)) {
- Result[] results = scanner.next(expectedCells.size() + 1);
- for (Result result : results) {
- actualCells.addAll(Lists.newArrayList(result.rawCells()));
- }
- Assert.assertEquals(expectedCells, actualCells);
- }
- }
-
- @Test
- public void testVisibilityAll() throws Exception {
- HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
- new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(nonTxTable,
- TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // start a transaction and create a delete marker
- txContext.start();
- //noinspection ConstantConditions
- long txWp0 = txContext.getCurrentTransaction().getWritePointer();
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // start a new transaction and write some values
- txContext.start();
- @SuppressWarnings("ConstantConditions")
- long txWp1 = txContext.getCurrentTransaction().getWritePointer();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
-
- // verify written data
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- TestBytes.value2);
-
- // checkpoint and make changes to written data now
- txContext.checkpoint();
- long txWp2 = txContext.getCurrentTransaction().getWritePointer();
- // delete a column
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
- // no change to a column
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- // update a column
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
- // delete column family
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
-
- // verify changed values
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- null);
-
- // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
- // and the raw change by prior transaction
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- List<KeyValue> expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // verify a Get is also able to return all snapshot versions
- Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
- Cell[] cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(3, cells.length);
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
- Assert.assertTrue(CellUtil.isDeleteColumns(cells[2]));
-
- verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- ImmutableList.of(TestBytes.value3, TestBytes.value));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
-
- // Verify VisibilityLevel.SNAPSHOT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
- txContext.finish();
-
- // finally verify values once more after commit, this time we should get only committed raw values for
- // all visibility levels
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- txContext.finish();
-
- // Test with regular HBase deletes in pre-existing data
- long now = System.currentTimeMillis();
- Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
- // to prevent Tephra from replacing delete with delete marker
- deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteColumn);
- Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
- // to prevent Tephra from replacing delete with delete marker
- deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteFamily);
- nonTxTable.flushCommits();
-
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
- new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
- );
- // test scan
- Scan scan = new Scan();
- scan.setRaw(true);
- verifyScan(txTable, scan, expected);
- txContext.finish();
- }
-
- @Test
- public void testFilters() throws Exception {
- // Add some values to table
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] val1 = Bytes.toBytes(1L);
- put.add(TestBytes.family, TestBytes.qualifier, val1);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row2);
- byte[] val2 = Bytes.toBytes(2L);
- put.add(TestBytes.family, TestBytes.qualifier, val2);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row3);
- byte[] val3 = Bytes.toBytes(3L);
- put.add(TestBytes.family, TestBytes.qualifier, val3);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row4);
- byte[] val4 = Bytes.toBytes(4L);
- put.add(TestBytes.family, TestBytes.qualifier, val4);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Delete cell with value 2
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row2);
- delete.deleteColumn(TestBytes.family, TestBytes.qualifier);
- transactionAwareHTable.delete(delete);
- transactionContext.finish();
-
- // Scan for values less than 4, should get only values 1 and 3
- transactionContext.start();
- Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(4)));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run a Get with a filter for less than 10 on row4, should get value 4
- transactionContext.start();
- Get get = new Get(TestBytes.row4);
- get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
- Result result = transactionAwareHTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
- transactionContext.finish();
-
- // Change value of row4 to 40
- transactionContext.start();
- put = new Put(TestBytes.row4);
- byte[] val40 = Bytes.toBytes(40L);
- put.add(TestBytes.family, TestBytes.qualifier, val40);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Scan for values less than 10, should get only values 1 and 3
- transactionContext.start();
- scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run the Get again with a filter for less than 10 on row4, this time should not get any results
- transactionContext.start();
- result = transactionAwareHTable.get(get);
- assertTrue(result.isEmpty());
- transactionContext.finish();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilterTest.java
deleted file mode 100644
index c605f66..0000000
--- a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/coprocessor/CellSkipFilterTest.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * HBase 0.98 specific test for testing {@link CellSkipFilter}.
- */
-public class CellSkipFilterTest {
-
- private static final String ROW1KEY = "row1";
- private static final String ROW2KEY = "row2";
- private static final String FAM1KEY = "fam1";
- private static final String COL1KEY = "col1";
- private static final String FAM2KEY = "fam2";
- private static final String COL2KEY = "col2";
- private static final String VALUE = "value";
-
- @Test
- public void testSkipFiltering() throws Exception {
- long timestamp = System.currentTimeMillis();
- // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
- Filter filter = new CellSkipFilter(new MyFilter(0));
- assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 1)));
-
- // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give SKIP from MyFilter
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
- timestamp - 4)));
-
- // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
- filter = new CellSkipFilter(new MyFilter(2));
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 1)));
-
- // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give NEXT_ROW from MyFilter
- assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
- timestamp - 4)));
-
- // Next call with the new key should returned the SEEK_NEXT_USING_HINT
- assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 5)));
- }
-
- private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
- timestamp, Bytes.toBytes(value));
- }
-
- /**
- * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
- * start index in the list and start serving the return codes corresponding that that index. Every time the
- * return code is served, index is incremented.
- */
- class MyFilter extends FilterBase {
-
- private final List<ReturnCode> returnCodes;
- private int counter;
-
- public MyFilter(int startIndex) {
- returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
- ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
- counter = startIndex;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- ReturnCode code = returnCodes.get(counter % returnCodes.size());
- counter++;
- return code;
- }
- }
-}
[02/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java
new file mode 100644
index 0000000..6ff4bb1
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java
@@ -0,0 +1,1552 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase10;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.LongComparator;
+import org.apache.hadoop.hbase.filter.ValueFilter;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionConflictException;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.hbase10.coprocessor.TransactionProcessor;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests for TransactionAwareHTables.
+ */
+public class TransactionAwareHTableTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
+
+ private static HBaseTestingUtility testUtil;
+ private static HBaseAdmin hBaseAdmin;
+ private static TransactionStateStorage txStateStorage;
+ private static TransactionManager txManager;
+ private static Configuration conf;
+ private TransactionContext transactionContext;
+ private TransactionAwareHTable transactionAwareHTable;
+ private HTable hTable;
+
+ private static final class TestBytes {
+ private static final byte[] table = Bytes.toBytes("testtable");
+ private static final byte[] family = Bytes.toBytes("f1");
+ private static final byte[] family2 = Bytes.toBytes("f2");
+ private static final byte[] qualifier = Bytes.toBytes("col1");
+ private static final byte[] qualifier2 = Bytes.toBytes("col2");
+ private static final byte[] row = Bytes.toBytes("row");
+ private static final byte[] row2 = Bytes.toBytes("row2");
+ private static final byte[] row3 = Bytes.toBytes("row3");
+ private static final byte[] row4 = Bytes.toBytes("row4");
+ private static final byte[] value = Bytes.toBytes("value");
+ private static final byte[] value2 = Bytes.toBytes("value2");
+ private static final byte[] value3 = Bytes.toBytes("value3");
+ }
+
+ private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
+
+ public static class TestRegionObserver extends BaseRegionObserver {
+ @Override
+ public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Put put, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (put.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Put should preserve attributes");
+ }
+ if (put.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+
+ @Override
+ public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Delete delete, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Delete should preserve attributes");
+ }
+ if (delete.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ conf = testUtil.getConfiguration();
+
+ // Tune down the connection thread pool size
+ conf.setInt("hbase.hconnection.threads.core", 5);
+ conf.setInt("hbase.hconnection.threads.max", 10);
+ // Tunn down handler threads in regionserver
+ conf.setInt("hbase.regionserver.handler.count", 10);
+
+ // Set to random port
+ conf.setInt("hbase.master.port", 0);
+ conf.setInt("hbase.master.info.port", 0);
+ conf.setInt("hbase.regionserver.port", 0);
+ conf.setInt("hbase.regionserver.info.port", 0);
+
+ testUtil.startMiniCluster();
+ hBaseAdmin = testUtil.getHBaseAdmin();
+ txStateStorage = new InMemoryTransactionStateStorage();
+ txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
+ txManager.startAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ testUtil.shutdownMiniCluster();
+ hBaseAdmin.close();
+ }
+
+ @Before
+ public void setupBeforeTest() throws Exception {
+ hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
+ transactionAwareHTable = new TransactionAwareHTable(hTable);
+ transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
+ }
+
+ @After
+ public void shutdownAfterTest() throws IOException {
+ hBaseAdmin.disableTable(TestBytes.table);
+ hBaseAdmin.deleteTable(TestBytes.table);
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
+ return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
+ List<String> coprocessors) throws Exception {
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+ for (byte[] family : columnFamilies) {
+ HColumnDescriptor columnDesc = new HColumnDescriptor(family);
+ columnDesc.setMaxVersions(Integer.MAX_VALUE);
+ columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
+ desc.addFamily(columnDesc);
+ }
+ if (existingData) {
+ desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
+ }
+ // Divide individually to prevent any overflow
+ int priority = Coprocessor.PRIORITY_USER;
+ desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
+ // order in list is the same order that coprocessors will be invoked
+ for (String coprocessor : coprocessors) {
+ desc.addCoprocessor(coprocessor, null, ++priority, null);
+ }
+ hBaseAdmin.createTable(desc);
+ testUtil.waitTableAvailable(tableName, 5000);
+ return new HTable(testUtil.getConfiguration(), tableName);
+ }
+
+ /**
+ * Test transactional put and get requests.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Test aborted put requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(value, null);
+ }
+
+ /**
+ * Test transactional delete operations.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
+ new byte[][]{TestBytes.family, TestBytes.family2});
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+
+ // test column delete
+ // load 10 rows
+ txContext.start();
+ int rowCount = 10;
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
+ }
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertFalse(r.isEmpty());
+ for (int j = 0; j < 10; j++) {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
+ }
+ }
+ txContext.finish();
+
+ // delete odds columns from odd rows and even columns from even rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ if (i % 2 == j % 2) {
+ LOG.info("Deleting row={}, column={}", i, j);
+ d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
+ }
+ }
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted columns
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertEquals(5, r.size());
+ for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
+ int col = Bytes.toInt(entry.getKey());
+ LOG.info("Got row={}, col={}", i, col);
+ // each row should only have the opposite mod (odd=even, even=odd)
+ assertNotEquals(i % 2, col % 2);
+ assertArrayEquals(TestBytes.value, entry.getValue());
+ }
+ }
+ txContext.finish();
+
+ // test family delete
+ // load 10 rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("famrow" + i));
+ p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify all loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(2, r.size());
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ txContext.finish();
+
+ // delete family1 for even rows, family2 for odd rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("famrow" + i));
+ d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted families
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(1, r.size());
+ if (i % 2 == 0) {
+ assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ } else {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ }
+ txContext.finish();
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test that put and delete attributes are preserved
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAttributesPreserved() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, false,
+ Lists.newArrayList(TestRegionObserver.class.getName()));
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ // set an attribute on the put, TestRegionObserver will verify it still exists
+ put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete, TestRegionObserver will verify it still exists
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test aborted transactional delete requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionalDelete() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ transactionAwareHTable.delete(delete);
+ transactionContext.abort();
+
+ transactionContext.start();
+ result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
+ String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
+ HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ // Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ // Start a tx, delete a column family and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
+ txContext.abort();
+
+ // Above operations should have no effect on the row, since they were aborted
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testDeleteRollback() throws Exception {
+ testDeleteRollback(TxConstants.ConflictDetection.ROW);
+ testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
+ testDeleteRollback(TxConstants.ConflictDetection.NONE);
+ }
+
+ @Test
+ public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ txContext.start();
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ Result result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+
+ //Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ //Start a tx and scan all the col families to make sure none of them have delete markers
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testRowDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][]{TestBytes.family, TestBytes.family2});
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
+ try {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Test 1: full row delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // delete entire row
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.finish();
+
+ // verify row is now empty
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for explicit column retrieval
+ result = txTable.get(new Get(TestBytes.row)
+ .addColumn(TestBytes.family, TestBytes.qualifier)
+ .addFamily(TestBytes.family2));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for scan
+ ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
+ assertNull(scanner.next());
+ scanner.close();
+
+ // verify row is empty for scan with explicit column
+ scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
+ assertNull(scanner.next());
+ scanner.close();
+ txContext.finish();
+
+ // write swapped values to one column per family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
+ txContext.finish();
+
+ // verify new values appear
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertFalse(result.isEmpty());
+ assertEquals(2, result.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+
+ scanner = txTable.getScanner(new Scan(TestBytes.row));
+ Result result1 = scanner.next();
+ assertNotNull(result1);
+ assertFalse(result1.isEmpty());
+ assertEquals(2, result1.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ scanner.close();
+ txContext.finish();
+
+ // Test 2: delete of first column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row2)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
+ txContext.finish();
+
+ txContext.start();
+ Result fam1Result = txTable.get(new Get(TestBytes.row2));
+ assertFalse(fam1Result.isEmpty());
+ assertEquals(2, fam1Result.size());
+ assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 3: delete of second column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
+ txContext.finish();
+
+ txContext.start();
+ Result fam2Result = txTable.get(new Get(TestBytes.row3));
+ assertFalse(fam2Result.isEmpty());
+ assertEquals(2, fam2Result.size());
+ assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 4: delete specific rows in a range
+ txContext.start();
+ for (int i = 0; i < 10; i++) {
+ txTable.put(new Put(Bytes.toBytes("z" + i))
+ .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
+ .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ // delete odd rows
+ for (int i = 1; i < 10; i += 2) {
+ txTable.delete(new Delete(Bytes.toBytes("z" + i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ int cnt = 0;
+ ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
+ Result res;
+ while ((res = zScanner.next()) != null) {
+ assertFalse(res.isEmpty());
+ assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
+ cnt += 2;
+ }
+
+ // Test 5: delete prior writes in the same transaction
+ txContext.start();
+ txTable.put(new Put(TestBytes.row4)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txTable.delete(new Delete(TestBytes.row4));
+ txContext.finish();
+
+ txContext.start();
+ Result row4Result = txTable.get(new Get(TestBytes.row4));
+ assertTrue(row4Result.isEmpty());
+ txContext.finish();
+ } finally {
+ txTable.close();
+ }
+ }
+
+ /**
+ * Expect an exception since a transaction hasn't been started.
+ *
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testTransactionlessFailure() throws Exception {
+ transactionAwareHTable.get(new Get(TestBytes.row));
+ }
+
+ /**
+ * Tests that each transaction can see its own persisted writes, while not seeing writes from other
+ * in-progress transactions.
+ */
+ @Test
+ public void testReadYourWrites() throws Exception {
+ // In-progress tx1: started before our main transaction
+ HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
+ TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
+
+ // In-progress tx2: started while our main transaction is running
+ HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
+ TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
+
+ // create an in-progress write that should be ignored
+ byte[] col2 = Bytes.toBytes("col2");
+ inprogressTxContext1.start();
+ Put putCol2 = new Put(TestBytes.row);
+ byte[] valueCol2 = Bytes.toBytes("writing in progress");
+ putCol2.add(TestBytes.family, col2, valueCol2);
+ txHTable1.put(putCol2);
+
+ // start a tx and write a value to test reading in same tx
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] value = Bytes.toBytes("writing");
+ put.add(TestBytes.family, TestBytes.qualifier, value);
+ transactionAwareHTable.put(put);
+
+ // test that a write from a tx started after the first is not visible
+ inprogressTxContext2.start();
+ Put put2 = new Put(TestBytes.row);
+ byte[] value2 = Bytes.toBytes("writing2");
+ put2.add(TestBytes.family, TestBytes.qualifier, value2);
+ txHTable2.put(put2);
+
+ Get get = new Get(TestBytes.row);
+ Result row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
+ Assert.assertNotNull(col1Value);
+ Assert.assertArrayEquals(value, col1Value);
+ // write from in-progress transaction should not be visible
+ byte[] col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ // commit in-progress transaction, should still not be visible
+ inprogressTxContext1.finish();
+
+ get = new Get(TestBytes.row);
+ row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ transactionContext.finish();
+
+ inprogressTxContext2.abort();
+ }
+
+ @Test
+ public void testRowLevelConflictDetection() throws Exception {
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ byte[] row1 = Bytes.toBytes("row1");
+ byte[] row2 = Bytes.toBytes("row2");
+ byte[] col1 = Bytes.toBytes("c1");
+ byte[] col2 = Bytes.toBytes("c2");
+ byte[] val1 = Bytes.toBytes("val1");
+ byte[] val2 = Bytes.toBytes("val2");
+
+ // test that concurrent writing to different rows succeeds
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
+
+ // should be no conflicts
+ txContext1.finish();
+ txContext2.finish();
+
+ transactionContext.start();
+ Result res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // test that writing to different columns in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
+
+ txContext2.start();
+ txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row-level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ // should now be val2
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ // col2 should not be visible due to conflict
+ assertNull(cell);
+ transactionContext.finish();
+
+ // test that writing to the same column in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row and column level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ assertNotNull(cell);
+ // should now be val1
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // verify change set that is being reported only on rows
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ Collection<byte[]> changeSet = txTable1.getTxChanges();
+ assertNotNull(changeSet);
+ assertEquals(2, changeSet.size());
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
+ txContext1.finish();
+ }
+
+ @Test
+ public void testNoneLevelConflictDetection() throws Exception {
+ InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+
+ // overlapping writes to the same row + column should not conflict
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // changes should not be visible yet
+ txContext2.start();
+ Result row = txTable2.get(new Get(TestBytes.row));
+ assertTrue(row.isEmpty());
+
+ txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // both commits should succeed
+ txContext1.finish();
+ txContext2.finish();
+
+ txContext1.start();
+ row = txTable1.get(new Get(TestBytes.row));
+ assertFalse(row.isEmpty());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext1.finish();
+
+ // transaction abort should still rollback changes
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext1.abort();
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row2 = txTable2.get(new Get(TestBytes.row2));
+ assertTrue(row2.isEmpty());
+ txContext2.finish();
+
+ // transaction invalidate should still make changes invisible
+
+ txContext1.start();
+ Transaction tx1 = txContext1.getCurrentTransaction();
+ txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ assertNotNull(tx1);
+ txClient.invalidate(tx1.getWritePointer());
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row3 = txTable2.get(new Get(TestBytes.row3));
+ assertTrue(row3.isEmpty());
+ txContext2.finish();
+ }
+
+ @Test
+ public void testCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
+ assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
+ long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
+ assertEquals(1, checkpointPtrs.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
+ assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
+ long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
+ assertEquals(2, checkpointPtrs2.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
+ assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // by default, all rows should be visible with Read-Your-Writes
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+
+ // when disabling current write pointer, only the previous checkpoints should be visible
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ Get get = new Get(TestBytes.row);
+ verifyRow(transactionAwareHTable, get, TestBytes.value);
+ get = new Get(TestBytes.row2);
+ verifyRow(transactionAwareHTable, get, TestBytes.value2);
+ get = new Get(TestBytes.row3);
+ verifyRow(transactionAwareHTable, get, null);
+
+ // test scan results excluding current write pointer
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+
+ Result row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row2, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNull(row);
+ scanner.close();
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+
+ // commit transaction, verify writes are visible
+ transactionContext.finish();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testInProgressCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // check that writes are still not visible to other clients
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ txContext2.start();
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ txContext2.finish();
+ txTable2.close();
+
+ transactionContext.finish();
+
+ // verify writes are visible after commit
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointRollback() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, null);
+ verifyRow(transactionAwareHTable, TestBytes.row2, null);
+ verifyRow(transactionAwareHTable, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointInvalidate() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
+
+ // check that writes are not visible
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+ txContext2.start();
+ Transaction newTx = txContext2.getCurrentTransaction();
+
+ // all 3 writes pointers from the previous transaction should now be excluded
+ assertTrue(newTx.isExcluded(origTx.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
+
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ verifyRow(txTable2, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = txTable2.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ txContext2.finish();
+ }
+
+ @Test
+ public void testExistingData() throws Exception {
+ byte[] val11 = Bytes.toBytes("val11");
+ byte[] val12 = Bytes.toBytes("val12");
+ byte[] val21 = Bytes.toBytes("val21");
+ byte[] val22 = Bytes.toBytes("val22");
+ byte[] val31 = Bytes.toBytes("val31");
+ byte[] val111 = Bytes.toBytes("val111");
+
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
+ Collections.<String>emptyList()));
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Add some pre-existing, non-transactional data
+ HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
+ HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.flushCommits();
+
+ // Add transactional data
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
+ txContext.finish();
+
+ txContext.start();
+ // test get
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+
+ // test scan
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+
+ // test update and delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
+ txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+
+ txContext.start();
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+ txContext.finish();
+
+ // test scan
+ txContext.start();
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+ }
+
+ private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
+ verifyRow(table, new Get(rowkey), expectedValue);
+ }
+
+ private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
+ verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
+ }
+
+ private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
+ Result result = table.get(get);
+ if (expectedValues == null) {
+ assertTrue(result.isEmpty());
+ } else {
+ assertFalse(result.isEmpty());
+ byte[] family = TestBytes.family;
+ byte[] col = TestBytes.qualifier;
+ if (get.hasFamilies()) {
+ family = get.getFamilyMap().keySet().iterator().next();
+ col = get.getFamilyMap().get(family).first();
+ }
+ Iterator<Cell> it = result.getColumnCells(family, col).iterator();
+ for (byte[] expectedValue : expectedValues) {
+ Assert.assertTrue(it.hasNext());
+ assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
+ }
+ }
+ }
+
+ private Cell[] getRow(HTableInterface table, Get get) throws Exception {
+ Result result = table.get(get);
+ return result.rawCells();
+ }
+
+ private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
+ List<Cell> actualCells = new ArrayList<>();
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ Result[] results = scanner.next(expectedCells.size() + 1);
+ for (Result result : results) {
+ actualCells.addAll(Lists.newArrayList(result.rawCells()));
+ }
+ Assert.assertEquals(expectedCells, actualCells);
+ }
+ }
+
+ @Test
+ public void testVisibilityAll() throws Exception {
+ HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(nonTxTable,
+ TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // start a transaction and create a delete marker
+ txContext.start();
+ //noinspection ConstantConditions
+ long txWp0 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // start a new transaction and write some values
+ txContext.start();
+ @SuppressWarnings("ConstantConditions")
+ long txWp1 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+
+ // verify written data
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ TestBytes.value2);
+
+ // checkpoint and make changes to written data now
+ txContext.checkpoint();
+ long txWp2 = txContext.getCurrentTransaction().getWritePointer();
+ // delete a column
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
+ // no change to a column
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ // update a column
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
+ // delete column family
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
+
+ // verify changed values
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ null);
+
+ // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
+ // and the raw change by prior transaction
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ List<KeyValue> expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // verify a Get is also able to return all snapshot versions
+ Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
+ Cell[] cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(3, cells.length);
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+ Assert.assertTrue(CellUtil.isDeleteColumns(cells[2]));
+
+ verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ ImmutableList.of(TestBytes.value3, TestBytes.value));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+
+ // Verify VisibilityLevel.SNAPSHOT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+ txContext.finish();
+
+ // finally verify values once more after commit, this time we should get only committed raw values for
+ // all visibility levels
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ txContext.finish();
+
+ // Test with regular HBase deletes in pre-existing data
+ long now = System.currentTimeMillis();
+ Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteColumn);
+ Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteFamily);
+ nonTxTable.flushCommits();
+
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
+ new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
+ );
+ // test scan
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ verifyScan(txTable, scan, expected);
+ txContext.finish();
+ }
+
+ @Test
+ public void testFilters() throws Exception {
+ // Add some values to table
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] val1 = Bytes.toBytes(1L);
+ put.add(TestBytes.family, TestBytes.qualifier, val1);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row2);
+ byte[] val2 = Bytes.toBytes(2L);
+ put.add(TestBytes.family, TestBytes.qualifier, val2);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row3);
+ byte[] val3 = Bytes.toBytes(3L);
+ put.add(TestBytes.family, TestBytes.qualifier, val3);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row4);
+ byte[] val4 = Bytes.toBytes(4L);
+ put.add(TestBytes.family, TestBytes.qualifier, val4);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Delete cell with value 2
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row2);
+ delete.addColumn(TestBytes.family, TestBytes.qualifier);
+ transactionAwareHTable.delete(delete);
+ transactionContext.finish();
+
+ // Scan for values less than 4, should get only values 1 and 3
+ transactionContext.start();
+ Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(4)));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run a Get with a filter for less than 10 on row4, should get value 4
+ transactionContext.start();
+ Get get = new Get(TestBytes.row4);
+ get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
+ Result result = transactionAwareHTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
+ transactionContext.finish();
+
+ // Change value of row4 to 40
+ transactionContext.start();
+ put = new Put(TestBytes.row4);
+ byte[] val40 = Bytes.toBytes(40L);
+ put.add(TestBytes.family, TestBytes.qualifier, val40);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Scan for values less than 10, should get only values 1 and 3
+ transactionContext.start();
+ scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run the Get again with a filter for less than 10 on row4, this time should not get any results
+ transactionContext.start();
+ result = transactionAwareHTable.get(get);
+ assertTrue(result.isEmpty());
+ transactionContext.finish();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilterTest.java
new file mode 100644
index 0000000..45a05fe
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilterTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * HBase 1.0 specific test for testing {@link CellSkipFilter}.
+ */
+public class CellSkipFilterTest {
+
+ private static final String ROW1KEY = "row1";
+ private static final String ROW2KEY = "row2";
+ private static final String FAM1KEY = "fam1";
+ private static final String COL1KEY = "col1";
+ private static final String FAM2KEY = "fam2";
+ private static final String COL2KEY = "col2";
+ private static final String VALUE = "value";
+
+ @Test
+ public void testSkipFiltering() throws Exception {
+ long timestamp = System.currentTimeMillis();
+ // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
+ Filter filter = new CellSkipFilter(new MyFilter(0));
+ assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give SKIP from MyFilter
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
+ timestamp - 4)));
+
+ // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
+ filter = new CellSkipFilter(new MyFilter(2));
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give NEXT_ROW from MyFilter
+ assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
+ timestamp - 4)));
+
+ // Next call with the new key should returned the SEEK_NEXT_USING_HINT
+ assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 5)));
+ }
+
+ private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
+ timestamp, Bytes.toBytes(value));
+ }
+
+ /**
+ * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
+ * start index in the list and start serving the return codes corresponding that that index. Every time the
+ * return code is served, index is incremented.
+ */
+ class MyFilter extends FilterBase {
+
+ private final List<ReturnCode> returnCodes;
+ private int counter;
+
+ public MyFilter(int startIndex) {
+ returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
+ ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
+ counter = startIndex;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ ReturnCode code = returnCodes.get(counter % returnCodes.size());
+ counter++;
+ return code;
+ }
+ }
+}
[22/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/TransactionAwareHTableTest.java b/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/TransactionAwareHTableTest.java
deleted file mode 100644
index e973ba4..0000000
--- a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/TransactionAwareHTableTest.java
+++ /dev/null
@@ -1,1555 +0,0 @@
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase96;
-
- import co.cask.tephra.Transaction;
- import co.cask.tephra.TransactionConflictException;
- import co.cask.tephra.TransactionContext;
- import co.cask.tephra.TransactionManager;
- import co.cask.tephra.TransactionSystemClient;
- import co.cask.tephra.TxConstants;
- import co.cask.tephra.hbase96.coprocessor.TransactionProcessor;
- import co.cask.tephra.inmemory.InMemoryTxSystemClient;
- import co.cask.tephra.metrics.TxMetricsCollector;
- import co.cask.tephra.persist.InMemoryTransactionStateStorage;
- import co.cask.tephra.persist.TransactionStateStorage;
- import com.google.common.collect.ImmutableList;
- import com.google.common.collect.Lists;
- import com.google.common.primitives.Longs;
- import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.hbase.Cell;
- import org.apache.hadoop.hbase.CellUtil;
- import org.apache.hadoop.hbase.Coprocessor;
- import org.apache.hadoop.hbase.DoNotRetryIOException;
- import org.apache.hadoop.hbase.HBaseTestingUtility;
- import org.apache.hadoop.hbase.HColumnDescriptor;
- import org.apache.hadoop.hbase.HConstants;
- import org.apache.hadoop.hbase.HTableDescriptor;
- import org.apache.hadoop.hbase.KeyValue;
- import org.apache.hadoop.hbase.TableName;
- import org.apache.hadoop.hbase.client.Delete;
- import org.apache.hadoop.hbase.client.Durability;
- import org.apache.hadoop.hbase.client.Get;
- import org.apache.hadoop.hbase.client.HBaseAdmin;
- import org.apache.hadoop.hbase.client.HTable;
- import org.apache.hadoop.hbase.client.HTableInterface;
- import org.apache.hadoop.hbase.client.Put;
- import org.apache.hadoop.hbase.client.Result;
- import org.apache.hadoop.hbase.client.ResultScanner;
- import org.apache.hadoop.hbase.client.Scan;
- import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
- import org.apache.hadoop.hbase.coprocessor.ObserverContext;
- import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
- import org.apache.hadoop.hbase.filter.BinaryComparator;
- import org.apache.hadoop.hbase.filter.CompareFilter;
- import org.apache.hadoop.hbase.filter.ValueFilter;
- import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
- import org.apache.hadoop.hbase.util.Bytes;
- import org.junit.After;
- import org.junit.AfterClass;
- import org.junit.Assert;
- import org.junit.Before;
- import org.junit.BeforeClass;
- import org.junit.Test;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
-
- import java.io.IOException;
- import java.util.ArrayList;
- import java.util.Collection;
- import java.util.Collections;
- import java.util.Iterator;
- import java.util.List;
- import java.util.Map;
-
- import static org.junit.Assert.assertArrayEquals;
- import static org.junit.Assert.assertEquals;
- import static org.junit.Assert.assertFalse;
- import static org.junit.Assert.assertNotEquals;
- import static org.junit.Assert.assertNotNull;
- import static org.junit.Assert.assertNull;
- import static org.junit.Assert.assertTrue;
- import static org.junit.Assert.fail;
-
-/**
- * Tests for TransactionAwareHTables.
- */
-public class TransactionAwareHTableTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
-
- private static HBaseTestingUtility testUtil;
- private static HBaseAdmin hBaseAdmin;
- private static TransactionStateStorage txStateStorage;
- private static TransactionManager txManager;
- private static Configuration conf;
- private TransactionContext transactionContext;
- private TransactionAwareHTable transactionAwareHTable;
- private HTable hTable;
-
- private static final class TestBytes {
- private static final byte[] table = Bytes.toBytes("testtable");
- private static final byte[] family = Bytes.toBytes("f1");
- private static final byte[] family2 = Bytes.toBytes("f2");
- private static final byte[] qualifier = Bytes.toBytes("col1");
- private static final byte[] qualifier2 = Bytes.toBytes("col2");
- private static final byte[] row = Bytes.toBytes("row");
- private static final byte[] row2 = Bytes.toBytes("row2");
- private static final byte[] row3 = Bytes.toBytes("row3");
- private static final byte[] row4 = Bytes.toBytes("row4");
- private static final byte[] value = Bytes.toBytes("value");
- private static final byte[] value2 = Bytes.toBytes("value2");
- private static final byte[] value3 = Bytes.toBytes("value3");
- }
-
- private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
-
- public static class TestRegionObserver extends BaseRegionObserver {
- @Override
- public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Put put, final WALEdit edit,
- final Durability durability) throws IOException {
- if (put.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Put should preserve attributes");
- }
- if (put.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
-
- @Override
- public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Delete delete, final WALEdit edit,
- final Durability durability) throws IOException {
- if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Delete should preserve attributes");
- }
- if (delete.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
- }
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- testUtil = new HBaseTestingUtility();
- conf = testUtil.getConfiguration();
-
- // Tune down the connection thread pool size
- conf.setInt("hbase.hconnection.threads.core", 5);
- conf.setInt("hbase.hconnection.threads.max", 10);
- // Tunn down handler threads in regionserver
- conf.setInt("hbase.regionserver.handler.count", 10);
-
- // Set to random port
- conf.setInt("hbase.master.port", 0);
- conf.setInt("hbase.master.info.port", 0);
- conf.setInt("hbase.regionserver.port", 0);
- conf.setInt("hbase.regionserver.info.port", 0);
-
- testUtil.startMiniCluster();
- hBaseAdmin = testUtil.getHBaseAdmin();
- txStateStorage = new InMemoryTransactionStateStorage();
- txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
- txManager.startAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- testUtil.shutdownMiniCluster();
- hBaseAdmin.close();
- }
-
- @Before
- public void setupBeforeTest() throws Exception {
- hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
- transactionAwareHTable = new TransactionAwareHTable(hTable);
- transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
- }
-
- @After
- public void shutdownAfterTest() throws IOException {
- hBaseAdmin.disableTable(TestBytes.table);
- hBaseAdmin.deleteTable(TestBytes.table);
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
- return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
- List<String> coprocessors) throws Exception {
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
- for (byte[] family : columnFamilies) {
- HColumnDescriptor columnDesc = new HColumnDescriptor(family);
- columnDesc.setMaxVersions(Integer.MAX_VALUE);
- columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
- desc.addFamily(columnDesc);
- }
- if (existingData) {
- desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
- }
- // Divide individually to prevent any overflow
- int priority = Coprocessor.PRIORITY_USER;
- desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
- // order in list is the same order that coprocessors will be invoked
- for (String coprocessor : coprocessors) {
- desc.addCoprocessor(coprocessor, null, ++priority, null);
- }
- hBaseAdmin.createTable(desc);
- testUtil.waitTableAvailable(tableName, 5000);
- return new HTable(testUtil.getConfiguration(), tableName);
- }
-
- /**
- * Test transactional put and get requests.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
-
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- /**
- * Test aborted put requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
-
- transactionContext.abort();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(value, null);
- }
-
- /**
- * Test transactional delete operations.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
- new byte[][]{TestBytes.family, TestBytes.family2});
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
-
- // test column delete
- // load 10 rows
- txContext.start();
- int rowCount = 10;
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
- }
- txTable.put(p);
- }
- txContext.finish();
-
- // verify loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertFalse(r.isEmpty());
- for (int j = 0; j < 10; j++) {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
- }
- }
- txContext.finish();
-
- // delete odds columns from odd rows and even columns from even rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- if (i % 2 == j % 2) {
- LOG.info("Deleting row={}, column={}", i, j);
- d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
- }
- }
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted columns
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertEquals(5, r.size());
- for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
- int col = Bytes.toInt(entry.getKey());
- LOG.info("Got row={}, col={}", i, col);
- // each row should only have the opposite mod (odd=even, even=odd)
- assertNotEquals(i % 2, col % 2);
- assertArrayEquals(TestBytes.value, entry.getValue());
- }
- }
- txContext.finish();
-
- // test family delete
- // load 10 rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("famrow" + i));
- p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
- txTable.put(p);
- }
- txContext.finish();
-
- // verify all loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(2, r.size());
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- txContext.finish();
-
- // delete family1 for even rows, family2 for odd rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("famrow" + i));
- d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted families
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(1, r.size());
- if (i % 2 == 0) {
- assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- } else {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- }
- txContext.finish();
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test that put and delete attributes are preserved
- *
- * @throws Exception
- */
- @Test
- public void testAttributesPreserved() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
- new byte[][]{TestBytes.family, TestBytes.family2}, false,
- Lists.newArrayList(TestRegionObserver.class.getName()));
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- // set an attribute on the put, TestRegionObserver will verify it still exists
- put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete, TestRegionObserver will verify it still exists
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test aborted transactional delete requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionalDelete() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
-
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row);
- transactionAwareHTable.delete(delete);
- transactionContext.abort();
-
- transactionContext.start();
- result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
- String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
- HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- // Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- // Start a tx, delete a column family and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
- txContext.abort();
-
- // Above operations should have no effect on the row, since they were aborted
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
- }
- }
-
- @Test
- public void testDeleteRollback() throws Exception {
- testDeleteRollback(TxConstants.ConflictDetection.ROW);
- testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
- testDeleteRollback(TxConstants.ConflictDetection.NONE);
- }
-
- @Test
- public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- txContext.start();
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- Result result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
-
- //Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- //Start a tx and scan all the col families to make sure none of them have delete markers
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
- }
- }
-
- @Test
- public void testRowDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][] {TestBytes.family, TestBytes.family2});
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
- try {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Test 1: full row delete
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // delete entire row
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.finish();
-
- // verify row is now empty
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertTrue(result.isEmpty());
-
- // verify row is empty for explicit column retrieval
- result = txTable.get(new Get(TestBytes.row)
- .addColumn(TestBytes.family, TestBytes.qualifier)
- .addFamily(TestBytes.family2));
- assertTrue(result.isEmpty());
-
- // verify row is empty for scan
- ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
- assertNull(scanner.next());
- scanner.close();
-
- // verify row is empty for scan with explicit column
- scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
- assertNull(scanner.next());
- scanner.close();
- txContext.finish();
-
- // write swapped values to one column per family
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
- txContext.finish();
-
- // verify new values appear
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertFalse(result.isEmpty());
- assertEquals(2, result.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
-
- scanner = txTable.getScanner(new Scan(TestBytes.row));
- Result result1 = scanner.next();
- assertNotNull(result1);
- assertFalse(result1.isEmpty());
- assertEquals(2, result1.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- scanner.close();
- txContext.finish();
-
- // Test 2: delete of first column family
- txContext.start();
- txTable.put(new Put(TestBytes.row2)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
- txContext.finish();
-
- txContext.start();
- Result fam1Result = txTable.get(new Get(TestBytes.row2));
- assertFalse(fam1Result.isEmpty());
- assertEquals(2, fam1Result.size());
- assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 3: delete of second column family
- txContext.start();
- txTable.put(new Put(TestBytes.row3)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
- txContext.finish();
-
- txContext.start();
- Result fam2Result = txTable.get(new Get(TestBytes.row3));
- assertFalse(fam2Result.isEmpty());
- assertEquals(2, fam2Result.size());
- assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 4: delete specific rows in a range
- txContext.start();
- for (int i = 0; i < 10; i++) {
- txTable.put(new Put(Bytes.toBytes("z" + i))
- .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
- .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
- }
- txContext.finish();
-
- txContext.start();
- // delete odd rows
- for (int i = 1; i < 10; i += 2) {
- txTable.delete(new Delete(Bytes.toBytes("z" + i)));
- }
- txContext.finish();
-
- txContext.start();
- int cnt = 0;
- ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
- Result res;
- while ((res = zScanner.next()) != null) {
- assertFalse(res.isEmpty());
- assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
- cnt += 2;
- }
-
- // Test 5: delete prior writes in the same transaction
- txContext.start();
- txTable.put(new Put(TestBytes.row4)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txTable.delete(new Delete(TestBytes.row4));
- txContext.finish();
-
- txContext.start();
- Result row4Result = txTable.get(new Get(TestBytes.row4));
- assertTrue(row4Result.isEmpty());
- txContext.finish();
- } finally {
- txTable.close();
- }
- }
-
- /**
- * Expect an exception since a transaction hasn't been started.
- *
- * @throws Exception
- */
- @Test(expected = IOException.class)
- public void testTransactionlessFailure() throws Exception {
- transactionAwareHTable.get(new Get(TestBytes.row));
- }
-
- /**
- * Tests that each transaction can see its own persisted writes, while not seeing writes from other
- * in-progress transactions.
- */
- @Test
- public void testReadYourWrites() throws Exception {
- // In-progress tx1: started before our main transaction
- HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
- TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
-
- // In-progress tx2: started while our main transaction is running
- HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
- TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
-
- // create an in-progress write that should be ignored
- byte[] col2 = Bytes.toBytes("col2");
- inprogressTxContext1.start();
- Put putCol2 = new Put(TestBytes.row);
- byte[] valueCol2 = Bytes.toBytes("writing in progress");
- putCol2.add(TestBytes.family, col2, valueCol2);
- txHTable1.put(putCol2);
-
- // start a tx and write a value to test reading in same tx
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] value = Bytes.toBytes("writing");
- put.add(TestBytes.family, TestBytes.qualifier, value);
- transactionAwareHTable.put(put);
-
- // test that a write from a tx started after the first is not visible
- inprogressTxContext2.start();
- Put put2 = new Put(TestBytes.row);
- byte[] value2 = Bytes.toBytes("writing2");
- put2.add(TestBytes.family, TestBytes.qualifier, value2);
- txHTable2.put(put2);
-
- Get get = new Get(TestBytes.row);
- Result row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
- assertNotNull(col1Value);
- assertArrayEquals(value, col1Value);
- // write from in-progress transaction should not be visible
- byte[] col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- // commit in-progress transaction, should still not be visible
- inprogressTxContext1.finish();
-
- get = new Get(TestBytes.row);
- row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- transactionContext.finish();
-
- inprogressTxContext2.abort();
- }
-
- @Test
- public void testRowLevelConflictDetection() throws Exception {
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- byte[] row1 = Bytes.toBytes("row1");
- byte[] row2 = Bytes.toBytes("row2");
- byte[] col1 = Bytes.toBytes("c1");
- byte[] col2 = Bytes.toBytes("c2");
- byte[] val1 = Bytes.toBytes("val1");
- byte[] val2 = Bytes.toBytes("val2");
-
- // test that concurrent writing to different rows succeeds
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
-
- // should be no conflicts
- txContext1.finish();
- txContext2.finish();
-
- transactionContext.start();
- Result res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
-
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // test that writing to different columns in the same row fails
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
-
- txContext2.start();
- txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row-level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- // should now be val2
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
-
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- // col2 should not be visible due to conflict
- assertNull(cell);
- transactionContext.finish();
-
- // test that writing to the same column in the same row fails
- txContext1.start();
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row and column level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- assertNotNull(cell);
- // should now be val1
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // verify change set that is being reported only on rows
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- Collection<byte[]> changeSet = txTable1.getTxChanges();
- assertNotNull(changeSet);
- assertEquals(2, changeSet.size());
- assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
- assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
- txContext1.finish();
- }
-
- @Test
- public void testNoneLevelConflictDetection() throws Exception {
- InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
-
- // overlapping writes to the same row + column should not conflict
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // changes should not be visible yet
- txContext2.start();
- Result row = txTable2.get(new Get(TestBytes.row));
- assertTrue(row.isEmpty());
-
- txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // both commits should succeed
- txContext1.finish();
- txContext2.finish();
-
- txContext1.start();
- row = txTable1.get(new Get(TestBytes.row));
- assertFalse(row.isEmpty());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
- txContext1.finish();
-
- // transaction abort should still rollback changes
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext1.abort();
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row2 = txTable2.get(new Get(TestBytes.row2));
- assertTrue(row2.isEmpty());
- txContext2.finish();
-
- // transaction invalidate should still make changes invisible
-
- txContext1.start();
- Transaction tx1 = txContext1.getCurrentTransaction();
- txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- assertNotNull(tx1);
- txClient.invalidate(tx1.getWritePointer());
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row3 = txTable2.get(new Get(TestBytes.row3));
- assertTrue(row3.isEmpty());
- txContext2.finish();
- }
-
- @Test
- public void testCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionContext.checkpoint();
- Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
- assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
- long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
- assertEquals(1, checkpointPtrs.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
-
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
- assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
- long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
- assertEquals(2, checkpointPtrs2.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
- assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
-
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // by default, all rows should be visible with Read-Your-Writes
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
-
- // when disabling current write pointer, only the previous checkpoints should be visible
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- Get get = new Get(TestBytes.row);
- verifyRow(transactionAwareHTable, get, TestBytes.value);
- get = new Get(TestBytes.row2);
- verifyRow(transactionAwareHTable, get, TestBytes.value2);
- get = new Get(TestBytes.row3);
- verifyRow(transactionAwareHTable, get, null);
-
- // test scan results excluding current write pointer
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
-
- Result row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row2, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNull(row);
- scanner.close();
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
-
- // commit transaction, verify writes are visible
- transactionContext.finish();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
- transactionContext.finish();
- }
-
- @Test
- public void testInProgressCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // check that writes are still not visible to other clients
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- txContext2.start();
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- txContext2.finish();
- txTable2.close();
-
- transactionContext.finish();
-
- // verify writes are visible after commit
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointRollback() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- transactionContext.abort();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, null);
- verifyRow(transactionAwareHTable, TestBytes.row2, null);
- verifyRow(transactionAwareHTable, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointInvalidate() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
- txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
-
- // check that writes are not visible
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
- txContext2.start();
- Transaction newTx = txContext2.getCurrentTransaction();
-
- // all 3 writes pointers from the previous transaction should now be excluded
- assertTrue(newTx.isExcluded(origTx.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
-
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- verifyRow(txTable2, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = txTable2.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- txContext2.finish();
- }
-
- @Test
- public void testExistingData() throws Exception {
- byte[] val11 = Bytes.toBytes("val11");
- byte[] val12 = Bytes.toBytes("val12");
- byte[] val21 = Bytes.toBytes("val21");
- byte[] val22 = Bytes.toBytes("val22");
- byte[] val31 = Bytes.toBytes("val31");
- byte[] val111 = Bytes.toBytes("val111");
-
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
- Collections.<String>emptyList()));
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Add some pre-existing, non-transactional data
- HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
- HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.flushCommits();
-
- // Add transactional data
- txContext.start();
- txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
- txContext.finish();
-
- txContext.start();
- // test get
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
-
- // test scan
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
-
- // test update and delete
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
- txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
-
- txContext.start();
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
- txContext.finish();
-
- // test scan
- txContext.start();
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
- }
-
- private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
- verifyRow(table, new Get(rowkey), expectedValue);
- }
-
- private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
- verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
- }
-
- private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
- Result result = table.get(get);
- if (expectedValues == null) {
- assertTrue(result.isEmpty());
- } else {
- assertFalse(result.isEmpty());
- byte[] family = TestBytes.family;
- byte[] col = TestBytes.qualifier;
- if (get.hasFamilies()) {
- family = get.getFamilyMap().keySet().iterator().next();
- col = get.getFamilyMap().get(family).first();
- }
- Iterator<Cell> it = result.getColumnCells(family, col).iterator();
- for (byte[] expectedValue : expectedValues) {
- Assert.assertTrue(it.hasNext());
- assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
- }
- }
- }
-
- private Cell[] getRow(HTableInterface table, Get get) throws Exception {
- Result result = table.get(get);
- return result.rawCells();
- }
-
- private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
- List<Cell> actualCells = new ArrayList<>();
- try (ResultScanner scanner = table.getScanner(scan)) {
- Result[] results = scanner.next(expectedCells.size() + 1);
- for (Result result : results) {
- actualCells.addAll(Lists.newArrayList(result.rawCells()));
- }
- Assert.assertEquals(expectedCells, actualCells);
- }
- }
-
- @Test
- public void testVisibilityAll() throws Exception {
- HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
- new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(nonTxTable,
- TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // start a transaction and create a delete marker
- txContext.start();
- //noinspection ConstantConditions
- long txWp0 = txContext.getCurrentTransaction().getWritePointer();
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // start a new transaction and write some values
- txContext.start();
- @SuppressWarnings("ConstantConditions")
- long txWp1 = txContext.getCurrentTransaction().getWritePointer();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
-
- // verify written data
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- TestBytes.value2);
-
- // checkpoint and make changes to written data now
- txContext.checkpoint();
- long txWp2 = txContext.getCurrentTransaction().getWritePointer();
- // delete a column
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
- // no change to a column
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- // update a column
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
- // delete column family
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
-
- // verify changed values
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- null);
-
- // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
- // and the raw change by prior transaction
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- List<KeyValue> expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // verify a Get is also able to return all snapshot versions
- Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
- Cell[] cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(3, cells.length);
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
- Assert.assertTrue(CellUtil.isDelete(cells[2]));
-
- verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- ImmutableList.of(TestBytes.value3, TestBytes.value));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
-
- // Verify VisibilityLevel.SNAPSHOT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
- txContext.finish();
-
- // finally verify values once more after commit, this time we should get only committed raw values for
- // all visibility levels
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- txContext.finish();
-
- // Test with regular HBase deletes in pre-existing data
- long now = System.currentTimeMillis();
- Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
- // to prevent Tephra from replacing delete with delete marker
- deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteColumn);
- Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
- // to prevent Tephra from replacing delete with delete marker
- deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteFamily);
- nonTxTable.flushCommits();
-
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
- new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
- );
- // test scan
- Scan scan = new Scan();
- scan.setRaw(true);
- verifyScan(txTable, scan, expected);
- txContext.finish();
- }
-
- @Test
- public void testFilters() throws Exception {
- // Add some values to table
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] val1 = Bytes.toBytes(1L);
- put.add(TestBytes.family, TestBytes.qualifier, val1);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row2);
- byte[] val2 = Bytes.toBytes(2L);
- put.add(TestBytes.family, TestBytes.qualifier, val2);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row3);
- byte[] val3 = Bytes.toBytes(3L);
- put.add(TestBytes.family, TestBytes.qualifier, val3);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row4);
- byte[] val4 = Bytes.toBytes(4L);
- put.add(TestBytes.family, TestBytes.qualifier, val4);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Delete cell with value 2
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row2);
- delete.deleteColumn(TestBytes.family, TestBytes.qualifier);
- transactionAwareHTable.delete(delete);
- transactionContext.finish();
-
- // Scan for values less than 4, should get only values 1 and 3
- transactionContext.start();
- Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS,
- new BinaryComparator(Longs.toByteArray(4))));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run a Get with a filter for less than 10 on row4, should get value 4
- transactionContext.start();
- Get get = new Get(TestBytes.row4);
- get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(Longs.toByteArray(10))));
- Result result = transactionAwareHTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
- transactionContext.finish();
-
- // Change value of row4 to 40
- transactionContext.start();
- put = new Put(TestBytes.row4);
- byte[] val40 = Bytes.toBytes(40L);
- put.add(TestBytes.family, TestBytes.qualifier, val40);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Scan for values less than 10, should get only values 1 and 3
- transactionContext.start();
- scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS,
- new BinaryComparator(Longs.toByteArray(10))));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run the Get again with a filter for less than 10 on row4, this time should not get any results
- transactionContext.start();
- result = transactionAwareHTable.get(get);
- assertTrue(result.isEmpty());
- transactionContext.finish();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilterTest.java
deleted file mode 100644
index 82bac83..0000000
--- a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/CellSkipFilterTest.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * HBase 0.96 specific test for testing {@link CellSkipFilter}.
- */
-public class CellSkipFilterTest {
-
- private static final String ROW1KEY = "row1";
- private static final String ROW2KEY = "row2";
- private static final String FAM1KEY = "fam1";
- private static final String COL1KEY = "col1";
- private static final String FAM2KEY = "fam2";
- private static final String COL2KEY = "col2";
- private static final String VALUE = "value";
-
- @Test
- public void testSkipFiltering() throws Exception {
- long timestamp = System.currentTimeMillis();
- // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
- Filter filter = new CellSkipFilter(new MyFilter(0));
- assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 1)));
-
- // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give SKIP from MyFilter
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
- timestamp - 4)));
-
- // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
- filter = new CellSkipFilter(new MyFilter(2));
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 1)));
-
- // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give NEXT_ROW from MyFilter
- assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
- timestamp - 4)));
-
- // Next call with the new key should returned the SEEK_NEXT_USING_HINT
- assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 5)));
- }
-
- private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
- timestamp, Bytes.toBytes(value));
- }
-
- /**
- * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
- * start index in the list and start serving the return codes corresponding that that index. Every time the
- * return code is served, index is incremented.
- */
- class MyFilter extends FilterBase {
-
- private final List<ReturnCode> returnCodes;
- private int counter;
-
- public MyFilter(int startIndex) {
- returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
- ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
- counter = startIndex;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- ReturnCode code = returnCodes.get(counter % returnCodes.size());
- counter++;
- return code;
- }
- }
-}
[09/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java
deleted file mode 100644
index fd1e9bc..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java
+++ /dev/null
@@ -1,620 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh.coprocessor;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.HDFSTransactionStateStorage;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.snapshot.DefaultSnapshotCodec;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.MockRegionServerServices;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALFactory;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
- */
-public class TransactionProcessorTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
-
- // 8 versions, 1 hour apart, latest is current ts.
- private static final long[] V;
-
- static {
- long now = System.currentTimeMillis();
- V = new long[9];
- for (int i = 0; i < V.length; i++) {
- V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
- }
- }
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
- private static MiniDFSCluster dfsCluster;
- private static Configuration conf;
- private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
- private static TransactionVisibilityState txVisibilityState;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- Configuration hConf = new Configuration();
- String rootDir = tmpFolder.newFolder().getAbsolutePath();
- hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
- hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
-
- dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
- dfsCluster.waitActive();
- conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
-
- conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
- conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- String localTestDir = tmpFolder.newFolder().getAbsolutePath();
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
-
- // write an initial transaction snapshot
- TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(
- System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
- // this will set visibility upper bound to V[6]
- Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
- TransactionType.SHORT))),
- new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
- txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
- txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
- txSnapshot.getInProgress());
- HDFSTransactionStateStorage tmpStorage =
- new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
- tmpStorage.startAndWait();
- tmpStorage.writeSnapshot(txSnapshot);
- tmpStorage.stopAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- dfsCluster.shutdown();
- }
-
- @Test
- public void testDataJanitorRegionScanner() throws Exception {
- String tableName = "TestRegionScanner";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- for (int i = 1; i <= 8; i++) {
- for (int k = 1; k <= i; k++) {
- Put p = new Put(Bytes.toBytes(i));
- p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
- region.put(p);
- }
- }
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should only return the valid rows
- // do not use a filter here to test that cleanup works on flush
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
-
- // first returned value should be "4" with version "4"
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 4, new long[] {V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 5, new long[] {V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});
-
- results.clear();
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteFiltering() throws Exception {
- String tableName = "TestDeleteFiltering";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- byte[] row = Bytes.toBytes(1);
- for (int i = 4; i < V.length; i++) {
- Put p = new Put(row);
- p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
- region.put(p);
- }
-
- // delete from the third entry back
- // take that cell's timestamp + 1 to simulate a delete in a new tx
- long deleteTs = V[5] + 1;
- Delete d = new Delete(row, deleteTs);
- LOG.info("Issuing delete at timestamp " + deleteTs);
- // row deletes are not yet supported (TransactionAwareHTable normally handles this)
- d.deleteColumns(familyBytes, columnBytes);
- region.delete(d);
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, we should drop the deleted version, but not the others
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should return row with versions at: V[8], V[6].
- // V[7] is invalid and V[5] and prior are deleted.
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
- // should be only one row
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 1,
- new long[]{V[8], V[6], deleteTs},
- new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteMarkerCleanup() throws Exception {
- String tableName = "TestDeleteMarkerCleanup";
- byte[] familyBytes = Bytes.toBytes("f");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
-
- // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
- long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
- // deletes are performed after the writes, but still before the visibility upper bound
- long deleteTs = writeTs + 1;
- // write separate columns to confirm that delete markers survive across flushes
- byte[] row = Bytes.toBytes(100);
- Put p = new Put(row);
-
- LOG.info("Writing columns at timestamp " + writeTs);
- for (int i = 0; i < 5; i++) {
- byte[] iBytes = Bytes.toBytes(i);
- p.add(familyBytes, iBytes, writeTs, iBytes);
- }
- region.put(p);
- // read all back
- Scan scan = new Scan(row);
- RegionScanner regionScanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- for (int i = 0; i < 5; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a flush to clear the memstore
- LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // delete the odd entries
- for (int i = 0; i < 5; i++) {
- if (i % 2 == 1) {
- // deletes are performed as puts with empty values
- Put deletePut = new Put(row);
- deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
- region.put(deletePut);
- }
- }
-
- // read all back
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- LOG.info("Got cell " + cell);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force another flush on the delete markers
- // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
- LOG.info("After delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a major compaction
- LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
- region.compactStores(true);
-
- // perform a raw scan (no filter) to confirm that the delete markers are now gone
- scan = new Scan(row);
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
- } finally {
- region.close();
- }
- }
-
- /**
- * Test that we correctly preserve the timestamp set for column family delete markers. This is not
- * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
- * that we make it easy to interoperate with other systems.
- */
- @Test
- public void testFamilyDeleteTimestamp() throws Exception {
- String tableName = "TestFamilyDeleteTimestamp";
- byte[] family1Bytes = Bytes.toBytes("f1");
- byte[] columnBytes = Bytes.toBytes("c");
- byte[] rowBytes = Bytes.toBytes("row");
- byte[] valBytes = Bytes.toBytes("val");
- HRegion region = createRegion(tableName, family1Bytes, 0);
- try {
- region.initialize();
-
- long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
- Put p = new Put(rowBytes);
- p.add(family1Bytes, columnBytes, now - 10, valBytes);
- region.put(p);
-
- // issue a family delete with an explicit timestamp
- Delete delete = new Delete(rowBytes, now);
- delete.deleteFamily(family1Bytes, now - 5);
- region.delete(delete);
-
- // test that the delete marker preserved the timestamp
- Scan scan = new Scan();
- scan.setMaxVersions();
- RegionScanner scanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(2, results.size());
- // delete marker should appear first
- Cell cell = results.get(0);
- assertArrayEquals(new byte[0], cell.getQualifier());
- assertArrayEquals(new byte[0], cell.getValue());
- assertEquals(now - 5, cell.getTimestamp());
- // since this is an unfiltered scan against the region, the original put should be next
- cell = results.get(1);
- assertArrayEquals(valBytes, cell.getValue());
- assertEquals(now - 10, cell.getTimestamp());
- scanner.close();
-
-
- // with a filtered scan the original put should disappear
- scan = new Scan();
- scan.setMaxVersions();
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- scanner = region.getScanner(scan);
- results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(0, results.size());
- scanner.close();
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testPreExistingData() throws Exception {
- String tableName = "TestPreExistingData";
- byte[] familyBytes = Bytes.toBytes("f");
- long ttlMillis = TimeUnit.DAYS.toMillis(14);
- HRegion region = createRegion(tableName, familyBytes, ttlMillis);
- try {
- region.initialize();
-
- // timestamps for pre-existing, non-transactional data
- long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
- long older = now - ttlMillis / 2;
- long newer = now - ttlMillis / 3;
- // timestamps for transactional data
- long nowTx = txVisibilityState.getVisibilityUpperBound();
- long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
- long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(familyBytes, ttlMillis);
-
- List<Cell> cells = new ArrayList<>();
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
-
- // Write non-transactional and transactional data
- for (Cell c : cells) {
- region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
- }
-
- Scan rawScan = new Scan();
- rawScan.setMaxVersions();
-
- Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
- Scan txScan = new Scan();
- txScan.setMaxVersions();
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // read all back with raw scanner
- scanAndAssert(region, cells, rawScan);
-
- // read all back with transaction filter
- scanAndAssert(region, cells, txScan);
-
- // force a flush to clear the memstore
- region.flushcache();
- scanAndAssert(region, cells, txScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- scanAndAssert(region, cells, txScan);
-
- // Reduce TTL, this should make cells with timestamps older and olderTx expire
- long newTtl = ttlMillis / 2 - 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // Raw scan should still give all cells
- scanAndAssert(region, cells, rawScan);
- // However, tx scan should not return expired cells
- scanAndAssert(region, select(cells, 1, 3, 5), txScan);
-
- region.flushcache();
- scanAndAssert(region, cells, rawScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- // This time raw scan too should not return expired cells, as they would be dropped during major compaction
- scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
-
- // Reduce TTL again to 1 ms, this should expire all cells
- newTtl = 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // force a major compaction to remove expired cells
- region.compactStores(true);
- // This time raw scan should not return any cells, as all cells have expired.
- scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
- } finally {
- region.close();
- }
- }
-
- private List<Cell> select(List<Cell> cells, int... indexes) {
- List<Cell> newCells = new ArrayList<>();
- for (int i : indexes) {
- newCells.add(cells.get(i));
- }
- return newCells;
- }
-
- @SuppressWarnings("StatementWithEmptyBody")
- private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
- try (RegionScanner regionScanner = region.getScanner(scan)) {
- List<Cell> results = Lists.newArrayList();
- while (regionScanner.next(results)) { }
- assertEquals(expected, results);
- }
- }
-
- private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
- region.close();
- HTableDescriptor htd = region.getTableDesc();
- HColumnDescriptor cfd = htd.getFamily(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
- new LocalRegionServerServices(conf, ServerName.valueOf(
- InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
- }
-
- private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
- HColumnDescriptor cfd = new HColumnDescriptor(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- htd.addFamily(cfd);
- htd.addCoprocessor(TransactionProcessor.class.getName());
- Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
- FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.mkdirs(tablePath));
- WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
- WAL hLog = walFactory.getWAL(new byte[]{1});
- HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
- HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
- return new HRegion(regionFS, hLog, conf, htd,
- new LocalRegionServerServices(conf, ServerName.valueOf(
- InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
- byte[][] values = new byte[versions.length][];
- for (int i = 0; i < versions.length; i++) {
- values[i] = Bytes.toBytes(versions[i]);
- }
- assertKeyValueMatches(results, index, versions, values);
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
- assertEquals(versions.length, results.size());
- assertEquals(values.length, results.size());
- for (int i = 0; i < versions.length; i++) {
- Cell kv = results.get(i);
- assertArrayEquals(Bytes.toBytes(index), kv.getRow());
- assertEquals(versions[i], kv.getTimestamp());
- assertArrayEquals(values[i], kv.getValue());
- }
- }
-
- @Test
- public void testTransactionStateCache() throws Exception {
- TransactionStateCache cache = new TransactionStateCache();
- cache.setConf(conf);
- cache.startAndWait();
- // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
- TransactionVisibilityState cachedSnapshot = cache.getLatestState();
- assertNotNull(cachedSnapshot);
- assertEquals(invalidSet, cachedSnapshot.getInvalid());
- cache.stopAndWait();
- }
-
- private static class LocalRegionServerServices extends MockRegionServerServices {
- private final ServerName serverName;
-
- public LocalRegionServerServices(Configuration conf, ServerName serverName) {
- super(conf);
- this.serverName = serverName;
- }
-
- @Override
- public ServerName getServerName() {
- return serverName;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java
deleted file mode 100644
index 40487a1..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase.AbstractTransactionVisibilityFilterTest;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * HBase 1.0 (CDH) specific test for filtering logic applied when reading data transactionally.
- */
-public class TransactionVisibilityFilterTest extends AbstractTransactionVisibilityFilterTest {
- /**
- * Test filtering of KeyValues for in-progress and invalid transactions.
- * @throws Exception
- */
- @Test
- public void testFiltering() throws Exception {
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
- }
-
- @Test
- public void testSubFilter() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- @Test
- public void testSubFilterOverride() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- private void runFilteringTest(TxFilterFactory txFilterFactory,
- List<Filter.ReturnCode> assertCodes) throws Exception {
- /*
- * Start and stop some transactions. This will give us a transaction state something like the following
- * (numbers only reflect ordering, not actual transaction IDs):
- * 6 - in progress
- * 5 - committed
- * 4 - invalid
- * 3 - in-progress
- * 2 - committed
- * 1 - committed
- *
- * read ptr = 5
- * write ptr = 6
- */
-
- Transaction tx1 = txManager.startShort();
- assertTrue(txManager.canCommit(tx1, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx1));
-
- Transaction tx2 = txManager.startShort();
- assertTrue(txManager.canCommit(tx2, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx2));
-
- Transaction tx3 = txManager.startShort();
- Transaction tx4 = txManager.startShort();
- txManager.invalidate(tx4.getTransactionId());
-
- Transaction tx5 = txManager.startShort();
- assertTrue(txManager.canCommit(tx5, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx5));
-
- Transaction tx6 = txManager.startShort();
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- Filter filter = txFilterFactory.getTxFilter(tx6, ttls);
-
- assertEquals(assertCodes.get(5),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx6.getTransactionId())));
- assertEquals(assertCodes.get(4),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx5.getTransactionId())));
- assertEquals(assertCodes.get(3),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx4.getTransactionId())));
- assertEquals(assertCodes.get(2),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx3.getTransactionId())));
- assertEquals(assertCodes.get(1),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx2.getTransactionId())));
- assertEquals(assertCodes.get(0),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx1.getTransactionId())));
- }
-
- /**
- * Test filtering for TTL settings.
- * @throws Exception
- */
- @Test
- public void testTTLFiltering() throws Exception {
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(FAM, 10L);
- ttls.put(FAM2, 30L);
- ttls.put(FAM3, 0L);
-
- Transaction tx = txManager.startShort();
- long now = tx.getVisibilityUpperBound();
- Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 21 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 1001 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
-
- // Verify ttl for pre-existing, non-transactional data
- long preNow = now / TxConstants.MAX_TX_PER_MS;
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 11L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 1001L)));
- }
-
- protected KeyValue newKeyValue(String rowkey, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), FAM, COL, timestamp, Bytes.toBytes(value));
- }
-
- protected KeyValue newKeyValue(String rowkey, byte[] family, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), family, COL, timestamp, Bytes.toBytes(value));
- }
-
- private interface TxFilterFactory {
- Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs);
- }
-
- private class CustomTxFilter extends TransactionVisibilityFilter {
- public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
- @Nullable Filter cellFilter) {
- super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
- }
-
- @Override
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- switch (subFilterCode) {
- case INCLUDE:
- return ReturnCode.INCLUDE;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java
new file mode 100644
index 0000000..7784812
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh;
+
+import org.apache.tephra.util.AbstractConfigurationProviderTest;
+import org.apache.tephra.util.HBaseVersion;
+
+/**
+ * Test for HBase 1.0 (CDH) version specific behavior.
+ */
+public class HBase10ConfigurationProviderTest extends AbstractConfigurationProviderTest {
+ @Override
+ protected HBaseVersion.Version getExpectedVersion() {
+ return HBaseVersion.Version.HBASE_10_CDH;
+ }
+}
[44/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionType.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionType.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionType.java
deleted file mode 100644
index 0d10eb3..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionType.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-
-import java.util.Map;
-import java.util.HashMap;
-import org.apache.thrift.TEnum;
-
-public enum TTransactionType implements org.apache.thrift.TEnum {
- SHORT(1),
- LONG(2);
-
- private final int value;
-
- private TTransactionType(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static TTransactionType findByValue(int value) {
- switch (value) {
- case 1:
- return SHORT;
- case 2:
- return LONG;
- default:
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TVisibilityLevel.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TVisibilityLevel.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TVisibilityLevel.java
deleted file mode 100644
index 6de6f87..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TVisibilityLevel.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-
-public enum TVisibilityLevel implements org.apache.thrift.TEnum {
- SNAPSHOT(1),
- SNAPSHOT_EXCLUDE_CURRENT(2),
- SNAPSHOT_ALL(3);
-
- private final int value;
-
- private TVisibilityLevel(int value) {
- this.value = value;
- }
-
- /**
- * Get the integer value of this enum value, as defined in the Thrift IDL.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Find a the enum type by its integer value, as defined in the Thrift IDL.
- * @return null if the value is not found.
- */
- public static TVisibilityLevel findByValue(int value) {
- switch (value) {
- case 1:
- return SNAPSHOT;
- case 2:
- return SNAPSHOT_EXCLUDE_CURRENT;
- case 3:
- return SNAPSHOT_ALL;
- default:
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/inmemory/DetachedTxSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/inmemory/DetachedTxSystemClient.java b/tephra-core/src/main/java/co/cask/tephra/inmemory/DetachedTxSystemClient.java
deleted file mode 100644
index 1731b0c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/inmemory/DetachedTxSystemClient.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.inmemory;
-
-import co.cask.tephra.InvalidTruncateTimeException;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCouldNotTakeSnapshotException;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Implementation of the tx system client that doesn't talk to any global service and tries to do its best to meet the
- * tx system requirements/expectations. In fact it implements enough logic to support running flows (when each flowlet
- * uses its own detached tx system client, without talking to each other and sharing any state) with "process exactly
- * once" guarantee if no failures happen.
- *
- * NOTE: Will NOT detect conflicts. May leave inconsistent state when process crashes. Does NOT provide even read
- * isolation guarantees.
- *
- * Good for performance testing. For demoing high throughput. For use-cases with relaxed tx guarantees.
- */
-public class DetachedTxSystemClient implements TransactionSystemClient {
- // Dataset and queue logic relies on tx id to grow monotonically even after restart. Hence we need to start with
- // value that is for sure bigger than the last one used before restart.
- // NOTE: with code below we assume we don't do more than InMemoryTransactionManager.MAX_TX_PER_MS tx/ms
- // by single client
- private AtomicLong generator = new AtomicLong(System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS);
-
- @Override
- public Transaction startShort() {
- long wp = getWritePointer();
- // NOTE: -1 here is because we have logic that uses (readpointer + 1) as a "exclusive stop key" in some datasets
- return new Transaction(
- Long.MAX_VALUE - 1, wp, new long[0], new long[0],
- Transaction.NO_TX_IN_PROGRESS, TransactionType.SHORT);
- }
-
- private long getWritePointer() {
- long wp = generator.incrementAndGet();
- // NOTE: using InMemoryTransactionManager.MAX_TX_PER_MS to be at least close to real one
- long now = System.currentTimeMillis();
- if (wp < now * TxConstants.MAX_TX_PER_MS) {
- // trying to advance to align with timestamp, but only once: if failed, we'll just try again later with next tx
- long advanced = now * TxConstants.MAX_TX_PER_MS;
- if (generator.compareAndSet(wp, advanced)) {
- wp = advanced;
- }
- }
- return wp;
- }
-
- @Override
- public Transaction startShort(int timeout) {
- return startShort();
- }
-
- @Override
- public Transaction startLong() {
- return startShort();
- }
-
- @Override
- public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) {
- return true;
- }
-
- @Override
- public boolean commit(Transaction tx) {
- return true;
- }
-
- @Override
- public void abort(Transaction tx) {
- // do nothing
- }
-
- @Override
- public boolean invalidate(long tx) {
- return true;
- }
-
- @Override
- public Transaction checkpoint(Transaction tx) {
- long newWritePointer = getWritePointer();
- LongArrayList newCheckpointPointers = new LongArrayList(tx.getCheckpointWritePointers());
- newCheckpointPointers.add(newWritePointer);
- return new Transaction(tx, newWritePointer, newCheckpointPointers.toLongArray());
- }
-
- @Override
- public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
- throw new TransactionCouldNotTakeSnapshotException(
- "Snapshot not implemented in detached transaction system client");
- }
-
- @Override
- public String status() {
- return TxConstants.STATUS_OK;
- }
-
- @Override
- public void resetState() {
- // do nothing
- }
-
- @Override
- public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
- return true;
- }
-
- @Override
- public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
- return true;
- }
-
- @Override
- public int getInvalidSize() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTransactionService.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTransactionService.java b/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTransactionService.java
deleted file mode 100644
index feecdce..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTransactionService.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.inmemory;
-
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TxConstants;
-import com.google.common.util.concurrent.AbstractService;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.common.Cancellable;
-import org.apache.twill.discovery.Discoverable;
-import org.apache.twill.discovery.DiscoveryService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-
-/**
- * Transaction server that manages transaction data for the Reactor.
- * <p>
- * Transaction server is HA, one can start multiple instances, only one of which is active and will register itself in
- * discovery service.
- * </p>
- */
-public class InMemoryTransactionService extends AbstractService {
- private static final Logger LOG = LoggerFactory.getLogger(InMemoryTransactionService.class);
-
- private final DiscoveryService discoveryService;
- private final String serviceName;
- protected final Provider<TransactionManager> txManagerProvider;
- private Cancellable cancelDiscovery;
- protected TransactionManager txManager;
-
- // thrift server config
- protected final String address;
- protected final int port;
- protected final int threads;
- protected final int ioThreads;
- protected final int maxReadBufferBytes;
-
- @Inject
- public InMemoryTransactionService(Configuration conf,
- DiscoveryService discoveryService,
- Provider<TransactionManager> txManagerProvider) {
-
- this.discoveryService = discoveryService;
- this.txManagerProvider = txManagerProvider;
- this.serviceName = conf.get(TxConstants.Service.CFG_DATA_TX_DISCOVERY_SERVICE_NAME,
- TxConstants.Service.DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME);
-
- address = conf.get(TxConstants.Service.CFG_DATA_TX_BIND_ADDRESS, TxConstants.Service.DEFAULT_DATA_TX_BIND_ADDRESS);
- port = conf.getInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, TxConstants.Service.DEFAULT_DATA_TX_BIND_PORT);
-
- // Retrieve the number of threads for the service
- threads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS,
- TxConstants.Service.DEFAULT_DATA_TX_SERVER_THREADS);
- ioThreads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS,
- TxConstants.Service.DEFAULT_DATA_TX_SERVER_IO_THREADS);
-
- maxReadBufferBytes = conf.getInt(TxConstants.Service.CFG_DATA_TX_THRIFT_MAX_READ_BUFFER,
- TxConstants.Service.DEFAULT_DATA_TX_THRIFT_MAX_READ_BUFFER);
-
- LOG.info("Configuring TransactionService" +
- ", address: " + address +
- ", port: " + port +
- ", threads: " + threads +
- ", io threads: " + ioThreads +
- ", max read buffer (bytes): " + maxReadBufferBytes);
- }
-
- protected void undoRegister() {
- if (cancelDiscovery != null) {
- cancelDiscovery.cancel();
- }
- }
-
- protected void doRegister() {
- cancelDiscovery = discoveryService.register(new Discoverable() {
- @Override
- public String getName() {
- return serviceName;
- }
-
- @Override
- public InetSocketAddress getSocketAddress() {
- return getAddress();
- }
- });
- }
-
- protected InetSocketAddress getAddress() {
- return new InetSocketAddress(1);
- }
-
- @Override
- protected void doStart() {
- try {
- txManager = txManagerProvider.get();
- txManager.startAndWait();
- doRegister();
- LOG.info("Transaction Thrift service started successfully on " + getAddress());
- notifyStarted();
- } catch (Throwable t) {
- LOG.info("Transaction Thrift service didn't start on " + getAddress());
- notifyFailed(t);
- }
- }
-
- @Override
- protected void doStop() {
- undoRegister();
- txManager.stopAndWait();
- notifyStopped();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTxSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTxSystemClient.java b/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTxSystemClient.java
deleted file mode 100644
index ba15269..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/inmemory/InMemoryTxSystemClient.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.inmemory;
-
-import co.cask.tephra.InvalidTruncateTimeException;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCouldNotTakeSnapshotException;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionNotInProgressException;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import com.google.inject.Inject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.Set;
-
-/**
- *
- */
-public class InMemoryTxSystemClient implements TransactionSystemClient {
-
- private static final Logger LOG = LoggerFactory.getLogger(InMemoryTxSystemClient.class);
-
- TransactionManager txManager;
-
- @Inject
- public InMemoryTxSystemClient(TransactionManager txmgr) {
- txManager = txmgr;
- }
-
- @Override
- public Transaction startLong() {
- return txManager.startLong();
- }
-
- @Override
- public Transaction startShort() {
- return txManager.startShort();
- }
-
- @Override
- public Transaction startShort(int timeout) {
- return txManager.startShort(timeout);
- }
-
- @Override
- public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
- return changeIds.isEmpty() || txManager.canCommit(tx, changeIds);
- }
-
- @Override
- public boolean commit(Transaction tx) throws TransactionNotInProgressException {
- return txManager.commit(tx);
- }
-
- @Override
- public void abort(Transaction tx) {
- txManager.abort(tx);
- }
-
- @Override
- public boolean invalidate(long tx) {
- return txManager.invalidate(tx);
- }
-
- @Override
- public Transaction checkpoint(Transaction tx) throws TransactionNotInProgressException {
- return txManager.checkpoint(tx);
- }
-
- @Override
- public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
- try {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- try {
- boolean snapshotTaken = txManager.takeSnapshot(out);
- if (!snapshotTaken) {
- throw new TransactionCouldNotTakeSnapshotException("Transaction manager did not take a snapshot.");
- }
- } finally {
- out.close();
- }
- return new ByteArrayInputStream(out.toByteArray());
- } catch (IOException e) {
- LOG.error("Snapshot could not be taken", e);
- throw new TransactionCouldNotTakeSnapshotException(e.getMessage());
- }
- }
-
- @Override
- public String status() {
- return txManager.isRunning() ? TxConstants.STATUS_OK : TxConstants.STATUS_NOTOK;
- }
-
- @Override
- public void resetState() {
- txManager.resetState();
- }
-
- @Override
- public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
- return txManager.truncateInvalidTx(invalidTxIds);
- }
-
- @Override
- public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
- return txManager.truncateInvalidTxBefore(time);
- }
-
- @Override
- public int getInvalidSize() {
- return txManager.getInvalidSize();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/inmemory/MinimalTxSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/inmemory/MinimalTxSystemClient.java b/tephra-core/src/main/java/co/cask/tephra/inmemory/MinimalTxSystemClient.java
deleted file mode 100644
index 6f83565..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/inmemory/MinimalTxSystemClient.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.inmemory;
-
-import co.cask.tephra.InvalidTruncateTimeException;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCouldNotTakeSnapshotException;
-import co.cask.tephra.TransactionNotInProgressException;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.Set;
-
-/**
- * Dummy implementation of TxSystemClient. May be useful for perf testing.
- */
-public class MinimalTxSystemClient implements TransactionSystemClient {
- private long currentTxPointer = 1;
-
- @Override
- public Transaction startShort() {
- long wp = currentTxPointer++;
- // NOTE: -1 here is because we have logic that uses (readpointer + 1) as a "exclusive stop key" in some datasets
- return new Transaction(
- Long.MAX_VALUE - 1, wp, new long[0], new long[0],
- Transaction.NO_TX_IN_PROGRESS, TransactionType.SHORT);
- }
-
- @Override
- public Transaction startShort(int timeout) {
- return startShort();
- }
-
- @Override
- public Transaction startLong() {
- return startShort();
- }
-
- @Override
- public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) {
- return true;
- }
-
- @Override
- public boolean commit(Transaction tx) {
- return true;
- }
-
- @Override
- public void abort(Transaction tx) {
- // do nothing
- }
-
- @Override
- public boolean invalidate(long tx) {
- return true;
- }
-
- @Override
- public Transaction checkpoint(Transaction tx) throws TransactionNotInProgressException {
- return tx;
- }
-
- @Override
- public InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException {
- throw new TransactionCouldNotTakeSnapshotException("Not snapshot to take.");
- }
-
- @Override
- public String status() {
- return TxConstants.STATUS_OK;
- }
-
- @Override
- public void resetState() {
- // do nothing
- }
-
- @Override
- public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
- return true;
- }
-
- @Override
- public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
- return true;
- }
-
- @Override
- public int getInvalidSize() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/inmemory/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/inmemory/package-info.java b/tephra-core/src/main/java/co/cask/tephra/inmemory/package-info.java
deleted file mode 100644
index b7a7c59..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/inmemory/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains in memory implementation of the transaction system v2.
- */
-package co.cask.tephra.inmemory;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/metrics/DefaultMetricsCollector.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/metrics/DefaultMetricsCollector.java b/tephra-core/src/main/java/co/cask/tephra/metrics/DefaultMetricsCollector.java
deleted file mode 100644
index 8e33b4d..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/metrics/DefaultMetricsCollector.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.metrics;
-
-import co.cask.tephra.TxConstants;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.JmxReporter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Slf4jReporter;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Default metrics collector implementation using <a href="http://metrics.dropwizard.io">Yammer Metrics</a>.
- *
- * <p>The reporting frequency for this collector can be configured by setting the
- * {@code data.tx.metrics.period} configuration property to the reporting frequency in seconds.
- * </p>
- */
-public class DefaultMetricsCollector extends TxMetricsCollector {
- private static final Logger LOG = LoggerFactory.getLogger(DefaultMetricsCollector.class);
-
- private final MetricRegistry metrics = new MetricRegistry();
- private JmxReporter jmxReporter;
- private ScheduledReporter reporter;
- private int reportPeriod;
- private ConcurrentMap<String, AtomicLong> gauges = Maps.newConcurrentMap();
-
- @Override
- public void configure(Configuration conf) {
- // initialize selected output reporter
- reportPeriod = conf.getInt(TxConstants.Metrics.REPORT_PERIOD_KEY, TxConstants.Metrics.REPORT_PERIOD_DEFAULT);
- LOG.info("Configured metrics report to emit every {} seconds", reportPeriod);
- // TODO: reporters should be pluggable based on injection
- jmxReporter = JmxReporter.forRegistry(metrics).build();
- reporter = Slf4jReporter.forRegistry(metrics)
- .outputTo(LoggerFactory.getLogger("tephra-metrics"))
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- }
-
-
- @Override
- public void gauge(String metricName, int value, String... tags) {
- AtomicLong gauge = gauges.get(metricName);
- if (gauge == null) {
- final AtomicLong newValue = new AtomicLong();
- if (gauges.putIfAbsent(metricName, newValue) == null) {
- // first to set the value, need to register the metric
- metrics.register(metricName, new Gauge<Long>() {
- @Override
- public Long getValue() {
- return newValue.get();
- }
- });
- gauge = newValue;
- } else {
- // someone else set it first
- gauge = gauges.get(metricName);
- }
- }
- gauge.set(value);
- }
-
- @Override
- public void histogram(String metricName, int value) {
- metrics.histogram(metricName).update(value);
- }
-
- @Override
- public void rate(String metricName) {
- metrics.meter(metricName).mark();
- }
-
- @Override
- public void rate(String metricName, int count) {
- metrics.meter(metricName).mark(count);
- }
-
- @Override
- protected void startUp() throws Exception {
- jmxReporter.start();
- reporter.start(reportPeriod, TimeUnit.SECONDS);
- LOG.info("Started metrics reporter");
- }
-
- @Override
- protected void shutDown() throws Exception {
- jmxReporter.stop();
- reporter.stop();
- LOG.info("Stopped metrics reporter");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/metrics/MetricsCollector.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/metrics/MetricsCollector.java b/tephra-core/src/main/java/co/cask/tephra/metrics/MetricsCollector.java
deleted file mode 100644
index 3aae4e0..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/metrics/MetricsCollector.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.metrics;
-
-import com.google.common.util.concurrent.Service;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Basic API for Tephra to support system metrics.
- */
-public interface MetricsCollector extends Service {
- /**
- * Report a metric as an absolute value.
- */
- void gauge(String metricName, int value, String... tags);
-
- /**
- * Report a metric as a count over a given time duration. This method uses an implicit count of 1.
- */
- void rate(String metricName);
-
- /**
- * Report a metric as a count over a given time duration.
- */
- void rate(String metricName, int count);
-
- /**
- * Report a metric calculating the distribution of the value.
- */
- void histogram(String metricName, int value);
-
- /**
- * Called before the collector service is started, allowing the collector to setup any
- * required configuration.
- */
- void configure(Configuration conf);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/metrics/TxMetricsCollector.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/metrics/TxMetricsCollector.java b/tephra-core/src/main/java/co/cask/tephra/metrics/TxMetricsCollector.java
deleted file mode 100644
index 7fba8e5..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/metrics/TxMetricsCollector.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.metrics;
-
-import com.google.common.util.concurrent.AbstractIdleService;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Metrics Collector Class, to emit Transaction Related Metrics.
- * Note: This default implementation is a no-op and doesn't emit any metrics
- */
-public class TxMetricsCollector extends AbstractIdleService implements MetricsCollector {
-
- @Override
- public void gauge(String metricName, int value, String... tags) {
- //no-op
- }
-
- @Override
- public void rate(String metricName) {
- // no-op
- }
-
- @Override
- public void rate(String metricName, int count) {
- // no-op
- }
-
- @Override
- public void histogram(String metricName, int value) {
- // no-op
- }
-
- @Override
- public void configure(Configuration conf) {
- // no-op
- }
-
- /* Service methods */
-
- @Override
- protected void startUp() throws Exception {
- // no-op
- }
-
- @Override
- protected void shutDown() throws Exception {
- // no-op
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/package-info.java b/tephra-core/src/main/java/co/cask/tephra/package-info.java
deleted file mode 100644
index 6f2f858..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains implementations of the transaction system v2.
- */
-package co.cask.tephra;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionLog.java b/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionLog.java
deleted file mode 100644
index 173cc9f..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionLog.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.metrics.MetricsCollector;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Common implementation of a transaction log, backed by file reader and writer based storage. Classes extending
- * this class, must also implement {@link TransactionLogWriter} and {@link TransactionLogReader}.
- */
-public abstract class AbstractTransactionLog implements TransactionLog {
- /** Time limit, in milliseconds, of an append to the transaction log before we log it as "slow". */
- private static final long SLOW_APPEND_THRESHOLD = 1000L;
-
- private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionLog.class);
-
- private final AtomicLong logSequence = new AtomicLong();
- private final MetricsCollector metricsCollector;
- protected long timestamp;
- private volatile boolean initialized;
- private volatile boolean closed;
- private AtomicLong syncedUpTo = new AtomicLong();
- private List<Entry> pendingWrites = Lists.newLinkedList();
- private TransactionLogWriter writer;
-
- public AbstractTransactionLog(long timestamp, MetricsCollector metricsCollector) {
- this.timestamp = timestamp;
- this.metricsCollector = metricsCollector;
- }
-
- /**
- * Initializes the log file, opening a file writer. Clients calling {@code init()} should ensure that they
- * also call {@link HDFSTransactionLog#close()}.
- * @throws java.io.IOException If an error is encountered initializing the file writer.
- */
- public synchronized void init() throws IOException {
- if (initialized) {
- return;
- }
- this.writer = createWriter();
- this.initialized = true;
- }
-
- /**
- * Returns a log writer to be used for appending any new {@link TransactionEdit} objects.
- */
- protected abstract TransactionLogWriter createWriter() throws IOException;
-
- @Override
- public abstract String getName();
-
- @Override
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public void append(TransactionEdit edit) throws IOException {
- long startTime = System.nanoTime();
- synchronized (this) {
- ensureAvailable();
-
- Entry entry = new Entry(new LongWritable(logSequence.getAndIncrement()), edit);
-
- // add to pending edits
- append(entry);
- }
-
- // wait for sync to complete
- sync();
- long durationMillis = (System.nanoTime() - startTime) / 1000000L;
- if (durationMillis > SLOW_APPEND_THRESHOLD) {
- LOG.info("Slow append to log " + getName() + ", took " + durationMillis + " msec.");
- }
- }
-
- @Override
- public void append(List<TransactionEdit> edits) throws IOException {
- long startTime = System.nanoTime();
- synchronized (this) {
- ensureAvailable();
-
- for (TransactionEdit edit : edits) {
- Entry entry = new Entry(new LongWritable(logSequence.getAndIncrement()), edit);
-
- // add to pending edits
- append(entry);
- }
- }
-
- // wait for sync to complete
- sync();
- long durationMillis = (System.nanoTime() - startTime) / 1000000L;
- if (durationMillis > SLOW_APPEND_THRESHOLD) {
- LOG.info("Slow append to log " + getName() + ", took " + durationMillis + " msec.");
- }
- }
-
- private void ensureAvailable() throws IOException {
- if (closed) {
- throw new IOException("Log " + getName() + " is already closed, cannot append!");
- }
- if (!initialized) {
- init();
- }
- }
-
- /*
- * Appends new writes to the pendingWrites. It is better to keep it in
- * our own queue rather than writing it to the HDFS output stream because
- * HDFSOutputStream.writeChunk is not lightweight at all.
- */
- private void append(Entry e) throws IOException {
- pendingWrites.add(e);
- }
-
- // Returns all currently pending writes. New writes
- // will accumulate in a new list.
- private List<Entry> getPendingWrites() {
- synchronized (this) {
- List<Entry> save = this.pendingWrites;
- this.pendingWrites = new LinkedList<>();
- return save;
- }
- }
-
- private void sync() throws IOException {
- // writes out pending entries to the HLog
- TransactionLogWriter tmpWriter = null;
- long latestSeq = 0;
- int entryCount = 0;
- synchronized (this) {
- if (closed) {
- return;
- }
- // prevent writer being dereferenced
- tmpWriter = writer;
-
- List<Entry> currentPending = getPendingWrites();
- if (!currentPending.isEmpty()) {
- tmpWriter.commitMarker(currentPending.size());
- }
-
- // write out all accumulated entries to log.
- for (Entry e : currentPending) {
- tmpWriter.append(e);
- entryCount++;
- latestSeq = Math.max(latestSeq, e.getKey().get());
- }
- }
-
- long lastSynced = syncedUpTo.get();
- // someone else might have already synced our edits, avoid double syncing
- if (lastSynced < latestSeq) {
- tmpWriter.sync();
- metricsCollector.histogram("wal.sync.size", entryCount);
- syncedUpTo.compareAndSet(lastSynced, latestSeq);
- }
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (closed) {
- return;
- }
- // perform a final sync if any outstanding writes
- if (!pendingWrites.isEmpty()) {
- sync();
- }
- // NOTE: writer is lazy-inited, so it can be null
- if (writer != null) {
- this.writer.close();
- }
- this.closed = true;
- }
-
- public boolean isClosed() {
- return closed;
- }
-
- @Override
- public abstract TransactionLogReader getReader() throws IOException;
-
- /**
- * Represents an entry in the transaction log. Each entry consists of a key, generated from an incrementing sequence
- * number, and a value, the {@link TransactionEdit} being stored.
- */
- public static class Entry implements Writable {
- private LongWritable key;
- private TransactionEdit edit;
-
- // for Writable
- public Entry() {
- this.key = new LongWritable();
- this.edit = new TransactionEdit();
- }
-
- public Entry(LongWritable key, TransactionEdit edit) {
- this.key = key;
- this.edit = edit;
- }
-
- public LongWritable getKey() {
- return this.key;
- }
-
- public TransactionEdit getEdit() {
- return this.edit;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- this.key.write(out);
- this.edit.write(out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- this.key.readFields(in);
- this.edit.readFields(in);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionStateStorage.java b/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionStateStorage.java
deleted file mode 100644
index 682435e..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/AbstractTransactionStateStorage.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import com.google.common.util.concurrent.AbstractIdleService;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Common base class for all transaction storage implementations. This implement logic to prefix a snapshot
- * with a version when encoding, and to select the correct codec for decoding based on this version prefix.
- */
-public abstract class AbstractTransactionStateStorage extends AbstractIdleService implements TransactionStateStorage {
-
- protected final SnapshotCodecProvider codecProvider;
-
- protected AbstractTransactionStateStorage(SnapshotCodecProvider codecProvider) {
- this.codecProvider = codecProvider;
- }
-
- @Override
- public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException {
- codecProvider.encode(out, snapshot);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/CommitMarkerCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/CommitMarkerCodec.java b/tephra-core/src/main/java/co/cask/tephra/persist/CommitMarkerCodec.java
deleted file mode 100644
index c1796bd..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/CommitMarkerCodec.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Charsets;
-import com.google.common.primitives.Bytes;
-import com.google.common.primitives.Ints;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.SequenceFile;
-
-import java.io.Closeable;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/**
- * Class to read and write commit markers used in {@link HDFSTransactionLogReaderV2} and above.
- */
-public class CommitMarkerCodec implements Closeable {
- private static final byte[] KEY_BYTES = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED.getBytes(Charsets.UTF_8);
- private final DataOutputBuffer rawKey;
- private final DataOutputBuffer rawValue;
- private SequenceFile.ValueBytes valueBytes;
-
- public CommitMarkerCodec() {
- this.rawKey = new DataOutputBuffer();
- this.rawValue = new DataOutputBuffer();
- }
-
- @Override
- public void close() throws IOException {
- rawKey.close();
- rawValue.close();
- }
-
- // 1. Returns the count when the marker is written correctly
- // 2. If data is incorrect (for ex, incorrect key, mismatch in key/value/record length), we throw IOException
- // since this indicates corrupted log file
- // 3. If data is incomplete, then we throw EOFException which is handled gracefully by the calling method
- // since we can recover without any consequence
- public int readMarker(SequenceFile.Reader reader) throws IOException {
- if (valueBytes == null) {
- valueBytes = reader.createValueBytes();
- }
- rawKey.reset();
- rawValue.reset();
-
- // valueBytes need not be reset since nextRaw call does it (and it is a private method)
- int status = reader.nextRaw(rawKey, valueBytes);
-
- // if we reach EOF, return -1
- if (status == -1) {
- return -1;
- }
-
- // Check if the marker key is valid and return the count
- if (isMarkerValid()) {
- valueBytes.writeUncompressedBytes(rawValue);
- rawValue.flush();
- // rawValue.getData() may return a larger byte array but Ints.fromByteArray will only read the first four bytes
- return Ints.fromByteArray(rawValue.getData());
- }
-
- // EOF not reached and marker is not valid, then thrown an IOException since we can't make progress
- throw new IOException(String.format("Invalid key for num entries appended found %s, expected : %s",
- new String(rawKey.getData()), TxConstants.TransactionLog.NUM_ENTRIES_APPENDED));
- }
-
- private boolean isMarkerValid() {
- // rawKey should have the expected length and the matching bytes should start at index 0
- return rawKey.getLength() == KEY_BYTES.length && Bytes.indexOf(rawKey.getData(), KEY_BYTES) == 0;
- }
-
- public static void writeMarker(SequenceFile.Writer writer, int count) throws IOException {
- writer.appendRaw(KEY_BYTES, 0, KEY_BYTES.length, new CommitEntriesCount(count));
- }
-
- @VisibleForTesting
- static final class CommitEntriesCount implements SequenceFile.ValueBytes {
- private final int numEntries;
-
- public CommitEntriesCount(int numEntries) {
- this.numEntries = numEntries;
- }
-
- @Override
- public void writeUncompressedBytes(DataOutputStream outStream) throws IOException {
- outStream.write(Ints.toByteArray(numEntries));
- }
-
- @Override
- public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException {
- throw new IllegalArgumentException("Commit Entries count writing is not expected to be compressed.");
- }
-
- @Override
- public int getSize() {
- return Ints.BYTES;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLog.java b/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLog.java
deleted file mode 100644
index bed90c2..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLog.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.MetricsCollector;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- * Allows reading from and writing to a transaction write-ahead log stored in HDFS.
- */
-public class HDFSTransactionLog extends AbstractTransactionLog {
- private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionLog.class);
-
- private final FileSystem fs;
- private final Configuration hConf;
- private final Path logPath;
-
- /**
- * Creates a new HDFS-backed write-ahead log for storing transaction state.
- * @param fs Open FileSystem instance for opening log files in HDFS.
- * @param hConf HDFS cluster configuration.
- * @param logPath Path to the log file.
- */
- public HDFSTransactionLog(final FileSystem fs, final Configuration hConf,
- final Path logPath, long timestamp, MetricsCollector metricsCollector) {
- super(timestamp, metricsCollector);
- this.fs = fs;
- this.hConf = hConf;
- this.logPath = logPath;
- }
-
- @Override
- protected TransactionLogWriter createWriter() throws IOException {
- return new LogWriter(fs, hConf, logPath);
- }
-
- @Override
- public String getName() {
- return logPath.getName();
- }
-
- @Override
- public TransactionLogReader getReader() throws IOException {
- FileStatus status = fs.getFileStatus(logPath);
- long length = status.getLen();
-
- TransactionLogReader reader = null;
- // check if this file needs to be recovered due to failure
- // Check for possibly empty file. With appends, currently Hadoop reports a
- // zero length even if the file has been sync'd. Revisit if HDFS-376 or
- // HDFS-878 is committed.
- if (length <= 0) {
- LOG.warn("File " + logPath + " might be still open, length is 0");
- }
-
- try {
- HDFSUtil hdfsUtil = new HDFSUtil();
- hdfsUtil.recoverFileLease(fs, logPath, hConf);
- try {
- FileStatus newStatus = fs.getFileStatus(logPath);
- LOG.info("New file size for " + logPath + " is " + newStatus.getLen());
- SequenceFile.Reader fileReader = new SequenceFile.Reader(fs, logPath, hConf);
- reader = new HDFSTransactionLogReaderSupplier(fileReader).get();
- } catch (EOFException e) {
- if (length <= 0) {
- // TODO should we ignore an empty, not-last log file if skip.errors
- // is false? Either way, the caller should decide what to do. E.g.
- // ignore if this is the last log in sequence.
- // TODO is this scenario still possible if the log has been
- // recovered (i.e. closed)
- LOG.warn("Could not open " + logPath + " for reading. File is empty", e);
- return null;
- } else {
- // EOFException being ignored
- return null;
- }
- }
- } catch (IOException e) {
- throw e;
- }
- return reader;
- }
-
- @VisibleForTesting
- static final class LogWriter implements TransactionLogWriter {
- private final SequenceFile.Writer internalWriter;
- public LogWriter(FileSystem fs, Configuration hConf, Path logPath) throws IOException {
- // TODO: retry a few times to ride over transient failures?
- SequenceFile.Metadata metadata = new SequenceFile.Metadata();
- metadata.set(new Text(TxConstants.TransactionLog.VERSION_KEY),
- new Text(Byte.toString(TxConstants.TransactionLog.CURRENT_VERSION)));
-
- this.internalWriter = SequenceFile.createWriter(fs, hConf, logPath, LongWritable.class, TransactionEdit.class,
- SequenceFile.CompressionType.NONE, null, null, metadata);
- LOG.debug("Created a new TransactionLog writer for " + logPath);
- }
-
- @Override
- public void append(Entry entry) throws IOException {
- internalWriter.append(entry.getKey(), entry.getEdit());
- }
-
- @Override
- public void commitMarker(int count) throws IOException {
- CommitMarkerCodec.writeMarker(internalWriter, count);
- }
-
- @Override
- public void sync() throws IOException {
- internalWriter.syncFs();
- }
-
- @Override
- public void close() throws IOException {
- internalWriter.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderSupplier.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderSupplier.java b/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderSupplier.java
deleted file mode 100644
index c407945..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderSupplier.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import com.google.common.base.Supplier;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-
-/**
- * Provides the correct version of {@link TransactionLogReader}, based on the log's version metadata,
- * to read HDFS Transaction Logs.
- */
-public class HDFSTransactionLogReaderSupplier implements Supplier<TransactionLogReader> {
- private final SequenceFile.Reader reader;
- private final byte version;
- private TransactionLogReader logReader;
-
- public HDFSTransactionLogReaderSupplier(SequenceFile.Reader reader) {
- this.reader = reader;
- Text versionInfo = reader.getMetadata().get(new Text(TxConstants.TransactionLog.VERSION_KEY));
- this.version = versionInfo == null ? 1 : Byte.parseByte(versionInfo.toString());
- }
-
- @Override
- public TransactionLogReader get() {
- if (logReader != null) {
- return logReader;
- }
-
- switch (version) {
- case 2:
- logReader = new HDFSTransactionLogReaderV2(reader);
- return logReader;
- case 1:
- logReader = new HDFSTransactionLogReaderV1(reader);
- return logReader;
- default:
- throw new IllegalArgumentException(String.format("Invalid version %s found in the Transaction Log", version));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV1.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV1.java b/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV1.java
deleted file mode 100644
index cb2ce7c..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV1.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- * {@link TransactionLogReader} that can read v1 (default) version of Transaction logs. The logs are expected to
- * have a sequence of {@link TransactionEdit}s.
- */
-public class HDFSTransactionLogReaderV1 implements TransactionLogReader {
- private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionLogReaderV1.class);
- private final SequenceFile.Reader reader;
- private final LongWritable key;
- private boolean closed;
-
- public HDFSTransactionLogReaderV1(SequenceFile.Reader reader) {
- this.reader = reader;
- this.key = new LongWritable();
- }
-
- @Override
- public TransactionEdit next() throws IOException {
- return next(new TransactionEdit());
- }
-
- @Override
- public TransactionEdit next(TransactionEdit reuse) throws IOException {
- if (closed) {
- return null;
- }
-
- try {
- boolean successful = reader.next(key, reuse);
- return successful ? reuse : null;
- } catch (EOFException e) {
- LOG.warn("Hit an unexpected EOF while trying to read the Transaction Edit. Skipping the entry.", e);
- return null;
- }
- }
-
- @Override
- public void close() throws IOException {
- if (closed) {
- return;
- }
- reader.close();
- closed = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV2.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV2.java b/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV2.java
deleted file mode 100644
index 6981a3b..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionLogReaderV2.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Queue;
-
-/**
- * {@link TransactionLogReader} that can read v2 version of Transaction logs. The logs are expected to
- * have commit markers that indicates the size of the batch of {@link TransactionEdit}s (follows the marker),
- * that were synced together. If the expected number of {@link TransactionEdit}s are not present then that set of
- * {@link TransactionEdit}s are discarded.
- */
-public class HDFSTransactionLogReaderV2 implements TransactionLogReader {
- private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionLogReaderV2.class);
-
- private final SequenceFile.Reader reader;
- private final Queue<TransactionEdit> transactionEdits;
- private final CommitMarkerCodec commitMarkerCodec;
- private final LongWritable key;
-
- private boolean closed;
-
- public HDFSTransactionLogReaderV2(SequenceFile.Reader reader) {
- this.reader = reader;
- this.transactionEdits = new ArrayDeque<>();
- this.key = new LongWritable();
- this.commitMarkerCodec = new CommitMarkerCodec();
- }
-
- @Override
- public void close() throws IOException {
- if (closed) {
- return;
- }
- try {
- commitMarkerCodec.close();
- } finally {
- reader.close();
- closed = true;
- }
- }
-
- @Override
- public TransactionEdit next() throws IOException {
- return next(null);
- }
-
- @Override
- public TransactionEdit next(TransactionEdit reuse) throws IOException {
- if (closed) {
- return null;
- }
-
- if (!transactionEdits.isEmpty()) {
- return transactionEdits.remove();
- }
-
- // Fetch the 'marker' and read 'marker' number of edits
- populateTransactionEdits();
- return transactionEdits.poll();
- }
-
- private void populateTransactionEdits() throws IOException {
- // read the marker to determine numEntries to read.
- int numEntries = 0;
- try {
- // can throw EOFException if reading of incomplete commit marker, no other action required since we can safely
- // ignore this
- numEntries = commitMarkerCodec.readMarker(reader);
- } catch (EOFException e) {
- LOG.warn("Reached EOF in log while trying to read commit marker", e);
- }
-
- for (int i = 0; i < numEntries; i++) {
- TransactionEdit edit = new TransactionEdit();
- try {
- if (reader.next(key, edit)) {
- transactionEdits.add(edit);
- } else {
- throw new EOFException("Attempt to read TransactionEdit failed.");
- }
- } catch (EOFException e) {
- // we have reached EOF before reading back numEntries, we clear the partial list and return.
- LOG.warn("Reached EOF in log before reading {} entries. Ignoring all {} edits since the last marker",
- numEntries, transactionEdits.size(), e);
- transactionEdits.clear();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionStateStorage.java b/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionStateStorage.java
deleted file mode 100644
index bc7fb9f..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSTransactionStateStorage.java
+++ /dev/null
@@ -1,492 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.MetricsCollector;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.util.ConfigurationFactory;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.io.CountingInputStream;
-import com.google.common.primitives.Longs;
-import com.google.inject.Inject;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import javax.annotation.Nullable;
-
-/**
- * Handles persistence of transaction snapshot and logs to a directory in HDFS.
- *
- * The directory used for file storage is configured using the {@code data.tx.snapshot.dir} configuration property.
- * Both snapshot and transaction log files are suffixed with a timestamp to allow easy ordering. Snapshot files
- * are written with the filename "snapshot.<timestamp>". Transaction log files are written with the filename
- * "txlog.<timestamp>".
- */
-public class HDFSTransactionStateStorage extends AbstractTransactionStateStorage {
- private static final Logger LOG = LoggerFactory.getLogger(HDFSTransactionStateStorage.class);
-
- private static final String SNAPSHOT_FILE_PREFIX = "snapshot.";
- private static final String TMP_SNAPSHOT_FILE_PREFIX = ".in-progress.snapshot.";
- private static final String LOG_FILE_PREFIX = "txlog.";
-
- private static final PathFilter SNAPSHOT_FILE_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith(SNAPSHOT_FILE_PREFIX);
- }
- };
-
- // buffer size used for HDFS reads and writes
- private static final int BUFFER_SIZE = 16384;
-
- private final Configuration hConf;
- private final String configuredSnapshotDir;
- private final MetricsCollector metricsCollector;
- private FileSystem fs;
- private Path snapshotDir;
-
- @Inject
- public HDFSTransactionStateStorage(Configuration hConf, SnapshotCodecProvider codecProvider,
- MetricsCollector metricsCollector) {
- super(codecProvider);
- this.hConf = hConf;
- this.configuredSnapshotDir = hConf.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR);
- this.metricsCollector = metricsCollector;
- }
-
- @Override
- protected void startUp() throws Exception {
- Preconditions.checkState(configuredSnapshotDir != null,
- "Snapshot directory is not configured. Please set " + TxConstants.Manager.CFG_TX_SNAPSHOT_DIR +
- " in configuration.");
- String hdfsUser = hConf.get(TxConstants.Manager.CFG_TX_HDFS_USER);
- if (hdfsUser == null || UserGroupInformation.isSecurityEnabled()) {
- if (hdfsUser != null && LOG.isDebugEnabled()) {
- LOG.debug("Ignoring configuration {}={}, running on secure Hadoop",
- TxConstants.Manager.CFG_TX_HDFS_USER, hdfsUser);
- }
- // NOTE: we can start multiple times this storage. As hdfs uses per-jvm cache, we want to create new fs instead
- // of getting closed one
- fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf);
- } else {
- fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf, hdfsUser);
- }
- snapshotDir = new Path(configuredSnapshotDir);
- LOG.info("Using snapshot dir " + snapshotDir);
- }
-
- @Override
- protected void shutDown() throws Exception {
- fs.close();
- }
-
- @Override
- public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
- // create a temporary file, and save the snapshot
- Path snapshotTmpFile = new Path(snapshotDir, TMP_SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
- LOG.debug("Writing snapshot to temporary file {}", snapshotTmpFile);
-
- FSDataOutputStream out = fs.create(snapshotTmpFile, false, BUFFER_SIZE);
- // encode the snapshot and stream the serialized version to the file
- try {
- codecProvider.encode(out, snapshot);
- } finally {
- out.close();
- }
-
- // move the temporary file into place with the correct filename
- Path finalFile = new Path(snapshotDir, SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
- fs.rename(snapshotTmpFile, finalFile);
- LOG.debug("Completed snapshot to file {}", finalFile);
- }
-
- @Override
- public TransactionSnapshot getLatestSnapshot() throws IOException {
- InputStream in = getLatestSnapshotInputStream();
- if (in == null) {
- return null;
- }
- try {
- return readSnapshotInputStream(in);
- } finally {
- in.close();
- }
- }
-
- @Override
- public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
- InputStream in = getLatestSnapshotInputStream();
- if (in == null) {
- return null;
- }
- try {
- return readTransactionVisibilityStateFromInputStream(in);
- } finally {
- in.close();
- }
- }
-
- private InputStream getLatestSnapshotInputStream() throws IOException {
- TimestampedFilename[] snapshots = listSnapshotFiles();
- Arrays.sort(snapshots);
- if (snapshots.length > 0) {
- // last is the most recent
- return fs.open(snapshots[snapshots.length - 1].getPath(), BUFFER_SIZE);
- }
-
- LOG.info("No snapshot files found in {}", snapshotDir);
- return null;
- }
-
- private TransactionSnapshot readSnapshotInputStream(InputStream in) throws IOException {
- CountingInputStream countingIn = new CountingInputStream(in);
- TransactionSnapshot snapshot = codecProvider.decode(countingIn);
- LOG.info("Read encoded transaction snapshot of {} bytes", countingIn.getCount());
- return snapshot;
- }
-
- private TransactionVisibilityState readTransactionVisibilityStateFromInputStream(InputStream in) throws IOException {
- CountingInputStream countingIn = new CountingInputStream(in);
- TransactionVisibilityState state = codecProvider.decodeTransactionVisibilityState(countingIn);
- LOG.info("Read encoded transaction snapshot of {} bytes", countingIn.getCount());
- return state;
- }
-
- private TransactionSnapshot readSnapshotFile(Path filePath) throws IOException {
- FSDataInputStream in = fs.open(filePath, BUFFER_SIZE);
- try {
- return readSnapshotInputStream(in);
- } finally {
- in.close();
- }
- }
-
- private TimestampedFilename[] listSnapshotFiles() throws IOException {
- FileStatus[] snapshotFileStatuses = fs.listStatus(snapshotDir, SNAPSHOT_FILE_FILTER);
- TimestampedFilename[] snapshotFiles = new TimestampedFilename[snapshotFileStatuses.length];
- for (int i = 0; i < snapshotFileStatuses.length; i++) {
- snapshotFiles[i] = new TimestampedFilename(snapshotFileStatuses[i].getPath());
- }
- return snapshotFiles;
- }
-
- @Override
- public long deleteOldSnapshots(int numberToKeep) throws IOException {
- TimestampedFilename[] snapshots = listSnapshotFiles();
- if (snapshots.length == 0) {
- return -1;
- }
- Arrays.sort(snapshots, Collections.reverseOrder());
- if (snapshots.length <= numberToKeep) {
- // nothing to remove, oldest timestamp is the last snapshot
- return snapshots[snapshots.length - 1].getTimestamp();
- }
- int toRemoveCount = snapshots.length - numberToKeep;
- TimestampedFilename[] toRemove = new TimestampedFilename[toRemoveCount];
- System.arraycopy(snapshots, numberToKeep, toRemove, 0, toRemoveCount);
-
- for (TimestampedFilename f : toRemove) {
- LOG.debug("Removing old snapshot file {}", f.getPath());
- fs.delete(f.getPath(), false);
- }
- long oldestTimestamp = snapshots[numberToKeep - 1].getTimestamp();
- LOG.debug("Removed {} old snapshot files prior to {}", toRemoveCount, oldestTimestamp);
- return oldestTimestamp;
- }
-
- @Override
- public List<String> listSnapshots() throws IOException {
- FileStatus[] files = fs.listStatus(snapshotDir, SNAPSHOT_FILE_FILTER);
- return Lists.transform(Arrays.asList(files), new Function<FileStatus, String>() {
- @Nullable
- @Override
- public String apply(@Nullable FileStatus input) {
- return input.getPath().getName();
- }
- });
- }
-
- @Override
- public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
- FileStatus[] statuses = fs.listStatus(snapshotDir, new LogFileFilter(timestamp, Long.MAX_VALUE));
- TimestampedFilename[] timestampedFiles = new TimestampedFilename[statuses.length];
- for (int i = 0; i < statuses.length; i++) {
- timestampedFiles[i] = new TimestampedFilename(statuses[i].getPath());
- }
- return Lists.transform(Arrays.asList(timestampedFiles), new Function<TimestampedFilename, TransactionLog>() {
- @Nullable
- @Override
- public TransactionLog apply(@Nullable TimestampedFilename input) {
- return openLog(input.getPath(), input.getTimestamp());
- }
- });
- }
-
- @Override
- public TransactionLog createLog(long timestamp) throws IOException {
- Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timestamp);
- return openLog(newLog, timestamp);
- }
-
- private TransactionLog openLog(Path path, long timestamp) {
- return new HDFSTransactionLog(fs, hConf, path, timestamp, metricsCollector);
- }
-
- @Override
- public void deleteLogsOlderThan(long timestamp) throws IOException {
- FileStatus[] statuses = fs.listStatus(snapshotDir, new LogFileFilter(0, timestamp));
- int removedCnt = 0;
- for (FileStatus status : statuses) {
- LOG.debug("Removing old transaction log {}", status.getPath());
- if (fs.delete(status.getPath(), false)) {
- removedCnt++;
- } else {
- LOG.error("Failed to delete transaction log file {}", status.getPath());
- }
- }
- LOG.debug("Removed {} transaction logs older than {}", removedCnt, timestamp);
- }
-
- @Override
- public void setupStorage() throws IOException {
- if (!fs.exists(snapshotDir)) {
- LOG.info("Creating snapshot dir at {}", snapshotDir);
- fs.mkdirs(snapshotDir);
- } else {
- Preconditions.checkState(fs.isDirectory(snapshotDir),
- "Configured snapshot directory " + snapshotDir + " is not a directory!");
- }
- }
-
- @Override
- public List<String> listLogs() throws IOException {
- FileStatus[] files = fs.listStatus(snapshotDir, new LogFileFilter(0, Long.MAX_VALUE));
- return Lists.transform(Arrays.asList(files), new Function<FileStatus, String>() {
- @Nullable
- @Override
- public String apply(@Nullable FileStatus input) {
- return input.getPath().getName();
- }
- });
- }
-
- @Override
- public String getLocation() {
- return snapshotDir.toString();
- }
-
- private static class LogFileFilter implements PathFilter {
- // starting time of files to include (inclusive)
- private final long startTime;
- // ending time of files to include (exclusive)
- private final long endTime;
-
- public LogFileFilter(long startTime, long endTime) {
- this.startTime = startTime;
- this.endTime = endTime;
- }
-
- @Override
- public boolean accept(Path path) {
- if (path.getName().startsWith(LOG_FILE_PREFIX)) {
- String[] parts = path.getName().split("\\.");
- if (parts.length == 2) {
- try {
- long fileTime = Long.parseLong(parts[1]);
- return fileTime >= startTime && fileTime < endTime;
- } catch (NumberFormatException ignored) {
- LOG.warn("Filename {} did not match the expected pattern prefix.<timestamp>", path.getName());
- }
- }
- }
- return false;
- }
- }
-
- /**
- * Represents a filename composed of a prefix and a ".timestamp" suffix. This is useful for manipulating both
- * snapshot and transaction log filenames.
- */
- private static class TimestampedFilename implements Comparable<TimestampedFilename> {
- private Path path;
- private String prefix;
- private long timestamp;
-
- public TimestampedFilename(Path path) {
- this.path = path;
- String[] parts = path.getName().split("\\.");
- if (parts.length != 2) {
- throw new IllegalArgumentException("Filename " + path.getName() +
- " did not match the expected pattern prefix.timestamp");
- }
- prefix = parts[0];
- timestamp = Long.parseLong(parts[1]);
- }
-
- public Path getPath() {
- return path;
- }
-
- public String getPrefix() {
- return prefix;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public int compareTo(TimestampedFilename other) {
- int res = prefix.compareTo(other.getPrefix());
- if (res == 0) {
- res = Longs.compare(timestamp, other.getTimestamp());
- }
- return res;
- }
- }
-
- // TODO move this out as a separate command line tool
- private enum CLIMode { SNAPSHOT, TXLOG };
- /**
- * Reads a transaction state snapshot or transaction log from HDFS and prints the entries to stdout.
- *
- * Supports the following options:
- * -s read snapshot state (defaults to the latest)
- * -l read a transaction log
- * [filename] reads the given file
- * @param args
- */
- public static void main(String[] args) {
- List<String> filenames = Lists.newArrayList();
- CLIMode mode = null;
- for (String arg : args) {
- if ("-s".equals(arg)) {
- mode = CLIMode.SNAPSHOT;
- } else if ("-l".equals(arg)) {
- mode = CLIMode.TXLOG;
- } else if ("-h".equals(arg)) {
- printUsage(null);
- } else {
- filenames.add(arg);
- }
- }
-
- if (mode == null) {
- printUsage("ERROR: Either -s or -l is required to set mode.", 1);
- }
-
- Configuration config = new ConfigurationFactory().get();
-
- // Use the no-op metrics collector. We are being run as a command line tool, so there are no relevant metrics
- // to report
- HDFSTransactionStateStorage storage =
- new HDFSTransactionStateStorage(config, new SnapshotCodecProvider(config), new TxMetricsCollector());
- storage.startAndWait();
- try {
- switch (mode) {
- case SNAPSHOT:
- try {
- if (filenames.isEmpty()) {
- TransactionSnapshot snapshot = storage.getLatestSnapshot();
- printSnapshot(snapshot);
- }
- for (String file : filenames) {
- Path path = new Path(file);
- TransactionSnapshot snapshot = storage.readSnapshotFile(path);
- printSnapshot(snapshot);
- System.out.println();
- }
- } catch (IOException ioe) {
- System.err.println("Error reading snapshot files: " + ioe.getMessage());
- ioe.printStackTrace();
- System.exit(1);
- }
- break;
- case TXLOG:
- if (filenames.isEmpty()) {
- printUsage("ERROR: At least one transaction log filename is required!", 1);
- }
- for (String file : filenames) {
- TimestampedFilename timestampedFilename = new TimestampedFilename(new Path(file));
- TransactionLog log = storage.openLog(timestampedFilename.getPath(), timestampedFilename.getTimestamp());
- printLog(log);
- System.out.println();
- }
- break;
- }
- } finally {
- storage.stop();
- }
- }
-
- private static void printUsage(String message) {
- printUsage(message, 0);
- }
-
- private static void printUsage(String message, int exitCode) {
- if (message != null) {
- System.out.println(message);
- }
- System.out.println("Usage: java " + HDFSTransactionStateStorage.class.getName() + " (-s|-l) file1 [file2...]");
- System.out.println();
- System.out.println("\t-s\tRead files as transaction state snapshots (will default to latest if no file given)");
- System.out.println("\t-l\tRead files as transaction logs [filename is required]");
- System.out.println("\t-h\tPrint this message");
- System.exit(exitCode);
- }
-
- private static void printSnapshot(TransactionSnapshot snapshot) {
- Date snapshotDate = new Date(snapshot.getTimestamp());
- System.out.println("TransactionSnapshot at " + snapshotDate.toString());
- System.out.println("\t" + snapshot.toString());
- }
-
- private static void printLog(TransactionLog log) {
- try {
- System.out.println("TransactionLog " + log.getName());
- TransactionLogReader reader = log.getReader();
- TransactionEdit edit;
- long seq = 0;
- while ((edit = reader.next()) != null) {
- System.out.println(String.format(" %d: %s", seq++, edit.toString()));
- }
- } catch (IOException ioe) {
- System.err.println("ERROR reading log " + log.getName() + ": " + ioe.getMessage());
- ioe.printStackTrace();
- }
- }
-}
[47/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftHandler.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftHandler.java b/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftHandler.java
deleted file mode 100644
index ab298cc..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/TransactionServiceThriftHandler.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.InvalidTruncateTimeException;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionNotInProgressException;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.distributed.thrift.TBoolean;
-import co.cask.tephra.distributed.thrift.TInvalidTruncateTimeException;
-import co.cask.tephra.distributed.thrift.TTransaction;
-import co.cask.tephra.distributed.thrift.TTransactionCouldNotTakeSnapshotException;
-import co.cask.tephra.distributed.thrift.TTransactionNotInProgressException;
-import co.cask.tephra.distributed.thrift.TTransactionServer;
-import co.cask.tephra.rpc.RPCServiceHandler;
-import com.google.common.collect.Sets;
-import org.apache.thrift.TException;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-/**
- * The implementation of a thrift service for tx service.
- * All operations arrive over the wire as Thrift objects.
- * <p/>
- * Why all this conversion (wrap/unwrap), and not define all Operations
- * themselves as Thrift objects?
- * <ul><li>
- * All the non-thrift executors would have to use the Thrift objects
- * </li><li>
- * Thrift's object model is too restrictive: it has only limited inheritance
- * and no overloading
- * </li><li>
- * Thrift objects are bare-bone, all they have are getters, setters, and
- * basic object methods.
- * </li></ul>
- */
-public class TransactionServiceThriftHandler implements TTransactionServer.Iface, RPCServiceHandler {
-
- private final TransactionManager txManager;
-
- public TransactionServiceThriftHandler(TransactionManager txManager) {
- this.txManager = txManager;
- }
-
- @Override
- public TTransaction startLong() throws TException {
- return TransactionConverterUtils.wrap(txManager.startLong());
- }
-
- @Override
- public TTransaction startShort() throws TException {
- return TransactionConverterUtils.wrap(txManager.startShort());
- }
-
- @Override
- public TTransaction startShortTimeout(int timeout) throws TException {
- return TransactionConverterUtils.wrap(txManager.startShort(timeout));
- }
-
-
- @Override
- public TBoolean canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws TException {
-
- Set<byte[]> changeIds = Sets.newHashSet();
- for (ByteBuffer bb : changes) {
- byte[] changeId = new byte[bb.remaining()];
- bb.get(changeId);
- changeIds.add(changeId);
- }
- try {
- return new TBoolean(txManager.canCommit(TransactionConverterUtils.unwrap(tx), changeIds));
- } catch (TransactionNotInProgressException e) {
- throw new TTransactionNotInProgressException(e.getMessage());
- }
- }
-
- @Override
- public TBoolean commitTx(TTransaction tx) throws TException {
- try {
- return new TBoolean(txManager.commit(TransactionConverterUtils.unwrap(tx)));
- } catch (TransactionNotInProgressException e) {
- throw new TTransactionNotInProgressException(e.getMessage());
- }
- }
-
- @Override
- public void abortTx(TTransaction tx) throws TException {
- txManager.abort(TransactionConverterUtils.unwrap(tx));
- }
-
- @Override
- public boolean invalidateTx(long tx) throws TException {
- return txManager.invalidate(tx);
- }
-
- @Override
- public ByteBuffer getSnapshot() throws TException {
- try {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- try {
- boolean snapshotTaken = txManager.takeSnapshot(out);
- if (!snapshotTaken) {
- throw new TTransactionCouldNotTakeSnapshotException("Transaction manager could not get a snapshot.");
- }
- } finally {
- out.close();
- }
- // todo find a way to encode directly to the stream, without having the snapshot in memory twice
- return ByteBuffer.wrap(out.toByteArray());
- } catch (IOException e) {
- throw new TTransactionCouldNotTakeSnapshotException(e.getMessage());
- }
- }
-
- @Override
- public void resetState() throws TException {
- txManager.resetState();
- }
-
- @Override
- public String status() throws TException {
- return txManager.isRunning() ? TxConstants.STATUS_OK : TxConstants.STATUS_NOTOK;
- }
-
- @Override
- public TBoolean truncateInvalidTx(Set<Long> txns) throws TException {
- return new TBoolean(txManager.truncateInvalidTx(txns));
- }
-
- @Override
- public TBoolean truncateInvalidTxBefore(long time) throws TException {
- try {
- return new TBoolean(txManager.truncateInvalidTxBefore(time));
- } catch (InvalidTruncateTimeException e) {
- throw new TInvalidTruncateTimeException(e.getMessage());
- }
- }
-
- @Override
- public int invalidTxSize() throws TException {
- return txManager.getInvalidSize();
- }
-
- @Override
- public TTransaction checkpoint(TTransaction originalTx) throws TException {
- try {
- return TransactionConverterUtils.wrap(
- txManager.checkpoint(TransactionConverterUtils.unwrap(originalTx)));
- } catch (TransactionNotInProgressException e) {
- throw new TTransactionNotInProgressException(e.getMessage());
- }
- }
-
- /* RPCServiceHandler implementation */
-
- @Override
- public void init() throws Exception {
- txManager.startAndWait();
- }
-
- @Override
- public void destroy() throws Exception {
- txManager.stopAndWait();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/package-info.java b/tephra-core/src/main/java/co/cask/tephra/distributed/package-info.java
deleted file mode 100644
index 2cc7677..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains standalone server of the transaction system v2. It is now simple netty piece, but will change
- * we integrate new RPC
- */
-package co.cask.tephra.distributed;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TBoolean.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TBoolean.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TBoolean.java
deleted file mode 100644
index 48ee2f9..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TBoolean.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TBoolean implements org.apache.thrift.TBase<TBoolean, TBoolean._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBoolean");
-
- private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.BOOL, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TBooleanStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TBooleanTupleSchemeFactory());
- }
-
- public boolean value; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- VALUE((short)1, "value");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // VALUE
- return VALUE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __VALUE_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBoolean.class, metaDataMap);
- }
-
- public TBoolean() {
- }
-
- public TBoolean(
- boolean value)
- {
- this();
- this.value = value;
- setValueIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TBoolean(TBoolean other) {
- __isset_bitfield = other.__isset_bitfield;
- this.value = other.value;
- }
-
- public TBoolean deepCopy() {
- return new TBoolean(this);
- }
-
- @Override
- public void clear() {
- setValueIsSet(false);
- this.value = false;
- }
-
- public boolean isValue() {
- return this.value;
- }
-
- public TBoolean setValue(boolean value) {
- this.value = value;
- setValueIsSet(true);
- return this;
- }
-
- public void unsetValue() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
- }
-
- /** Returns true if field value is set (has been assigned a value) and false otherwise */
- public boolean isSetValue() {
- return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
- }
-
- public void setValueIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case VALUE:
- if (value == null) {
- unsetValue();
- } else {
- setValue((Boolean)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case VALUE:
- return Boolean.valueOf(isValue());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case VALUE:
- return isSetValue();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TBoolean)
- return this.equals((TBoolean)that);
- return false;
- }
-
- public boolean equals(TBoolean that) {
- if (that == null)
- return false;
-
- boolean this_present_value = true;
- boolean that_present_value = true;
- if (this_present_value || that_present_value) {
- if (!(this_present_value && that_present_value))
- return false;
- if (this.value != that.value)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(TBoolean other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- TBoolean typedOther = (TBoolean)other;
-
- lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetValue()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TBoolean(");
- boolean first = true;
-
- sb.append("value:");
- sb.append(this.value);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TBooleanStandardSchemeFactory implements SchemeFactory {
- public TBooleanStandardScheme getScheme() {
- return new TBooleanStandardScheme();
- }
- }
-
- private static class TBooleanStandardScheme extends StandardScheme<TBoolean> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TBoolean struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // VALUE
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.value = iprot.readBool();
- struct.setValueIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TBoolean struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(VALUE_FIELD_DESC);
- oprot.writeBool(struct.value);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TBooleanTupleSchemeFactory implements SchemeFactory {
- public TBooleanTupleScheme getScheme() {
- return new TBooleanTupleScheme();
- }
- }
-
- private static class TBooleanTupleScheme extends TupleScheme<TBoolean> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TBoolean struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetValue()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetValue()) {
- oprot.writeBool(struct.value);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TBoolean struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.value = iprot.readBool();
- struct.setValueIsSet(true);
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TInvalidTruncateTimeException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TInvalidTruncateTimeException.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TInvalidTruncateTimeException.java
deleted file mode 100644
index 9a321e6..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TInvalidTruncateTimeException.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-import org.apache.thrift.scheme.TupleScheme;
-
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TInvalidTruncateTimeException extends TException implements org.apache.thrift.TBase<TInvalidTruncateTimeException, TInvalidTruncateTimeException._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TInvalidTruncateTimeException");
-
- private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TInvalidTruncateTimeExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TInvalidTruncateTimeExceptionTupleSchemeFactory());
- }
-
- public String message; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- MESSAGE((short)1, "message");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // MESSAGE
- return MESSAGE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TInvalidTruncateTimeException.class, metaDataMap);
- }
-
- public TInvalidTruncateTimeException() {
- }
-
- public TInvalidTruncateTimeException(
- String message)
- {
- this();
- this.message = message;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TInvalidTruncateTimeException(TInvalidTruncateTimeException other) {
- if (other.isSetMessage()) {
- this.message = other.message;
- }
- }
-
- public TInvalidTruncateTimeException deepCopy() {
- return new TInvalidTruncateTimeException(this);
- }
-
- @Override
- public void clear() {
- this.message = null;
- }
-
- public String getMessage() {
- return this.message;
- }
-
- public TInvalidTruncateTimeException setMessage(String message) {
- this.message = message;
- return this;
- }
-
- public void unsetMessage() {
- this.message = null;
- }
-
- /** Returns true if field message is set (has been assigned a value) and false otherwise */
- public boolean isSetMessage() {
- return this.message != null;
- }
-
- public void setMessageIsSet(boolean value) {
- if (!value) {
- this.message = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case MESSAGE:
- if (value == null) {
- unsetMessage();
- } else {
- setMessage((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case MESSAGE:
- return getMessage();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case MESSAGE:
- return isSetMessage();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TInvalidTruncateTimeException)
- return this.equals((TInvalidTruncateTimeException)that);
- return false;
- }
-
- public boolean equals(TInvalidTruncateTimeException that) {
- if (that == null)
- return false;
-
- boolean this_present_message = true && this.isSetMessage();
- boolean that_present_message = true && that.isSetMessage();
- if (this_present_message || that_present_message) {
- if (!(this_present_message && that_present_message))
- return false;
- if (!this.message.equals(that.message))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(TInvalidTruncateTimeException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- TInvalidTruncateTimeException typedOther = (TInvalidTruncateTimeException)other;
-
- lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMessage()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TInvalidTruncateTimeException(");
- boolean first = true;
-
- sb.append("message:");
- if (this.message == null) {
- sb.append("null");
- } else {
- sb.append(this.message);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TInvalidTruncateTimeExceptionStandardSchemeFactory implements SchemeFactory {
- public TInvalidTruncateTimeExceptionStandardScheme getScheme() {
- return new TInvalidTruncateTimeExceptionStandardScheme();
- }
- }
-
- private static class TInvalidTruncateTimeExceptionStandardScheme extends StandardScheme<TInvalidTruncateTimeException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // MESSAGE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.message = iprot.readString();
- struct.setMessageIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.message != null) {
- oprot.writeFieldBegin(MESSAGE_FIELD_DESC);
- oprot.writeString(struct.message);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TInvalidTruncateTimeExceptionTupleSchemeFactory implements SchemeFactory {
- public TInvalidTruncateTimeExceptionTupleScheme getScheme() {
- return new TInvalidTruncateTimeExceptionTupleScheme();
- }
- }
-
- private static class TInvalidTruncateTimeExceptionTupleScheme extends TupleScheme<TInvalidTruncateTimeException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetMessage()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetMessage()) {
- oprot.writeString(struct.message);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.message = iprot.readString();
- struct.setMessageIsSet(true);
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransaction.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransaction.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransaction.java
deleted file mode 100644
index 39c5af6..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransaction.java
+++ /dev/null
@@ -1,1364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-import org.apache.thrift.scheme.TupleScheme;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class TTransaction implements org.apache.thrift.TBase<TTransaction, TTransaction._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTransaction");
-
- private static final org.apache.thrift.protocol.TField TRANSACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("transactionId", org.apache.thrift.protocol.TType.I64, (short)1);
- private static final org.apache.thrift.protocol.TField READ_POINTER_FIELD_DESC = new org.apache.thrift.protocol.TField("readPointer", org.apache.thrift.protocol.TType.I64, (short)2);
- private static final org.apache.thrift.protocol.TField INVALIDS_FIELD_DESC = new org.apache.thrift.protocol.TField("invalids", org.apache.thrift.protocol.TType.LIST, (short)3);
- private static final org.apache.thrift.protocol.TField IN_PROGRESS_FIELD_DESC = new org.apache.thrift.protocol.TField("inProgress", org.apache.thrift.protocol.TType.LIST, (short)4);
- private static final org.apache.thrift.protocol.TField FIRST_SHORT_FIELD_DESC = new org.apache.thrift.protocol.TField("firstShort", org.apache.thrift.protocol.TType.I64, (short)5);
- private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)6);
- private static final org.apache.thrift.protocol.TField WRITE_POINTER_FIELD_DESC = new org.apache.thrift.protocol.TField("writePointer", org.apache.thrift.protocol.TType.I64, (short)7);
- private static final org.apache.thrift.protocol.TField CHECKPOINT_WRITE_POINTERS_FIELD_DESC = new org.apache.thrift.protocol.TField("checkpointWritePointers", org.apache.thrift.protocol.TType.LIST, (short)8);
- private static final org.apache.thrift.protocol.TField VISIBILITY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("visibilityLevel", org.apache.thrift.protocol.TType.I32, (short)9);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TTransactionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TTransactionTupleSchemeFactory());
- }
-
- public long transactionId; // required
- public long readPointer; // required
- public List<Long> invalids; // required
- public List<Long> inProgress; // required
- public long firstShort; // required
- /**
- *
- * @see TTransactionType
- */
- public TTransactionType type; // required
- public long writePointer; // required
- public List<Long> checkpointWritePointers; // required
- /**
- *
- * @see TVisibilityLevel
- */
- public TVisibilityLevel visibilityLevel; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- TRANSACTION_ID((short)1, "transactionId"),
- READ_POINTER((short)2, "readPointer"),
- INVALIDS((short)3, "invalids"),
- IN_PROGRESS((short)4, "inProgress"),
- FIRST_SHORT((short)5, "firstShort"),
- /**
- *
- * @see TTransactionType
- */
- TYPE((short)6, "type"),
- WRITE_POINTER((short)7, "writePointer"),
- CHECKPOINT_WRITE_POINTERS((short)8, "checkpointWritePointers"),
- /**
- *
- * @see TVisibilityLevel
- */
- VISIBILITY_LEVEL((short)9, "visibilityLevel");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // TRANSACTION_ID
- return TRANSACTION_ID;
- case 2: // READ_POINTER
- return READ_POINTER;
- case 3: // INVALIDS
- return INVALIDS;
- case 4: // IN_PROGRESS
- return IN_PROGRESS;
- case 5: // FIRST_SHORT
- return FIRST_SHORT;
- case 6: // TYPE
- return TYPE;
- case 7: // WRITE_POINTER
- return WRITE_POINTER;
- case 8: // CHECKPOINT_WRITE_POINTERS
- return CHECKPOINT_WRITE_POINTERS;
- case 9: // VISIBILITY_LEVEL
- return VISIBILITY_LEVEL;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __TRANSACTIONID_ISSET_ID = 0;
- private static final int __READPOINTER_ISSET_ID = 1;
- private static final int __FIRSTSHORT_ISSET_ID = 2;
- private static final int __WRITEPOINTER_ISSET_ID = 3;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.TRANSACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("transactionId", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.READ_POINTER, new org.apache.thrift.meta_data.FieldMetaData("readPointer", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.INVALIDS, new org.apache.thrift.meta_data.FieldMetaData("invalids", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
- tmpMap.put(_Fields.IN_PROGRESS, new org.apache.thrift.meta_data.FieldMetaData("inProgress", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
- tmpMap.put(_Fields.FIRST_SHORT, new org.apache.thrift.meta_data.FieldMetaData("firstShort", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TTransactionType.class)));
- tmpMap.put(_Fields.WRITE_POINTER, new org.apache.thrift.meta_data.FieldMetaData("writePointer", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.CHECKPOINT_WRITE_POINTERS, new org.apache.thrift.meta_data.FieldMetaData("checkpointWritePointers", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
- tmpMap.put(_Fields.VISIBILITY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("visibilityLevel", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TVisibilityLevel.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTransaction.class, metaDataMap);
- }
-
- public TTransaction() {
- }
-
- public TTransaction(
- long transactionId,
- long readPointer,
- List<Long> invalids,
- List<Long> inProgress,
- long firstShort,
- TTransactionType type,
- long writePointer,
- List<Long> checkpointWritePointers,
- TVisibilityLevel visibilityLevel)
- {
- this();
- this.transactionId = transactionId;
- setTransactionIdIsSet(true);
- this.readPointer = readPointer;
- setReadPointerIsSet(true);
- this.invalids = invalids;
- this.inProgress = inProgress;
- this.firstShort = firstShort;
- setFirstShortIsSet(true);
- this.type = type;
- this.writePointer = writePointer;
- setWritePointerIsSet(true);
- this.checkpointWritePointers = checkpointWritePointers;
- this.visibilityLevel = visibilityLevel;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TTransaction(TTransaction other) {
- __isset_bitfield = other.__isset_bitfield;
- this.transactionId = other.transactionId;
- this.readPointer = other.readPointer;
- if (other.isSetInvalids()) {
- List<Long> __this__invalids = new ArrayList<Long>();
- for (Long other_element : other.invalids) {
- __this__invalids.add(other_element);
- }
- this.invalids = __this__invalids;
- }
- if (other.isSetInProgress()) {
- List<Long> __this__inProgress = new ArrayList<Long>();
- for (Long other_element : other.inProgress) {
- __this__inProgress.add(other_element);
- }
- this.inProgress = __this__inProgress;
- }
- this.firstShort = other.firstShort;
- if (other.isSetType()) {
- this.type = other.type;
- }
- this.writePointer = other.writePointer;
- if (other.isSetCheckpointWritePointers()) {
- List<Long> __this__checkpointWritePointers = new ArrayList<Long>();
- for (Long other_element : other.checkpointWritePointers) {
- __this__checkpointWritePointers.add(other_element);
- }
- this.checkpointWritePointers = __this__checkpointWritePointers;
- }
- if (other.isSetVisibilityLevel()) {
- this.visibilityLevel = other.visibilityLevel;
- }
- }
-
- public TTransaction deepCopy() {
- return new TTransaction(this);
- }
-
- @Override
- public void clear() {
- setTransactionIdIsSet(false);
- this.transactionId = 0;
- setReadPointerIsSet(false);
- this.readPointer = 0;
- this.invalids = null;
- this.inProgress = null;
- setFirstShortIsSet(false);
- this.firstShort = 0;
- this.type = null;
- setWritePointerIsSet(false);
- this.writePointer = 0;
- this.checkpointWritePointers = null;
- this.visibilityLevel = null;
- }
-
- public long getTransactionId() {
- return this.transactionId;
- }
-
- public TTransaction setTransactionId(long transactionId) {
- this.transactionId = transactionId;
- setTransactionIdIsSet(true);
- return this;
- }
-
- public void unsetTransactionId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TRANSACTIONID_ISSET_ID);
- }
-
- /** Returns true if field transactionId is set (has been assigned a value) and false otherwise */
- public boolean isSetTransactionId() {
- return EncodingUtils.testBit(__isset_bitfield, __TRANSACTIONID_ISSET_ID);
- }
-
- public void setTransactionIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TRANSACTIONID_ISSET_ID, value);
- }
-
- public long getReadPointer() {
- return this.readPointer;
- }
-
- public TTransaction setReadPointer(long readPointer) {
- this.readPointer = readPointer;
- setReadPointerIsSet(true);
- return this;
- }
-
- public void unsetReadPointer() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __READPOINTER_ISSET_ID);
- }
-
- /** Returns true if field readPointer is set (has been assigned a value) and false otherwise */
- public boolean isSetReadPointer() {
- return EncodingUtils.testBit(__isset_bitfield, __READPOINTER_ISSET_ID);
- }
-
- public void setReadPointerIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __READPOINTER_ISSET_ID, value);
- }
-
- public int getInvalidsSize() {
- return (this.invalids == null) ? 0 : this.invalids.size();
- }
-
- public java.util.Iterator<Long> getInvalidsIterator() {
- return (this.invalids == null) ? null : this.invalids.iterator();
- }
-
- public void addToInvalids(long elem) {
- if (this.invalids == null) {
- this.invalids = new ArrayList<Long>();
- }
- this.invalids.add(elem);
- }
-
- public List<Long> getInvalids() {
- return this.invalids;
- }
-
- public TTransaction setInvalids(List<Long> invalids) {
- this.invalids = invalids;
- return this;
- }
-
- public void unsetInvalids() {
- this.invalids = null;
- }
-
- /** Returns true if field invalids is set (has been assigned a value) and false otherwise */
- public boolean isSetInvalids() {
- return this.invalids != null;
- }
-
- public void setInvalidsIsSet(boolean value) {
- if (!value) {
- this.invalids = null;
- }
- }
-
- public int getInProgressSize() {
- return (this.inProgress == null) ? 0 : this.inProgress.size();
- }
-
- public java.util.Iterator<Long> getInProgressIterator() {
- return (this.inProgress == null) ? null : this.inProgress.iterator();
- }
-
- public void addToInProgress(long elem) {
- if (this.inProgress == null) {
- this.inProgress = new ArrayList<Long>();
- }
- this.inProgress.add(elem);
- }
-
- public List<Long> getInProgress() {
- return this.inProgress;
- }
-
- public TTransaction setInProgress(List<Long> inProgress) {
- this.inProgress = inProgress;
- return this;
- }
-
- public void unsetInProgress() {
- this.inProgress = null;
- }
-
- /** Returns true if field inProgress is set (has been assigned a value) and false otherwise */
- public boolean isSetInProgress() {
- return this.inProgress != null;
- }
-
- public void setInProgressIsSet(boolean value) {
- if (!value) {
- this.inProgress = null;
- }
- }
-
- public long getFirstShort() {
- return this.firstShort;
- }
-
- public TTransaction setFirstShort(long firstShort) {
- this.firstShort = firstShort;
- setFirstShortIsSet(true);
- return this;
- }
-
- public void unsetFirstShort() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __FIRSTSHORT_ISSET_ID);
- }
-
- /** Returns true if field firstShort is set (has been assigned a value) and false otherwise */
- public boolean isSetFirstShort() {
- return EncodingUtils.testBit(__isset_bitfield, __FIRSTSHORT_ISSET_ID);
- }
-
- public void setFirstShortIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __FIRSTSHORT_ISSET_ID, value);
- }
-
- /**
- *
- * @see TTransactionType
- */
- public TTransactionType getType() {
- return this.type;
- }
-
- /**
- *
- * @see TTransactionType
- */
- public TTransaction setType(TTransactionType type) {
- this.type = type;
- return this;
- }
-
- public void unsetType() {
- this.type = null;
- }
-
- /** Returns true if field type is set (has been assigned a value) and false otherwise */
- public boolean isSetType() {
- return this.type != null;
- }
-
- public void setTypeIsSet(boolean value) {
- if (!value) {
- this.type = null;
- }
- }
-
- public long getWritePointer() {
- return this.writePointer;
- }
-
- public TTransaction setWritePointer(long writePointer) {
- this.writePointer = writePointer;
- setWritePointerIsSet(true);
- return this;
- }
-
- public void unsetWritePointer() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __WRITEPOINTER_ISSET_ID);
- }
-
- /** Returns true if field writePointer is set (has been assigned a value) and false otherwise */
- public boolean isSetWritePointer() {
- return EncodingUtils.testBit(__isset_bitfield, __WRITEPOINTER_ISSET_ID);
- }
-
- public void setWritePointerIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __WRITEPOINTER_ISSET_ID, value);
- }
-
- public int getCheckpointWritePointersSize() {
- return (this.checkpointWritePointers == null) ? 0 : this.checkpointWritePointers.size();
- }
-
- public java.util.Iterator<Long> getCheckpointWritePointersIterator() {
- return (this.checkpointWritePointers == null) ? null : this.checkpointWritePointers.iterator();
- }
-
- public void addToCheckpointWritePointers(long elem) {
- if (this.checkpointWritePointers == null) {
- this.checkpointWritePointers = new ArrayList<Long>();
- }
- this.checkpointWritePointers.add(elem);
- }
-
- public List<Long> getCheckpointWritePointers() {
- return this.checkpointWritePointers;
- }
-
- public TTransaction setCheckpointWritePointers(List<Long> checkpointWritePointers) {
- this.checkpointWritePointers = checkpointWritePointers;
- return this;
- }
-
- public void unsetCheckpointWritePointers() {
- this.checkpointWritePointers = null;
- }
-
- /** Returns true if field checkpointWritePointers is set (has been assigned a value) and false otherwise */
- public boolean isSetCheckpointWritePointers() {
- return this.checkpointWritePointers != null;
- }
-
- public void setCheckpointWritePointersIsSet(boolean value) {
- if (!value) {
- this.checkpointWritePointers = null;
- }
- }
-
- /**
- *
- * @see TVisibilityLevel
- */
- public TVisibilityLevel getVisibilityLevel() {
- return this.visibilityLevel;
- }
-
- /**
- *
- * @see TVisibilityLevel
- */
- public TTransaction setVisibilityLevel(TVisibilityLevel visibilityLevel) {
- this.visibilityLevel = visibilityLevel;
- return this;
- }
-
- public void unsetVisibilityLevel() {
- this.visibilityLevel = null;
- }
-
- /** Returns true if field visibilityLevel is set (has been assigned a value) and false otherwise */
- public boolean isSetVisibilityLevel() {
- return this.visibilityLevel != null;
- }
-
- public void setVisibilityLevelIsSet(boolean value) {
- if (!value) {
- this.visibilityLevel = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case TRANSACTION_ID:
- if (value == null) {
- unsetTransactionId();
- } else {
- setTransactionId((Long)value);
- }
- break;
-
- case READ_POINTER:
- if (value == null) {
- unsetReadPointer();
- } else {
- setReadPointer((Long)value);
- }
- break;
-
- case INVALIDS:
- if (value == null) {
- unsetInvalids();
- } else {
- setInvalids((List<Long>)value);
- }
- break;
-
- case IN_PROGRESS:
- if (value == null) {
- unsetInProgress();
- } else {
- setInProgress((List<Long>)value);
- }
- break;
-
- case FIRST_SHORT:
- if (value == null) {
- unsetFirstShort();
- } else {
- setFirstShort((Long)value);
- }
- break;
-
- case TYPE:
- if (value == null) {
- unsetType();
- } else {
- setType((TTransactionType)value);
- }
- break;
-
- case WRITE_POINTER:
- if (value == null) {
- unsetWritePointer();
- } else {
- setWritePointer((Long)value);
- }
- break;
-
- case CHECKPOINT_WRITE_POINTERS:
- if (value == null) {
- unsetCheckpointWritePointers();
- } else {
- setCheckpointWritePointers((List<Long>)value);
- }
- break;
-
- case VISIBILITY_LEVEL:
- if (value == null) {
- unsetVisibilityLevel();
- } else {
- setVisibilityLevel((TVisibilityLevel)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case TRANSACTION_ID:
- return Long.valueOf(getTransactionId());
-
- case READ_POINTER:
- return Long.valueOf(getReadPointer());
-
- case INVALIDS:
- return getInvalids();
-
- case IN_PROGRESS:
- return getInProgress();
-
- case FIRST_SHORT:
- return Long.valueOf(getFirstShort());
-
- case TYPE:
- return getType();
-
- case WRITE_POINTER:
- return Long.valueOf(getWritePointer());
-
- case CHECKPOINT_WRITE_POINTERS:
- return getCheckpointWritePointers();
-
- case VISIBILITY_LEVEL:
- return getVisibilityLevel();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case TRANSACTION_ID:
- return isSetTransactionId();
- case READ_POINTER:
- return isSetReadPointer();
- case INVALIDS:
- return isSetInvalids();
- case IN_PROGRESS:
- return isSetInProgress();
- case FIRST_SHORT:
- return isSetFirstShort();
- case TYPE:
- return isSetType();
- case WRITE_POINTER:
- return isSetWritePointer();
- case CHECKPOINT_WRITE_POINTERS:
- return isSetCheckpointWritePointers();
- case VISIBILITY_LEVEL:
- return isSetVisibilityLevel();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TTransaction)
- return this.equals((TTransaction)that);
- return false;
- }
-
- public boolean equals(TTransaction that) {
- if (that == null)
- return false;
-
- boolean this_present_transactionId = true;
- boolean that_present_transactionId = true;
- if (this_present_transactionId || that_present_transactionId) {
- if (!(this_present_transactionId && that_present_transactionId))
- return false;
- if (this.transactionId != that.transactionId)
- return false;
- }
-
- boolean this_present_readPointer = true;
- boolean that_present_readPointer = true;
- if (this_present_readPointer || that_present_readPointer) {
- if (!(this_present_readPointer && that_present_readPointer))
- return false;
- if (this.readPointer != that.readPointer)
- return false;
- }
-
- boolean this_present_invalids = true && this.isSetInvalids();
- boolean that_present_invalids = true && that.isSetInvalids();
- if (this_present_invalids || that_present_invalids) {
- if (!(this_present_invalids && that_present_invalids))
- return false;
- if (!this.invalids.equals(that.invalids))
- return false;
- }
-
- boolean this_present_inProgress = true && this.isSetInProgress();
- boolean that_present_inProgress = true && that.isSetInProgress();
- if (this_present_inProgress || that_present_inProgress) {
- if (!(this_present_inProgress && that_present_inProgress))
- return false;
- if (!this.inProgress.equals(that.inProgress))
- return false;
- }
-
- boolean this_present_firstShort = true;
- boolean that_present_firstShort = true;
- if (this_present_firstShort || that_present_firstShort) {
- if (!(this_present_firstShort && that_present_firstShort))
- return false;
- if (this.firstShort != that.firstShort)
- return false;
- }
-
- boolean this_present_type = true && this.isSetType();
- boolean that_present_type = true && that.isSetType();
- if (this_present_type || that_present_type) {
- if (!(this_present_type && that_present_type))
- return false;
- if (!this.type.equals(that.type))
- return false;
- }
-
- boolean this_present_writePointer = true;
- boolean that_present_writePointer = true;
- if (this_present_writePointer || that_present_writePointer) {
- if (!(this_present_writePointer && that_present_writePointer))
- return false;
- if (this.writePointer != that.writePointer)
- return false;
- }
-
- boolean this_present_checkpointWritePointers = true && this.isSetCheckpointWritePointers();
- boolean that_present_checkpointWritePointers = true && that.isSetCheckpointWritePointers();
- if (this_present_checkpointWritePointers || that_present_checkpointWritePointers) {
- if (!(this_present_checkpointWritePointers && that_present_checkpointWritePointers))
- return false;
- if (!this.checkpointWritePointers.equals(that.checkpointWritePointers))
- return false;
- }
-
- boolean this_present_visibilityLevel = true && this.isSetVisibilityLevel();
- boolean that_present_visibilityLevel = true && that.isSetVisibilityLevel();
- if (this_present_visibilityLevel || that_present_visibilityLevel) {
- if (!(this_present_visibilityLevel && that_present_visibilityLevel))
- return false;
- if (!this.visibilityLevel.equals(that.visibilityLevel))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(TTransaction other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- TTransaction typedOther = (TTransaction)other;
-
- lastComparison = Boolean.valueOf(isSetTransactionId()).compareTo(typedOther.isSetTransactionId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTransactionId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.transactionId, typedOther.transactionId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetReadPointer()).compareTo(typedOther.isSetReadPointer());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetReadPointer()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.readPointer, typedOther.readPointer);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetInvalids()).compareTo(typedOther.isSetInvalids());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetInvalids()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.invalids, typedOther.invalids);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetInProgress()).compareTo(typedOther.isSetInProgress());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetInProgress()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.inProgress, typedOther.inProgress);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetFirstShort()).compareTo(typedOther.isSetFirstShort());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetFirstShort()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.firstShort, typedOther.firstShort);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetType()).compareTo(typedOther.isSetType());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetType()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, typedOther.type);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetWritePointer()).compareTo(typedOther.isSetWritePointer());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetWritePointer()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.writePointer, typedOther.writePointer);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCheckpointWritePointers()).compareTo(typedOther.isSetCheckpointWritePointers());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCheckpointWritePointers()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.checkpointWritePointers, typedOther.checkpointWritePointers);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetVisibilityLevel()).compareTo(typedOther.isSetVisibilityLevel());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetVisibilityLevel()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.visibilityLevel, typedOther.visibilityLevel);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TTransaction(");
- boolean first = true;
-
- sb.append("transactionId:");
- sb.append(this.transactionId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("readPointer:");
- sb.append(this.readPointer);
- first = false;
- if (!first) sb.append(", ");
- sb.append("invalids:");
- if (this.invalids == null) {
- sb.append("null");
- } else {
- sb.append(this.invalids);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("inProgress:");
- if (this.inProgress == null) {
- sb.append("null");
- } else {
- sb.append(this.inProgress);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("firstShort:");
- sb.append(this.firstShort);
- first = false;
- if (!first) sb.append(", ");
- sb.append("type:");
- if (this.type == null) {
- sb.append("null");
- } else {
- sb.append(this.type);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("writePointer:");
- sb.append(this.writePointer);
- first = false;
- if (!first) sb.append(", ");
- sb.append("checkpointWritePointers:");
- if (this.checkpointWritePointers == null) {
- sb.append("null");
- } else {
- sb.append(this.checkpointWritePointers);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("visibilityLevel:");
- if (this.visibilityLevel == null) {
- sb.append("null");
- } else {
- sb.append(this.visibilityLevel);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TTransactionStandardSchemeFactory implements SchemeFactory {
- public TTransactionStandardScheme getScheme() {
- return new TTransactionStandardScheme();
- }
- }
-
- private static class TTransactionStandardScheme extends StandardScheme<TTransaction> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TTransaction struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // TRANSACTION_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.transactionId = iprot.readI64();
- struct.setTransactionIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // READ_POINTER
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.readPointer = iprot.readI64();
- struct.setReadPointerIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // INVALIDS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list0 = iprot.readListBegin();
- struct.invalids = new ArrayList<Long>(_list0.size);
- for (int _i1 = 0; _i1 < _list0.size; ++_i1)
- {
- long _elem2; // required
- _elem2 = iprot.readI64();
- struct.invalids.add(_elem2);
- }
- iprot.readListEnd();
- }
- struct.setInvalidsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // IN_PROGRESS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list3 = iprot.readListBegin();
- struct.inProgress = new ArrayList<Long>(_list3.size);
- for (int _i4 = 0; _i4 < _list3.size; ++_i4)
- {
- long _elem5; // required
- _elem5 = iprot.readI64();
- struct.inProgress.add(_elem5);
- }
- iprot.readListEnd();
- }
- struct.setInProgressIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // FIRST_SHORT
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.firstShort = iprot.readI64();
- struct.setFirstShortIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 6: // TYPE
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.type = TTransactionType.findByValue(iprot.readI32());
- struct.setTypeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 7: // WRITE_POINTER
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.writePointer = iprot.readI64();
- struct.setWritePointerIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 8: // CHECKPOINT_WRITE_POINTERS
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list6 = iprot.readListBegin();
- struct.checkpointWritePointers = new ArrayList<Long>(_list6.size);
- for (int _i7 = 0; _i7 < _list6.size; ++_i7)
- {
- long _elem8; // required
- _elem8 = iprot.readI64();
- struct.checkpointWritePointers.add(_elem8);
- }
- iprot.readListEnd();
- }
- struct.setCheckpointWritePointersIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 9: // VISIBILITY_LEVEL
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.visibilityLevel = TVisibilityLevel.findByValue(iprot.readI32());
- struct.setVisibilityLevelIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TTransaction struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(TRANSACTION_ID_FIELD_DESC);
- oprot.writeI64(struct.transactionId);
- oprot.writeFieldEnd();
- oprot.writeFieldBegin(READ_POINTER_FIELD_DESC);
- oprot.writeI64(struct.readPointer);
- oprot.writeFieldEnd();
- if (struct.invalids != null) {
- oprot.writeFieldBegin(INVALIDS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.invalids.size()));
- for (long _iter9 : struct.invalids)
- {
- oprot.writeI64(_iter9);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.inProgress != null) {
- oprot.writeFieldBegin(IN_PROGRESS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.inProgress.size()));
- for (long _iter10 : struct.inProgress)
- {
- oprot.writeI64(_iter10);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(FIRST_SHORT_FIELD_DESC);
- oprot.writeI64(struct.firstShort);
- oprot.writeFieldEnd();
- if (struct.type != null) {
- oprot.writeFieldBegin(TYPE_FIELD_DESC);
- oprot.writeI32(struct.type.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(WRITE_POINTER_FIELD_DESC);
- oprot.writeI64(struct.writePointer);
- oprot.writeFieldEnd();
- if (struct.checkpointWritePointers != null) {
- oprot.writeFieldBegin(CHECKPOINT_WRITE_POINTERS_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.checkpointWritePointers.size()));
- for (long _iter11 : struct.checkpointWritePointers)
- {
- oprot.writeI64(_iter11);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.visibilityLevel != null) {
- oprot.writeFieldBegin(VISIBILITY_LEVEL_FIELD_DESC);
- oprot.writeI32(struct.visibilityLevel.getValue());
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TTransactionTupleSchemeFactory implements SchemeFactory {
- public TTransactionTupleScheme getScheme() {
- return new TTransactionTupleScheme();
- }
- }
-
- private static class TTransactionTupleScheme extends TupleScheme<TTransaction> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TTransaction struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetTransactionId()) {
- optionals.set(0);
- }
- if (struct.isSetReadPointer()) {
- optionals.set(1);
- }
- if (struct.isSetInvalids()) {
- optionals.set(2);
- }
- if (struct.isSetInProgress()) {
- optionals.set(3);
- }
- if (struct.isSetFirstShort()) {
- optionals.set(4);
- }
- if (struct.isSetType()) {
- optionals.set(5);
- }
- if (struct.isSetWritePointer()) {
- optionals.set(6);
- }
- if (struct.isSetCheckpointWritePointers()) {
- optionals.set(7);
- }
- if (struct.isSetVisibilityLevel()) {
- optionals.set(8);
- }
- oprot.writeBitSet(optionals, 9);
- if (struct.isSetTransactionId()) {
- oprot.writeI64(struct.transactionId);
- }
- if (struct.isSetReadPointer()) {
- oprot.writeI64(struct.readPointer);
- }
- if (struct.isSetInvalids()) {
- {
- oprot.writeI32(struct.invalids.size());
- for (long _iter12 : struct.invalids)
- {
- oprot.writeI64(_iter12);
- }
- }
- }
- if (struct.isSetInProgress()) {
- {
- oprot.writeI32(struct.inProgress.size());
- for (long _iter13 : struct.inProgress)
- {
- oprot.writeI64(_iter13);
- }
- }
- }
- if (struct.isSetFirstShort()) {
- oprot.writeI64(struct.firstShort);
- }
- if (struct.isSetType()) {
- oprot.writeI32(struct.type.getValue());
- }
- if (struct.isSetWritePointer()) {
- oprot.writeI64(struct.writePointer);
- }
- if (struct.isSetCheckpointWritePointers()) {
- {
- oprot.writeI32(struct.checkpointWritePointers.size());
- for (long _iter14 : struct.checkpointWritePointers)
- {
- oprot.writeI64(_iter14);
- }
- }
- }
- if (struct.isSetVisibilityLevel()) {
- oprot.writeI32(struct.visibilityLevel.getValue());
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TTransaction struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(9);
- if (incoming.get(0)) {
- struct.transactionId = iprot.readI64();
- struct.setTransactionIdIsSet(true);
- }
- if (incoming.get(1)) {
- struct.readPointer = iprot.readI64();
- struct.setReadPointerIsSet(true);
- }
- if (incoming.get(2)) {
- {
- org.apache.thrift.protocol.TList _list15 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
- struct.invalids = new ArrayList<Long>(_list15.size);
- for (int _i16 = 0; _i16 < _list15.size; ++_i16)
- {
- long _elem17; // required
- _elem17 = iprot.readI64();
- struct.invalids.add(_elem17);
- }
- }
- struct.setInvalidsIsSet(true);
- }
- if (incoming.get(3)) {
- {
- org.apache.thrift.protocol.TList _list18 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
- struct.inProgress = new ArrayList<Long>(_list18.size);
- for (int _i19 = 0; _i19 < _list18.size; ++_i19)
- {
- long _elem20; // required
- _elem20 = iprot.readI64();
- struct.inProgress.add(_elem20);
- }
- }
- struct.setInProgressIsSet(true);
- }
- if (incoming.get(4)) {
- struct.firstShort = iprot.readI64();
- struct.setFirstShortIsSet(true);
- }
- if (incoming.get(5)) {
- struct.type = TTransactionType.findByValue(iprot.readI32());
- struct.setTypeIsSet(true);
- }
- if (incoming.get(6)) {
- struct.writePointer = iprot.readI64();
- struct.setWritePointerIsSet(true);
- }
- if (incoming.get(7)) {
- {
- org.apache.thrift.protocol.TList _list21 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
- struct.checkpointWritePointers = new ArrayList<Long>(_list21.size);
- for (int _i22 = 0; _i22 < _list21.size; ++_i22)
- {
- long _elem23; // required
- _elem23 = iprot.readI64();
- struct.checkpointWritePointers.add(_elem23);
- }
- }
- struct.setCheckpointWritePointersIsSet(true);
- }
- if (incoming.get(8)) {
- struct.visibilityLevel = TVisibilityLevel.findByValue(iprot.readI32());
- struct.setVisibilityLevelIsSet(true);
- }
- }
- }
-
-}
-
[05/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java
new file mode 100644
index 0000000..0d6ef17
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10.coprocessor;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCodec;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+
+/**
+ * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
+ * for transactions:
+ * <ul>
+ * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
+ * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
+ * or expired due to TTL.</li>
+ * </ul>
+ *
+ * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
+ * or on all user tables by adding the following to hbase-site.xml:
+ * {@code
+ * <property>
+ * <name>hbase.coprocessor.region.classes</name>
+ * <value>TransactionProcessor</value>
+ * </property>
+ * }
+ * </p>
+ *
+ * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
+ * as an attribute:
+ * {@code
+ * Transaction t = ...;
+ * Get get = new Get(...);
+ * TransactionCodec codec = new TransactionCodec();
+ * codec.addToOperation(get, t);
+ * }
+ * </p>
+ */
+public class TransactionProcessor extends BaseRegionObserver {
+ private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
+
+ private TransactionStateCache cache;
+ private final TransactionCodec txCodec;
+ protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
+ protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
+
+ public TransactionProcessor() {
+ this.txCodec = new TransactionCodec();
+ }
+
+ /* RegionObserver implementation */
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ if (e instanceof RegionCoprocessorEnvironment) {
+ RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
+ Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
+ this.cache = cacheSupplier.get();
+
+ HTableDescriptor tableDesc = env.getRegion().getTableDesc();
+ for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
+ String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
+ long ttl = 0;
+ if (columnTTL != null) {
+ try {
+ ttl = Long.parseLong(columnTTL);
+ LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
+ ", value = " + columnTTL);
+ }
+ }
+ ttlByFamily.put(columnDesc.getName(), ttl);
+ }
+
+ this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
+ TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
+ this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
+ if (readNonTxnData) {
+ LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
+ }
+ }
+ }
+
+ protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
+ return new TransactionStateCacheSupplier(env.getConfiguration());
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ // nothing to do
+ }
+
+ @Override
+ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
+ throws IOException {
+ Transaction tx = getFromOperation(get);
+ if (tx != null) {
+ projectFamilyDeletes(get);
+ get.setMaxVersions();
+ get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
+ get.setFilter(newFilter);
+ }
+ }
+
+ @Override
+ public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
+ Durability durability) throws IOException {
+ // Translate deletes into our own delete tombstones
+ // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
+ // us to rollback the changes (by a real delete) if the transaction fails
+
+ // Deletes that are part of a transaction rollback do not need special handling.
+ // They will never be rolled back, so are performed as normal HBase deletes.
+ if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ return;
+ }
+
+ // Other deletes are client-initiated and need to be translated into our own tombstones
+ // TODO: this should delegate to the DeleteStrategy implementation.
+ Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
+ for (byte[] family : delete.getFamilyCellMap().keySet()) {
+ List<Cell> familyCells = delete.getFamilyCellMap().get(family);
+ if (isFamilyDelete(familyCells)) {
+ deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ } else {
+ int cellSize = familyCells.size();
+ for (int i = 0; i < cellSize; i++) {
+ Cell cell = familyCells.get(i);
+ deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
+ }
+ e.getEnvironment().getRegion().put(deleteMarkers);
+ // skip normal delete handling
+ e.bypass();
+ }
+
+ private boolean isFamilyDelete(List<Cell> familyCells) {
+ return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
+ }
+
+ @Override
+ public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
+ throws IOException {
+ Transaction tx = getFromOperation(scan);
+ if (tx != null) {
+ projectFamilyDeletes(scan);
+ scan.setMaxVersions();
+ scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
+ scan.setFilter(newFilter);
+ }
+ return s;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any scan operation.
+ * @param scan The original scan request
+ * @return The modified scan request with the family delete qualifiers represented
+ */
+ private Scan projectFamilyDeletes(Scan scan) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return scan;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any get operation.
+ * @param get The original get request
+ * @return The modified get request with the family delete qualifiers represented
+ */
+ private Get projectFamilyDeletes(Get get) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return get;
+ }
+
+ @Override
+ public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ KeyValueScanner memstoreScanner, InternalScanner scanner)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
+ Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
+ HConstants.OLDEST_TIMESTAMP);
+ }
+
+ @Override
+ public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
+ CompactionRequest request)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners,
+ scanType, earliestPutTs);
+ }
+
+ protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
+ TransactionVisibilityState snapshot, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType type,
+ long earliestPutTs) throws IOException {
+ if (snapshot == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
+ ", no current transaction state found, defaulting to normal " + action + " scanner");
+ }
+ return null;
+ }
+
+ // construct a dummy transaction from the latest snapshot
+ Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
+ Scan scan = new Scan();
+ // need to see all versions, since we filter out excludes and applications may rely on multiple versions
+ scan.setMaxVersions();
+ scan.setFilter(
+ new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
+ snapshot.getInvalid(),
+ getTransactionFilter(dummyTx, type, null)));
+
+ return new StoreScanner(store, store.getScanInfo(), scan, scanners,
+ type, store.getSmallestReadPoint(), earliestPutTs);
+ }
+
+ private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
+ byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded != null) {
+ return txCodec.decode(encoded);
+ }
+ return null;
+ }
+
+ /**
+ * Derived classes can override this method to customize the filter used to return data visible for the current
+ * transaction.
+ *
+ * @param tx the current transaction to apply
+ * @param type the type of scan being performed
+ */
+ protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
+ return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+ }
+
+ /**
+ * Filter used to include cells visible to in-progress transactions on flush and commit.
+ */
+ static class IncludeInProgressFilter extends FilterBase {
+ private final long visibilityUpperBound;
+ private final Set<Long> invalidIds;
+ private final Filter txFilter;
+
+ public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
+ this.visibilityUpperBound = upperBound;
+ this.invalidIds = Sets.newHashSet(invalids);
+ this.txFilter = transactionFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ // include all cells visible to in-progress transactions, except for those already marked as invalid
+ long ts = cell.getTimestamp();
+ if (ts > visibilityUpperBound) {
+ // include everything that could still be in-progress except invalids
+ if (invalidIds.contains(ts)) {
+ return ReturnCode.SKIP;
+ }
+ return ReturnCode.INCLUDE;
+ }
+ return txFilter.filterKeyValue(cell);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java
new file mode 100644
index 0000000..d06cdd6
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10.coprocessor;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Applies filtering of data based on transactional visibility (HBase 1.0+ specific version).
+ * Note: this is intended for server-side use only, as additional properties need to be set on
+ * any {@code Scan} or {@code Get} operation performed.
+ */
+public class TransactionVisibilityFilter extends FilterBase {
+ private final Transaction tx;
+ // oldest visible timestamp by column family, used to apply TTL when reading
+ private final Map<ImmutableBytesWritable, Long> oldestTsByFamily;
+ // if false, empty values will be interpreted as deletes
+ private final boolean allowEmptyValues;
+ // whether or not we can remove delete markers
+ // these can only be safely removed when we are traversing all storefiles
+ private final boolean clearDeletes;
+ // optional sub-filter to apply to visible cells
+ private final Filter cellFilter;
+ // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
+ private final ImmutableBytesWritable currentFamily = new ImmutableBytesWritable(HConstants.EMPTY_BYTE_ARRAY);
+
+ private long currentOldestTs;
+
+ private DeleteTracker deleteTracker = new DeleteTracker();
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+ }
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ this.tx = tx;
+ this.oldestTsByFamily = Maps.newTreeMap();
+ for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
+ long familyTTL = ttlEntry.getValue();
+ oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
+ familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+ }
+ this.allowEmptyValues = allowEmptyValues;
+ this.clearDeletes =
+ scanType == ScanType.COMPACT_DROP_DELETES ||
+ (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ this.cellFilter = cellFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (!CellUtil.matchingFamily(cell, currentFamily.get(), currentFamily.getOffset(), currentFamily.getLength())) {
+ // column family changed
+ currentFamily.set(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
+ Long familyOldestTs = oldestTsByFamily.get(currentFamily);
+ currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
+ deleteTracker.reset();
+ }
+ // need to apply TTL for the column family here
+ long kvTimestamp = cell.getTimestamp();
+ if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
+ // passed TTL for this column, seek to next
+ return ReturnCode.NEXT_COL;
+ } else if (tx.isVisible(kvTimestamp)) {
+ // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
+ // cell is visible
+ // visibility SNAPSHOT_ALL needs all matches
+ return runSubFilter(ReturnCode.INCLUDE, cell);
+ }
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ deleteTracker.addFamilyDelete(cell);
+ if (clearDeletes) {
+ return ReturnCode.NEXT_COL;
+ } else {
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // check if masked by family delete
+ if (deleteTracker.isDeleted(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+ // check for column delete
+ if (isColumnDelete(cell)) {
+ if (clearDeletes) {
+ // skip "deleted" cell
+ return ReturnCode.NEXT_COL;
+ } else {
+ // keep the marker but skip any remaining versions
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ } else {
+ return ReturnCode.SKIP;
+ }
+ }
+
+ private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
+ if (cellFilter != null) {
+ ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
+ return determineReturnCode(txFilterCode, subFilterCode);
+ }
+ return txFilterCode;
+ }
+
+ /**
+ * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
+ * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
+ * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
+ * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
+ *
+ * @param txFilterCode return code from TransactionVisibilityFilter
+ * @param subFilterCode return code from sub-filter
+ * @return final return code
+ */
+ protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
+ // Return the more restrictive of the two filter responses
+ switch (subFilterCode) {
+ case INCLUDE:
+ return txFilterCode;
+ case INCLUDE_AND_NEXT_COL:
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ case SKIP:
+ return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
+ default:
+ return subFilterCode;
+ }
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRow();
+ }
+ return super.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ // Convert Tephra deletes back into HBase deletes
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
+ KeyValue.Type.DeleteFamily);
+ } else if (isColumnDelete(cell)) {
+ // Note: in some cases KeyValue.Type.Delete is used in Delete object,
+ // and in some other cases KeyValue.Type.DeleteColumn is used.
+ // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
+ // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
+ // work in both cases.
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
+ cell.getTimestamp(), KeyValue.Type.DeleteColumn);
+ }
+ }
+ return cell;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ deleteTracker.reset();
+ if (cellFilter != null) {
+ cellFilter.reset();
+ }
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRowKey(buffer, offset, length);
+ }
+ return super.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterAllRemaining();
+ }
+ return super.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ if (cellFilter != null) {
+ cellFilter.filterRowCells(kvs);
+ } else {
+ super.filterRowCells(kvs);
+ }
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ if (cellFilter != null) {
+ return cellFilter.hasFilterRow();
+ }
+ return super.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextKeyHint(currentKV);
+ }
+ return super.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextCellHint(currentKV);
+ }
+ return super.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.isFamilyEssential(name);
+ }
+ return super.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return super.toByteArray();
+ }
+
+ private boolean isColumnDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
+ }
+
+ private static final class DeleteTracker {
+ private long familyDeleteTs;
+
+ public static boolean isFamilyDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
+ CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
+ CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
+ }
+
+ public void addFamilyDelete(Cell delete) {
+ this.familyDeleteTs = delete.getTimestamp();
+ }
+
+ public boolean isDeleted(Cell cell) {
+ return cell.getTimestamp() <= familyDeleteTs;
+ }
+
+ public void reset() {
+ this.familyDeleteTs = 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/HBase10ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/HBase10ConfigurationProviderTest.java b/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/HBase10ConfigurationProviderTest.java
deleted file mode 100644
index 108023b..0000000
--- a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/HBase10ConfigurationProviderTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10;
-
-import co.cask.tephra.util.AbstractConfigurationProviderTest;
-import co.cask.tephra.util.HBaseVersion;
-
-/**
- * Test for HBase 1.0 version specific behavior.
- */
-public class HBase10ConfigurationProviderTest extends AbstractConfigurationProviderTest {
- @Override
- protected HBaseVersion.Version getExpectedVersion() {
- return HBaseVersion.Version.HBASE_10;
- }
-}
[27/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/TransactionExecutorTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/TransactionExecutorTest.java b/tephra-core/src/test/java/org/apache/tephra/TransactionExecutorTest.java
new file mode 100644
index 0000000..28ccc6e
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/TransactionExecutorTest.java
@@ -0,0 +1,590 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import com.google.inject.util.Modules;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.snapshot.DefaultSnapshotCodec;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * Tests the transaction executor.
+ */
+public class TransactionExecutorTest {
+ private static DummyTxClient txClient;
+ private static TransactionExecutorFactory factory;
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ final Configuration conf = new Configuration();
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new DiscoveryModules().getInMemoryModules(),
+ Modules.override(
+ new TransactionModules().getInMemoryModules()).with(new AbstractModule() {
+ @Override
+ protected void configure() {
+ TransactionManager txManager = new TransactionManager(conf);
+ txManager.startAndWait();
+ bind(TransactionManager.class).toInstance(txManager);
+ bind(TransactionSystemClient.class).to(DummyTxClient.class).in(Singleton.class);
+ }
+ }));
+
+ txClient = (DummyTxClient) injector.getInstance(TransactionSystemClient.class);
+ factory = injector.getInstance(TransactionExecutorFactory.class);
+ }
+
+ final DummyTxAware ds1 = new DummyTxAware(), ds2 = new DummyTxAware();
+ final Collection<TransactionAware> txAwares = ImmutableList.<TransactionAware>of(ds1, ds2);
+
+ private TransactionExecutor getExecutor() {
+ return factory.createExecutor(txAwares);
+ }
+
+ private TransactionExecutor getExecutorWithNoRetry() {
+ return new DefaultTransactionExecutor(txClient, txAwares, RetryStrategies.noRetries());
+ }
+
+ static final byte[] A = { 'a' };
+ static final byte[] B = { 'b' };
+
+ final TransactionExecutor.Function<Integer, Integer> testFunction =
+ new TransactionExecutor.Function<Integer, Integer>() {
+ @Override
+ public Integer apply(@Nullable Integer input) {
+ ds1.addChange(A);
+ ds2.addChange(B);
+ if (input == null) {
+ throw new RuntimeException("function failed");
+ }
+ return input * input;
+ }
+ };
+
+ @Before
+ public void resetTxAwares() {
+ ds1.reset();
+ ds2.reset();
+ }
+
+ @Test
+ public void testSuccessful() throws TransactionFailureException, InterruptedException {
+ // execute: add a change to ds1 and ds2
+ Integer result = getExecutor().execute(testFunction, 10);
+ // verify both are committed and post-committed
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertTrue(ds1.postCommitted);
+ Assert.assertTrue(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertTrue(100 == result);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
+ }
+
+ @Test
+ public void testPostCommitFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failPostCommitTxOnce = InduceFailure.ThrowException;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("post commit failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("post failure", e.getCause().getMessage());
+ }
+ // verify both are committed and post-committed
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertTrue(ds1.postCommitted);
+ Assert.assertTrue(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
+ }
+
+ @Test
+ public void testPersistFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ThrowException;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("persist failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("persist failure", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testPersistFalse() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("persist failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testPersistAndRollbackFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ThrowException;
+ ds1.failRollbackTxOnce = InduceFailure.ThrowException;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("persist failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("persist failure", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is invalidated
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
+ }
+
+ @Test
+ public void testPersistAndRollbackFalse() throws TransactionFailureException, InterruptedException {
+ ds1.failCommitTxOnce = InduceFailure.ReturnFalse;
+ ds1.failRollbackTxOnce = InduceFailure.ReturnFalse;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("persist failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertNull(e.getCause()); // in this case, the ds simply returned false
+ }
+ // verify both are rolled back and tx is invalidated
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
+ }
+
+ @Test
+ public void testNoIndefiniteRetryByDefault() throws TransactionFailureException, InterruptedException {
+ // we want retry by default, so that engineers don't miss it
+ txClient.failCommits = 1000;
+ try {
+ // execute: add a change to ds1 and ds2
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("commit failed too many times to retry - exception should be thrown");
+ } catch (TransactionConflictException e) {
+ Assert.assertNull(e.getCause());
+ }
+
+ txClient.failCommits = 0;
+
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testRetryByDefault() throws TransactionFailureException, InterruptedException {
+ // we want retry by default, so that engineers don't miss it
+ txClient.failCommits = 2;
+ // execute: add a change to ds1 and ds2
+ getExecutor().execute(testFunction, 10);
+ // should not fail, but continue
+
+ // verify both are committed
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertTrue(ds1.postCommitted);
+ Assert.assertTrue(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Committed);
+ }
+
+ @Test
+ public void testCommitFalse() throws TransactionFailureException, InterruptedException {
+ txClient.failCommits = 1;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutorWithNoRetry().execute(testFunction, 10);
+ Assert.fail("commit failed - exception should be thrown");
+ } catch (TransactionConflictException e) {
+ Assert.assertNull(e.getCause());
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertTrue(ds1.committed);
+ Assert.assertTrue(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testCanCommitFalse() throws TransactionFailureException, InterruptedException {
+ txClient.failCanCommitOnce = true;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutorWithNoRetry().execute(testFunction, 10);
+ Assert.fail("commit failed - exception should be thrown");
+ } catch (TransactionConflictException e) {
+ Assert.assertNull(e.getCause());
+ }
+ // verify both are rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertTrue(ds2.checked);
+ Assert.assertFalse(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ @Test
+ public void testChangesAndRollbackFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failChangesTxOnce = InduceFailure.ThrowException;
+ ds1.failRollbackTxOnce = InduceFailure.ThrowException;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("get changes failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("changes failure", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is invalidated
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertTrue(ds1.checked);
+ Assert.assertFalse(ds2.checked);
+ Assert.assertFalse(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
+ }
+
+ @Test
+ public void testFunctionAndRollbackFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failRollbackTxOnce = InduceFailure.ReturnFalse;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, null);
+ Assert.fail("function failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("function failed", e.getCause().getMessage());
+ }
+ // verify both are rolled back and tx is invalidated
+ Assert.assertTrue(ds1.started);
+ Assert.assertTrue(ds2.started);
+ Assert.assertFalse(ds1.checked);
+ Assert.assertFalse(ds2.checked);
+ Assert.assertFalse(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertTrue(ds1.rolledBack);
+ Assert.assertTrue(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Invalidated);
+ }
+
+ @Test
+ public void testStartAndRollbackFailure() throws TransactionFailureException, InterruptedException {
+ ds1.failStartTxOnce = InduceFailure.ThrowException;
+ // execute: add a change to ds1 and ds2
+ try {
+ getExecutor().execute(testFunction, 10);
+ Assert.fail("start failed - exception should be thrown");
+ } catch (TransactionFailureException e) {
+ Assert.assertEquals("start failure", e.getCause().getMessage());
+ }
+ // verify both are not rolled back and tx is aborted
+ Assert.assertTrue(ds1.started);
+ Assert.assertFalse(ds2.started);
+ Assert.assertFalse(ds1.checked);
+ Assert.assertFalse(ds2.checked);
+ Assert.assertFalse(ds1.committed);
+ Assert.assertFalse(ds2.committed);
+ Assert.assertFalse(ds1.postCommitted);
+ Assert.assertFalse(ds2.postCommitted);
+ Assert.assertFalse(ds1.rolledBack);
+ Assert.assertFalse(ds2.rolledBack);
+ Assert.assertEquals(txClient.state, DummyTxClient.CommitState.Aborted);
+ }
+
+ enum InduceFailure { NoFailure, ReturnFalse, ThrowException }
+
+ static class DummyTxAware implements TransactionAware {
+
+ Transaction tx;
+ boolean started = false;
+ boolean committed = false;
+ boolean checked = false;
+ boolean rolledBack = false;
+ boolean postCommitted = false;
+ List<byte[]> changes = Lists.newArrayList();
+
+ InduceFailure failStartTxOnce = InduceFailure.NoFailure;
+ InduceFailure failChangesTxOnce = InduceFailure.NoFailure;
+ InduceFailure failCommitTxOnce = InduceFailure.NoFailure;
+ InduceFailure failPostCommitTxOnce = InduceFailure.NoFailure;
+ InduceFailure failRollbackTxOnce = InduceFailure.NoFailure;
+
+ void addChange(byte[] key) {
+ changes.add(key);
+ }
+
+ void reset() {
+ tx = null;
+ started = false;
+ checked = false;
+ committed = false;
+ rolledBack = false;
+ postCommitted = false;
+ changes.clear();
+ }
+
+ @Override
+ public void startTx(Transaction tx) {
+ reset();
+ started = true;
+ this.tx = tx;
+ if (failStartTxOnce == InduceFailure.ThrowException) {
+ failStartTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("start failure");
+ }
+ }
+
+ @Override
+ public void updateTx(Transaction tx) {
+ this.tx = tx;
+ }
+
+ @Override
+ public Collection<byte[]> getTxChanges() {
+ checked = true;
+ if (failChangesTxOnce == InduceFailure.ThrowException) {
+ failChangesTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("changes failure");
+ }
+ return ImmutableList.copyOf(changes);
+ }
+
+ @Override
+ public boolean commitTx() throws Exception {
+ committed = true;
+ if (failCommitTxOnce == InduceFailure.ThrowException) {
+ failCommitTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("persist failure");
+ }
+ if (failCommitTxOnce == InduceFailure.ReturnFalse) {
+ failCommitTxOnce = InduceFailure.NoFailure;
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postTxCommit() {
+ postCommitted = true;
+ if (failPostCommitTxOnce == InduceFailure.ThrowException) {
+ failPostCommitTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("post failure");
+ }
+ }
+
+ @Override
+ public boolean rollbackTx() throws Exception {
+ rolledBack = true;
+ if (failRollbackTxOnce == InduceFailure.ThrowException) {
+ failRollbackTxOnce = InduceFailure.NoFailure;
+ throw new RuntimeException("rollback failure");
+ }
+ if (failRollbackTxOnce == InduceFailure.ReturnFalse) {
+ failRollbackTxOnce = InduceFailure.NoFailure;
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String getTransactionAwareName() {
+ return "dummy";
+ }
+ }
+
+ static class DummyTxClient extends InMemoryTxSystemClient {
+
+ boolean failCanCommitOnce = false;
+ int failCommits = 0;
+ enum CommitState {
+ Started, Committed, Aborted, Invalidated
+ }
+ CommitState state = CommitState.Started;
+
+ @Inject
+ DummyTxClient(TransactionManager txmgr) {
+ super(txmgr);
+ }
+
+ @Override
+ public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
+ if (failCanCommitOnce) {
+ failCanCommitOnce = false;
+ return false;
+ } else {
+ return super.canCommit(tx, changeIds);
+ }
+ }
+
+ @Override
+ public boolean commit(Transaction tx) throws TransactionNotInProgressException {
+ if (failCommits-- > 0) {
+ return false;
+ } else {
+ state = CommitState.Committed;
+ return super.commit(tx);
+ }
+ }
+
+ @Override
+ public Transaction startLong() {
+ state = CommitState.Started;
+ return super.startLong();
+ }
+
+ @Override
+ public Transaction startShort() {
+ state = CommitState.Started;
+ return super.startShort();
+ }
+
+ @Override
+ public Transaction startShort(int timeout) {
+ state = CommitState.Started;
+ return super.startShort(timeout);
+ }
+
+ @Override
+ public void abort(Transaction tx) {
+ state = CommitState.Aborted;
+ super.abort(tx);
+ }
+
+ @Override
+ public boolean invalidate(long tx) {
+ state = CommitState.Invalidated;
+ return super.invalidate(tx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/TransactionManagerTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/TransactionManagerTest.java b/tephra-core/src/test/java/org/apache/tephra/TransactionManagerTest.java
new file mode 100644
index 0000000..f74e209
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/TransactionManagerTest.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class TransactionManagerTest extends TransactionSystemTest {
+
+ static Configuration conf = new Configuration();
+
+ TransactionManager txManager = null;
+ TransactionStateStorage txStateStorage = null;
+
+ @Override
+ protected TransactionSystemClient getClient() {
+ return new InMemoryTxSystemClient(txManager);
+ }
+
+ @Override
+ protected TransactionStateStorage getStateStorage() {
+ return txStateStorage;
+ }
+
+ @Before
+ public void before() {
+ conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
+ // todo should create two sets of tests, one with LocalFileTxStateStorage and one with InMemoryTxStateStorage
+ txStateStorage = new InMemoryTransactionStateStorage();
+ txManager = new TransactionManager
+ (conf, txStateStorage, new TxMetricsCollector());
+ txManager.startAndWait();
+ }
+
+ @After
+ public void after() {
+ txManager.stopAndWait();
+ }
+
+ @Test
+ public void testTransactionCleanup() throws Exception {
+ conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 3);
+ conf.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, 2);
+ // using a new tx manager that cleans up
+ TransactionManager txm = new TransactionManager
+ (conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
+ txm.startAndWait();
+ try {
+ Assert.assertEquals(0, txm.getInvalidSize());
+ Assert.assertEquals(0, txm.getCommittedSize());
+ // start a transaction and leave it open
+ Transaction tx1 = txm.startShort();
+ // start a long running transaction and leave it open
+ Transaction tx2 = txm.startLong();
+ Transaction tx3 = txm.startLong();
+ // start and commit a bunch of transactions
+ for (int i = 0; i < 10; i++) {
+ Transaction tx = txm.startShort();
+ Assert.assertTrue(txm.canCommit(tx, Collections.singleton(new byte[] { (byte) i })));
+ Assert.assertTrue(txm.commit(tx));
+ }
+ // all of these should still be in the committed set
+ Assert.assertEquals(0, txm.getInvalidSize());
+ Assert.assertEquals(10, txm.getCommittedSize());
+ // sleep longer than the cleanup interval
+ TimeUnit.SECONDS.sleep(5);
+ // transaction should now be invalid
+ Assert.assertEquals(1, txm.getInvalidSize());
+ // run another transaction
+ Transaction txx = txm.startShort();
+ // verify the exclude
+ Assert.assertFalse(txx.isVisible(tx1.getTransactionId()));
+ Assert.assertFalse(txx.isVisible(tx2.getTransactionId()));
+ Assert.assertFalse(txx.isVisible(tx3.getTransactionId()));
+ // try to commit the last transaction that was started
+ Assert.assertTrue(txm.canCommit(txx, Collections.singleton(new byte[] { 0x0a })));
+ Assert.assertTrue(txm.commit(txx));
+
+ // now the committed change sets should be empty again
+ Assert.assertEquals(0, txm.getCommittedSize());
+ // cannot commit transaction as it was timed out
+ try {
+ txm.canCommit(tx1, Collections.singleton(new byte[] { 0x11 }));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+ txm.abort(tx1);
+ // abort should have removed from invalid
+ Assert.assertEquals(0, txm.getInvalidSize());
+ // run another bunch of transactions
+ for (int i = 0; i < 10; i++) {
+ Transaction tx = txm.startShort();
+ Assert.assertTrue(txm.canCommit(tx, Collections.singleton(new byte[] { (byte) i })));
+ Assert.assertTrue(txm.commit(tx));
+ }
+ // none of these should still be in the committed set (tx2 is long-running).
+ Assert.assertEquals(0, txm.getInvalidSize());
+ Assert.assertEquals(0, txm.getCommittedSize());
+ // commit tx2, abort tx3
+ Assert.assertTrue(txm.commit(tx2));
+ txm.abort(tx3);
+ // none of these should still be in the committed set (tx2 is long-running).
+ // Only tx3 is invalid list as it was aborted and is long-running. tx1 is short one and it rolled back its changes
+ // so it should NOT be in invalid list
+ Assert.assertEquals(1, txm.getInvalidSize());
+ Assert.assertEquals(tx3.getTransactionId(), (long) txm.getCurrentState().getInvalid().iterator().next());
+ Assert.assertEquals(1, txm.getExcludedListSize());
+ } finally {
+ txm.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testLongTransactionCleanup() throws Exception {
+ conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 3);
+ conf.setInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT, 2);
+ // using a new tx manager that cleans up
+ TransactionManager txm = new TransactionManager
+ (conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector());
+ txm.startAndWait();
+ try {
+ Assert.assertEquals(0, txm.getInvalidSize());
+ Assert.assertEquals(0, txm.getCommittedSize());
+
+ // start a long running transaction
+ Transaction tx1 = txm.startLong();
+
+ Assert.assertEquals(0, txm.getInvalidSize());
+ Assert.assertEquals(0, txm.getCommittedSize());
+
+ // sleep longer than the cleanup interval
+ TimeUnit.SECONDS.sleep(5);
+
+ // transaction should now be invalid
+ Assert.assertEquals(1, txm.getInvalidSize());
+ Assert.assertEquals(0, txm.getCommittedSize());
+
+ // cannot commit transaction as it was timed out
+ try {
+ txm.canCommit(tx1, Collections.singleton(new byte[] { 0x11 }));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+
+ txm.abort(tx1);
+ // abort should not remove long running transaction from invalid list
+ Assert.assertEquals(1, txm.getInvalidSize());
+ } finally {
+ txm.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testTruncateInvalid() throws Exception {
+ InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage();
+ Configuration testConf = new Configuration(conf);
+ // No snapshots
+ testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1);
+ TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector());
+ txm1.startAndWait();
+
+ TransactionManager txm2 = null;
+ Transaction tx1;
+ Transaction tx2;
+ Transaction tx3;
+ Transaction tx4;
+ Transaction tx5;
+ Transaction tx6;
+ try {
+ Assert.assertEquals(0, txm1.getInvalidSize());
+
+ // start a few transactions
+ tx1 = txm1.startLong();
+ tx2 = txm1.startShort();
+ tx3 = txm1.startLong();
+ tx4 = txm1.startShort();
+ tx5 = txm1.startLong();
+ tx6 = txm1.startShort();
+
+ // invalidate tx1, tx2, tx5 and tx6
+ txm1.invalidate(tx1.getTransactionId());
+ txm1.invalidate(tx2.getTransactionId());
+ txm1.invalidate(tx5.getTransactionId());
+ txm1.invalidate(tx6.getTransactionId());
+
+ // tx1, tx2, tx5 and tx6 should be in invalid list
+ Assert.assertEquals(
+ ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(), tx5.getTransactionId(),
+ tx6.getTransactionId()),
+ txm1.getCurrentState().getInvalid()
+ );
+
+ // remove tx1 and tx6 from invalid list
+ Assert.assertTrue(txm1.truncateInvalidTx(ImmutableSet.of(tx1.getTransactionId(), tx6.getTransactionId())));
+
+ // only tx2 and tx5 should be in invalid list now
+ Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
+ txm1.getCurrentState().getInvalid());
+
+ // removing in-progress transactions should not have any effect
+ Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
+ txm1.getCurrentState().getInProgress().keySet());
+ Assert.assertFalse(txm1.truncateInvalidTx(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId())));
+ // no change to in-progress
+ Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
+ txm1.getCurrentState().getInProgress().keySet());
+ // no change to invalid list
+ Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
+ txm1.getCurrentState().getInvalid());
+
+ // Test transaction edit logs replay
+ // Start another transaction manager without stopping txm1 so that snapshot does not get written,
+ // and all logs can be replayed.
+ txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector());
+ txm2.startAndWait();
+ Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
+ txm2.getCurrentState().getInvalid());
+ Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
+ txm2.getCurrentState().getInProgress().keySet());
+ } finally {
+ txm1.stopAndWait();
+ if (txm2 != null) {
+ txm2.stopAndWait();
+ }
+ }
+ }
+
+ @Test
+ public void testTruncateInvalidBeforeTime() throws Exception {
+ InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage();
+ Configuration testConf = new Configuration(conf);
+ // No snapshots
+ testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1);
+ TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector());
+ txm1.startAndWait();
+
+ TransactionManager txm2 = null;
+ Transaction tx1;
+ Transaction tx2;
+ Transaction tx3;
+ Transaction tx4;
+ Transaction tx5;
+ Transaction tx6;
+ try {
+ Assert.assertEquals(0, txm1.getInvalidSize());
+
+ // start a few transactions
+ tx1 = txm1.startLong();
+ tx2 = txm1.startShort();
+ // Sleep so that transaction ids get generated a millisecond apart for assertion
+ // TEPHRA-63 should eliminate the need to sleep
+ TimeUnit.MILLISECONDS.sleep(1);
+ long timeBeforeTx3 = System.currentTimeMillis();
+ tx3 = txm1.startLong();
+ tx4 = txm1.startShort();
+ TimeUnit.MILLISECONDS.sleep(1);
+ long timeBeforeTx5 = System.currentTimeMillis();
+ tx5 = txm1.startLong();
+ tx6 = txm1.startShort();
+
+ // invalidate tx1, tx2, tx5 and tx6
+ txm1.invalidate(tx1.getTransactionId());
+ txm1.invalidate(tx2.getTransactionId());
+ txm1.invalidate(tx5.getTransactionId());
+ txm1.invalidate(tx6.getTransactionId());
+
+ // tx1, tx2, tx5 and tx6 should be in invalid list
+ Assert.assertEquals(
+ ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(), tx5.getTransactionId(),
+ tx6.getTransactionId()),
+ txm1.getCurrentState().getInvalid()
+ );
+
+ // remove transactions before tx3 from invalid list
+ Assert.assertTrue(txm1.truncateInvalidTxBefore(timeBeforeTx3));
+
+ // only tx5 and tx6 should be in invalid list now
+ Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
+ txm1.getCurrentState().getInvalid());
+
+ // removing invalid transactions before tx5 should throw exception since tx3 and tx4 are in-progress
+ Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
+ txm1.getCurrentState().getInProgress().keySet());
+ try {
+ txm1.truncateInvalidTxBefore(timeBeforeTx5);
+ Assert.fail("Expected InvalidTruncateTimeException exception");
+ } catch (InvalidTruncateTimeException e) {
+ // Expected exception
+ }
+ // no change to in-progress
+ Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
+ txm1.getCurrentState().getInProgress().keySet());
+ // no change to invalid list
+ Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
+ txm1.getCurrentState().getInvalid());
+
+ // Test transaction edit logs replay
+ // Start another transaction manager without stopping txm1 so that snapshot does not get written,
+ // and all logs can be replayed.
+ txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector());
+ txm2.startAndWait();
+ Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
+ txm2.getCurrentState().getInvalid());
+ Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
+ txm2.getCurrentState().getInProgress().keySet());
+ } finally {
+ txm1.stopAndWait();
+ if (txm2 != null) {
+ txm2.stopAndWait();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/TransactionServiceMainTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/TransactionServiceMainTest.java b/tephra-core/src/test/java/org/apache/tephra/TransactionServiceMainTest.java
new file mode 100644
index 0000000..fa12acb
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/TransactionServiceMainTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.distributed.TransactionServiceClient;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * Test for verifying TransactionServiceMain works correctly.
+ */
+public class TransactionServiceMainTest {
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Test
+ public void testClientServer() throws Exception {
+ // Simply start a transaction server and connect to it with the client.
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
+ zkServer.startAndWait();
+
+ try {
+ Configuration conf = new Configuration();
+ conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
+
+ final TransactionServiceMain main = new TransactionServiceMain(conf);
+ final CountDownLatch latch = new CountDownLatch(1);
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ main.start();
+ latch.countDown();
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ };
+
+ try {
+ t.start();
+ // Wait for service to startup
+ latch.await();
+ TransactionServiceClient.doMain(true, conf);
+ } finally {
+ main.stop();
+ t.join();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/TransactionSystemTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/TransactionSystemTest.java b/tephra-core/src/test/java/org/apache/tephra/TransactionSystemTest.java
new file mode 100644
index 0000000..797c08a
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/TransactionSystemTest.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public abstract class TransactionSystemTest {
+
+ public static final byte[] C1 = new byte[] { 'c', '1' };
+ public static final byte[] C2 = new byte[] { 'c', '2' };
+ public static final byte[] C3 = new byte[] { 'c', '3' };
+ public static final byte[] C4 = new byte[] { 'c', '4' };
+
+ protected abstract TransactionSystemClient getClient() throws Exception;
+
+ protected abstract TransactionStateStorage getStateStorage() throws Exception;
+
+ @Test
+ public void testCommitRaceHandling() throws Exception {
+ TransactionSystemClient client1 = getClient();
+ TransactionSystemClient client2 = getClient();
+
+ Transaction tx1 = client1.startShort();
+ Transaction tx2 = client2.startShort();
+
+ Assert.assertTrue(client1.canCommit(tx1, asList(C1, C2)));
+ // second one also can commit even thought there are conflicts with first since first one hasn't committed yet
+ Assert.assertTrue(client2.canCommit(tx2, asList(C2, C3)));
+
+ Assert.assertTrue(client1.commit(tx1));
+
+ // now second one should not commit, since there are conflicts with tx1 that has been committed
+ Assert.assertFalse(client2.commit(tx2));
+ }
+
+ @Test
+ public void testMultipleCommitsAtSameTime() throws Exception {
+ // We want to check that if two txs finish at same time (wrt tx manager) they do not overwrite changesets of each
+ // other in tx manager used for conflicts detection (we had this bug)
+ // NOTE: you don't have to use multiple clients for that
+ TransactionSystemClient client1 = getClient();
+ TransactionSystemClient client2 = getClient();
+ TransactionSystemClient client3 = getClient();
+ TransactionSystemClient client4 = getClient();
+ TransactionSystemClient client5 = getClient();
+
+ Transaction tx1 = client1.startShort();
+ Transaction tx2 = client2.startShort();
+ Transaction tx3 = client3.startShort();
+ Transaction tx4 = client4.startShort();
+ Transaction tx5 = client5.startShort();
+
+ Assert.assertTrue(client1.canCommit(tx1, asList(C1)));
+ Assert.assertTrue(client1.commit(tx1));
+
+ Assert.assertTrue(client2.canCommit(tx2, asList(C2)));
+ Assert.assertTrue(client2.commit(tx2));
+
+ // verifying conflicts detection
+ Assert.assertFalse(client3.canCommit(tx3, asList(C1)));
+ Assert.assertFalse(client4.canCommit(tx4, asList(C2)));
+ Assert.assertTrue(client5.canCommit(tx5, asList(C3)));
+ }
+
+ @Test
+ public void testCommitTwice() throws Exception {
+ TransactionSystemClient client = getClient();
+ Transaction tx = client.startShort();
+
+ Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
+ Assert.assertTrue(client.commit(tx));
+ // cannot commit twice same tx
+ try {
+ Assert.assertFalse(client.commit(tx));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testAbortTwice() throws Exception {
+ TransactionSystemClient client = getClient();
+ Transaction tx = client.startShort();
+
+ Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
+ client.abort(tx);
+ // abort of not active tx has no affect
+ client.abort(tx);
+ }
+
+ @Test
+ public void testReuseTx() throws Exception {
+ TransactionSystemClient client = getClient();
+ Transaction tx = client.startShort();
+
+ Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
+ Assert.assertTrue(client.commit(tx));
+ // can't re-use same tx again
+ try {
+ client.canCommit(tx, asList(C3, C4));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+ try {
+ Assert.assertFalse(client.commit(tx));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+
+ // abort of not active tx has no affect
+ client.abort(tx);
+ }
+
+ @Test
+ public void testUseNotStarted() throws Exception {
+ TransactionSystemClient client = getClient();
+ Transaction tx1 = client.startShort();
+ Assert.assertTrue(client.commit(tx1));
+
+ // we know this is one is older than current writePointer and was not used
+ Transaction txOld = new Transaction(tx1.getReadPointer(), tx1.getTransactionId() - 1,
+ new long[] {}, new long[] {}, Transaction.NO_TX_IN_PROGRESS,
+ TransactionType.SHORT);
+ try {
+ Assert.assertFalse(client.canCommit(txOld, asList(C3, C4)));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+ try {
+ Assert.assertFalse(client.commit(txOld));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+ // abort of not active tx has no affect
+ client.abort(txOld);
+
+ // we know this is one is newer than current readPointer and was not used
+ Transaction txNew = new Transaction(tx1.getReadPointer(), tx1.getTransactionId() + 1,
+ new long[] {}, new long[] {}, Transaction.NO_TX_IN_PROGRESS,
+ TransactionType.SHORT);
+ try {
+ Assert.assertFalse(client.canCommit(txNew, asList(C3, C4)));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+ try {
+ Assert.assertFalse(client.commit(txNew));
+ Assert.fail();
+ } catch (TransactionNotInProgressException e) {
+ // expected
+ }
+ // abort of not active tx has no affect
+ client.abort(txNew);
+ }
+
+ @Test
+ public void testAbortAfterCommit() throws Exception {
+ TransactionSystemClient client = getClient();
+ Transaction tx = client.startShort();
+
+ Assert.assertTrue(client.canCommit(tx, asList(C1, C2)));
+ Assert.assertTrue(client.commit(tx));
+ // abort of not active tx has no affect
+ client.abort(tx);
+ }
+
+ // todo add test invalidate method
+ @Test
+ public void testInvalidateTx() throws Exception {
+ TransactionSystemClient client = getClient();
+ // Invalidate an in-progress tx
+ Transaction tx1 = client.startShort();
+ client.canCommit(tx1, asList(C1, C2));
+ Assert.assertTrue(client.invalidate(tx1.getTransactionId()));
+ // Cannot invalidate a committed tx
+ Transaction tx2 = client.startShort();
+ client.canCommit(tx2, asList(C3, C4));
+ client.commit(tx2);
+ Assert.assertFalse(client.invalidate(tx2.getTransactionId()));
+ }
+
+ @Test
+ public void testResetState() throws Exception {
+ // have tx in progress, committing and committed then reset,
+ // get the last snapshot and see that it is empty
+ TransactionSystemClient client = getClient();
+ TransactionStateStorage stateStorage = getStateStorage();
+
+ Transaction tx1 = client.startShort();
+ Transaction tx2 = client.startShort();
+ client.canCommit(tx1, asList(C1, C2));
+ client.commit(tx1);
+ client.canCommit(tx2, asList(C3, C4));
+
+ Transaction txPreReset = client.startShort();
+ long currentTs = System.currentTimeMillis();
+ client.resetState();
+
+ TransactionSnapshot snapshot = stateStorage.getLatestSnapshot();
+ Assert.assertTrue(snapshot.getTimestamp() >= currentTs);
+ Assert.assertEquals(0, snapshot.getInvalid().size());
+ Assert.assertEquals(0, snapshot.getInProgress().size());
+ Assert.assertEquals(0, snapshot.getCommittingChangeSets().size());
+ Assert.assertEquals(0, snapshot.getCommittedChangeSets().size());
+
+ // confirm that transaction IDs are not reset
+ Transaction txPostReset = client.startShort();
+ Assert.assertTrue("New tx ID should be greater than last ID before reset",
+ txPostReset.getTransactionId() > txPreReset.getTransactionId());
+ }
+
+ @Test
+ public void testTruncateInvalidTx() throws Exception {
+ // Start few transactions and invalidate all of them
+ TransactionSystemClient client = getClient();
+ Transaction tx1 = client.startLong();
+ Transaction tx2 = client.startShort();
+ Transaction tx3 = client.startLong();
+
+ client.invalidate(tx1.getTransactionId());
+ client.invalidate(tx2.getTransactionId());
+ client.invalidate(tx3.getTransactionId());
+
+ // Remove tx2 and tx3 from invalid list
+ Assert.assertTrue(client.truncateInvalidTx(ImmutableSet.of(tx2.getTransactionId(), tx3.getTransactionId())));
+
+ Transaction tx = client.startShort();
+ // Only tx1 should be in invalid list now
+ Assert.assertArrayEquals(new long[] {tx1.getTransactionId()}, tx.getInvalids());
+ client.abort(tx);
+ }
+
+ @Test
+ public void testTruncateInvalidTxBefore() throws Exception {
+ // Start few transactions
+ TransactionSystemClient client = getClient();
+ Transaction tx1 = client.startLong();
+ Transaction tx2 = client.startShort();
+ // Sleep so that transaction ids get generated a millisecond apart for assertion
+ // TEPHRA-63 should eliminate the need to sleep
+ TimeUnit.MILLISECONDS.sleep(1);
+ long beforeTx3 = System.currentTimeMillis();
+ Transaction tx3 = client.startLong();
+
+ try {
+ // throws exception since tx1 and tx2 are still in-progress
+ client.truncateInvalidTxBefore(beforeTx3);
+ Assert.fail("Expected InvalidTruncateTimeException exception");
+ } catch (InvalidTruncateTimeException e) {
+ // Expected exception
+ }
+
+ // Invalidate all of them
+ client.invalidate(tx1.getTransactionId());
+ client.invalidate(tx2.getTransactionId());
+ client.invalidate(tx3.getTransactionId());
+
+ // Remove transactions before time beforeTx3
+ Assert.assertTrue(client.truncateInvalidTxBefore(beforeTx3));
+
+ Transaction tx = client.startShort();
+ // Only tx3 should be in invalid list now
+ Assert.assertArrayEquals(new long[] {tx3.getTransactionId()}, tx.getInvalids());
+ client.abort(tx);
+ }
+
+ @Test
+ public void testGetInvalidSize() throws Exception {
+ // Start few transactions and invalidate all of them
+ TransactionSystemClient client = getClient();
+ Transaction tx1 = client.startLong();
+ Transaction tx2 = client.startShort();
+ Transaction tx3 = client.startLong();
+
+ Assert.assertEquals(0, client.getInvalidSize());
+
+ client.invalidate(tx1.getTransactionId());
+ client.invalidate(tx2.getTransactionId());
+ client.invalidate(tx3.getTransactionId());
+
+ Assert.assertEquals(3, client.getInvalidSize());
+ }
+
+ private Collection<byte[]> asList(byte[]... val) {
+ return Arrays.asList(val);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/TransactionTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/TransactionTest.java b/tephra-core/src/test/java/org/apache/tephra/TransactionTest.java
new file mode 100644
index 0000000..08961f4
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/TransactionTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.primitives.Longs;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Set;
+
+/**
+ *
+ */
+public class TransactionTest {
+ // Current transaction id
+ private final long txId = 200L;
+ // Read pointer for current transaction
+ private final long readPointer = 100L;
+ // Transactions committed before current transaction was started.
+ private final Set<Long> priorCommitted = ImmutableSet.of(80L, 99L, 100L);
+ // Transactions committed after current transaction was started.
+ private final Set<Long> postCommitted = ImmutableSet.of(150L, 180L, 210L);
+ // Invalid transactions before current transaction was started.
+ private final Set<Long> priorInvalids = ImmutableSet.of(90L, 110L, 190L);
+ // Invalid transactions after current transaction was started.
+ private final Set<Long> postInvalids = ImmutableSet.of(201L, 221L, 231L);
+ // Transactions in progress before current transaction was started.
+ private final Set<Long> priorInProgress = ImmutableSet.of(95L, 120L, 150L);
+ // Transactions in progress after current transaction was started.
+ private final Set<Long> postInProgress = ImmutableSet.of(205L, 215L, 300L);
+
+ @Test
+ public void testSnapshotVisibility() throws Exception {
+ Transaction.VisibilityLevel visibilityLevel = Transaction.VisibilityLevel.SNAPSHOT;
+
+ Set<Long> checkPointers = ImmutableSet.of(220L, 250L);
+ Transaction tx = new Transaction(readPointer, txId, 250L, toSortedArray(priorInvalids),
+ toSortedArray(priorInProgress), 95L,
+ TransactionType.SHORT, toSortedArray(checkPointers),
+ visibilityLevel);
+ Set<Long> visibleCurrent = ImmutableSet.of(200L, 220L, 250L);
+ Set<Long> notVisibleCurrent = ImmutableSet.of();
+ assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
+ visibleCurrent, notVisibleCurrent, tx);
+
+ checkPointers = ImmutableSet.of();
+ tx = new Transaction(readPointer, txId, txId, toSortedArray(priorInvalids), toSortedArray(priorInProgress), 95L,
+ TransactionType.SHORT, toSortedArray(checkPointers),
+ visibilityLevel);
+ visibleCurrent = ImmutableSet.of(txId);
+ notVisibleCurrent = ImmutableSet.of();
+ assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
+ visibleCurrent, notVisibleCurrent, tx);
+ }
+
+ @Test
+ public void testSnapshotExcludeVisibility() throws Exception {
+ Transaction.VisibilityLevel visibilityLevel = Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT;
+
+ Set<Long> checkPointers = ImmutableSet.of(220L, 250L);
+ Transaction tx = new Transaction(readPointer, txId, 250L, toSortedArray(priorInvalids),
+ toSortedArray(priorInProgress), 95L,
+ TransactionType.SHORT, toSortedArray(checkPointers),
+ visibilityLevel);
+ Set<Long> visibleCurrent = ImmutableSet.of(200L, 220L);
+ Set<Long> notVisibleCurrent = ImmutableSet.of(250L);
+ assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
+ visibleCurrent, notVisibleCurrent, tx);
+
+ checkPointers = ImmutableSet.of();
+ tx = new Transaction(readPointer, txId, txId, toSortedArray(priorInvalids), toSortedArray(priorInProgress), 95L,
+ TransactionType.SHORT, toSortedArray(checkPointers),
+ visibilityLevel);
+ visibleCurrent = ImmutableSet.of();
+ notVisibleCurrent = ImmutableSet.of(txId);
+ assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
+ visibleCurrent, notVisibleCurrent, tx);
+ }
+
+ @Test
+ public void testSnapshotAllVisibility() throws Exception {
+ Transaction.VisibilityLevel visibilityLevel = Transaction.VisibilityLevel.SNAPSHOT_ALL;
+
+ Set<Long> checkPointers = ImmutableSet.of(220L, 250L);
+ Transaction tx = new Transaction(readPointer, txId, 250L, toSortedArray(priorInvalids),
+ toSortedArray(priorInProgress), 95L,
+ TransactionType.SHORT, toSortedArray(checkPointers),
+ visibilityLevel);
+ Set<Long> visibleCurrent = ImmutableSet.of(200L, 220L, 250L);
+ Set<Long> notVisibleCurrent = ImmutableSet.of();
+ assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
+ visibleCurrent, notVisibleCurrent, tx);
+
+ checkPointers = ImmutableSet.of();
+ tx = new Transaction(readPointer, txId, txId, toSortedArray(priorInvalids),
+ toSortedArray(priorInProgress), 95L,
+ TransactionType.SHORT, toSortedArray(checkPointers),
+ visibilityLevel);
+ visibleCurrent = ImmutableSet.of(txId);
+ notVisibleCurrent = ImmutableSet.of();
+ assertVisibility(priorCommitted, postCommitted, priorInvalids, postInvalids, priorInProgress, postInProgress,
+ visibleCurrent, notVisibleCurrent, tx);
+ }
+
+ private void assertVisibility(Set<Long> priorCommitted, Set<Long> postCommitted, Set<Long> priorInvalids,
+ Set<Long> postInvalids, Set<Long> priorInProgress, Set<Long> postInProgress,
+ Set<Long> visibleCurrent, Set<Long> notVisibleCurrent,
+ Transaction tx) {
+ // Verify visible snapshots of tx are visible
+ for (long t : visibleCurrent) {
+ Assert.assertTrue("Assertion error for version = " + t, tx.isVisible(t));
+ }
+
+ // Verify not visible snapshots of tx are not visible
+ for (long t : notVisibleCurrent) {
+ Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t));
+ }
+
+ // Verify prior committed versions are visible
+ for (long t : priorCommitted) {
+ Assert.assertTrue("Assertion error for version = " + t, tx.isVisible(t));
+ }
+
+ // Verify versions committed after tx started, and not part of tx are not visible
+ for (long t : postCommitted) {
+ Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t));
+ }
+
+ // Verify invalid and in-progress versions are not visible
+ for (long t : Iterables.concat(priorInvalids, postInvalids, priorInProgress, postInProgress)) {
+ Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t));
+ }
+ }
+
+ private long[] toSortedArray(Set<Long> set) {
+ long[] array = Longs.toArray(set);
+ Arrays.sort(array);
+ return array;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/distributed/ElasticPoolTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/distributed/ElasticPoolTest.java b/tephra-core/src/test/java/org/apache/tephra/distributed/ElasticPoolTest.java
new file mode 100644
index 0000000..6a30280
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/distributed/ElasticPoolTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.base.Throwables;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Tests for {@link ElasticPool}.
+ */
+public class ElasticPoolTest {
+
+ static class Dummy {
+ static AtomicInteger count = new AtomicInteger(0);
+ boolean valid = true;
+ Dummy() {
+ count.incrementAndGet();
+ }
+ void markInvalid() {
+ valid = false;
+ }
+
+ public boolean isValid() {
+ return valid;
+ }
+ }
+
+ class DummyPool extends ElasticPool<Dummy, RuntimeException> {
+
+ public DummyPool(int sizeLimit) {
+ super(sizeLimit);
+ }
+
+ @Override
+ protected Dummy create() {
+ return new Dummy();
+ }
+
+ @Override
+ protected boolean recycle(Dummy element) {
+ return element.isValid();
+ }
+ }
+
+ @Test(timeout = 5000)
+ public void testFewerThreadsThanElements() throws InterruptedException {
+ final DummyPool pool = new DummyPool(5);
+ Dummy.count.set(0);
+ createAndRunThreads(2, pool, false);
+ // we only ran 2 threads, so only 2 elements got created, even though pool size is 5
+ Assert.assertEquals(2, Dummy.count.get());
+ }
+
+ @Test(timeout = 5000)
+ public void testMoreThreadsThanElements() throws InterruptedException {
+ final DummyPool pool = new DummyPool(2);
+ Dummy.count.set(0);
+ createAndRunThreads(5, pool, false);
+ // even though we ran 5 threads, only 2 elements got created because pool size is 2
+ Assert.assertEquals(2, Dummy.count.get());
+ }
+
+ @Test(timeout = 5000)
+ public void testMoreThreadsThanElementsWithDiscard() throws InterruptedException {
+ final DummyPool pool = new DummyPool(2);
+ Dummy.count.set(0);
+ int numThreads = 3;
+ // pass 'true' as the last parameter, which results in the elements being discarded after each obtain() call.
+ createAndRunThreads(numThreads, pool, true);
+ // this results in (5 * numThreads) number of elements being created since each thread obtains a client 5 times.
+ Assert.assertEquals(5 * numThreads, Dummy.count.get());
+ }
+
+ // Creates a list of threads which obtain a client from the pool, sleeps for a certain amount of time, and then
+ // releases the client back to the pool, optionally marking it invalid before doing so. It repeats this five times.
+ // Then, runs these threads to completion.
+ private void createAndRunThreads(int numThreads, final DummyPool pool,
+ final boolean discardAtEnd) throws InterruptedException {
+ Thread[] threads = new Thread[numThreads];
+ for (int i = 0; i < numThreads; i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ for (int j = 0; j < 5; ++j) {
+ Dummy dummy;
+ try {
+ dummy = pool.obtain();
+ } catch (InterruptedException e) {
+ throw Throwables.propagate(e);
+ }
+ try {
+ Thread.sleep(10L);
+ } catch (InterruptedException e) {
+ // ignored
+ }
+ if (discardAtEnd) {
+ dummy.markInvalid();
+ }
+ pool.release(dummy);
+ }
+ }
+ };
+ }
+ for (Thread t : threads) {
+ t.start();
+ }
+ for (Thread t : threads) {
+ t.join();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/distributed/PooledClientProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/distributed/PooledClientProviderTest.java b/tephra-core/src/test/java/org/apache/tephra/distributed/PooledClientProviderTest.java
new file mode 100644
index 0000000..90a69e9
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/distributed/PooledClientProviderTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.base.Throwables;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TransactionServiceMain;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
+
+public class PooledClientProviderTest {
+
+ public static final int MAX_CLIENT_COUNT = 3;
+ public static final long CLIENT_OBTAIN_TIMEOUT = 10;
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Test
+ public void testClientConnectionPoolMaximumNumberOfClients() throws Exception {
+ // We need a server for the client to connect to
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
+ zkServer.startAndWait();
+
+ try {
+ Configuration conf = new Configuration();
+ conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
+ conf.set("data.tx.client.count", Integer.toString(MAX_CLIENT_COUNT));
+ conf.set("data.tx.client.obtain.timeout", Long.toString(CLIENT_OBTAIN_TIMEOUT));
+
+ final TransactionServiceMain main = new TransactionServiceMain(conf);
+ final CountDownLatch latch = new CountDownLatch(1);
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ main.start();
+ latch.countDown();
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ };
+
+ try {
+ t.start();
+ // Wait for service to startup
+ latch.await();
+
+ startClientAndTestPool(conf);
+ } finally {
+ main.stop();
+ t.join();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ private void startClientAndTestPool(Configuration conf) throws Exception {
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ new TransactionModules().getDistributedModules(),
+ new TransactionClientModule()
+ );
+
+ ZKClientService zkClient = injector.getInstance(ZKClientService.class);
+ zkClient.startAndWait();
+
+ final PooledClientProvider clientProvider = new PooledClientProvider(conf,
+ injector.getInstance(DiscoveryServiceClient.class));
+
+ // test simple case of get + return. Note: this also initializes the provider's pool, which
+ // takes about one second (discovery). Doing it before we test the threads makes it so that one
+ // thread doesn't take exceptionally longer than the others.
+ try (CloseableThriftClient closeableThriftClient = clientProvider.getCloseableClient()) {
+ // do nothing with the client
+ }
+
+ //Now race to get MAX_CLIENT_COUNT+1 clients, exhausting the pool and requesting 1 more.
+ List<Future<Integer>> clientIds = new ArrayList<Future<Integer>>();
+ CountDownLatch countDownLatch = new CountDownLatch(1);
+ ExecutorService executor = Executors.newFixedThreadPool(MAX_CLIENT_COUNT + 1);
+ for (int i = 0; i < MAX_CLIENT_COUNT + 1; i++) {
+ clientIds.add(executor.submit(new RetrieveClient(clientProvider, CLIENT_OBTAIN_TIMEOUT / 2, countDownLatch)));
+ }
+ countDownLatch.countDown();
+
+ Set<Integer> ids = new HashSet<Integer>();
+ for (Future<Integer> id : clientIds) {
+ ids.add(id.get());
+ }
+ Assert.assertEquals(MAX_CLIENT_COUNT, ids.size());
+
+ // now, try it again with, where each thread holds onto the client for twice the client.obtain.timeout value.
+ // one of the threads should throw a TimeOutException, because the other threads don't release their clients
+ // within the configured timeout.
+ countDownLatch = new CountDownLatch(1);
+ for (int i = 0; i < MAX_CLIENT_COUNT + 1; i++) {
+ clientIds.add(executor.submit(new RetrieveClient(clientProvider, CLIENT_OBTAIN_TIMEOUT * 2, countDownLatch)));
+ }
+ countDownLatch.countDown();
+ int numTimeoutExceptions = 0;
+ for (Future<Integer> clientId : clientIds) {
+ try {
+ clientId.get();
+ } catch (ExecutionException expected) {
+ Assert.assertEquals(TimeoutException.class, expected.getCause().getClass());
+ numTimeoutExceptions++;
+ }
+ }
+ // expect that exactly one of the threads hit the TimeoutException
+ Assert.assertEquals(String.format("Expected one thread to not obtain a client within %s milliseconds.",
+ CLIENT_OBTAIN_TIMEOUT),
+ 1, numTimeoutExceptions);
+
+ executor.shutdown();
+ }
+
+ private static class RetrieveClient implements Callable<Integer> {
+ private final PooledClientProvider pool;
+ private final long holdClientMs;
+ private final CountDownLatch begin;
+
+ public RetrieveClient(PooledClientProvider pool, long holdClientMs,
+ CountDownLatch begin) {
+ this.pool = pool;
+ this.holdClientMs = holdClientMs;
+ this.begin = begin;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ begin.await();
+ try (CloseableThriftClient client = pool.getCloseableClient()) {
+ int id = System.identityHashCode(client.getThriftClient());
+ // "use" the client for a configured amount of milliseconds
+ Thread.sleep(holdClientMs);
+ return id;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/distributed/ThriftTransactionServerTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/distributed/ThriftTransactionServerTest.java b/tephra-core/src/test/java/org/apache/tephra/distributed/ThriftTransactionServerTest.java
new file mode 100644
index 0000000..a930720
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/distributed/ThriftTransactionServerTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.util.concurrent.Service;
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.util.Modules;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.ThriftTransactionSystemTest;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionEdit;
+import org.apache.tephra.persist.TransactionLog;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This tests whether transaction service hangs on stop when heavily loaded - https://issues.cask.co/browse/TEPHRA-132
+ */
+public class ThriftTransactionServerTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ThriftTransactionSystemTest.class);
+
+ private static InMemoryZKServer zkServer;
+ private static ZKClientService zkClientService;
+ private static TransactionService txService;
+ private static TransactionStateStorage storage;
+ static Injector injector;
+
+ private static final int NUM_CLIENTS = 17;
+ private static final CountDownLatch STORAGE_WAIT_LATCH = new CountDownLatch(1);
+ private static final CountDownLatch CLIENTS_DONE_LATCH = new CountDownLatch(NUM_CLIENTS);
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @BeforeClass
+ public static void start() throws Exception {
+ zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
+ zkServer.startAndWait();
+
+ Configuration conf = new Configuration();
+ conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
+ conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
+ conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
+ conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
+ conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT, NUM_CLIENTS);
+ conf.setLong(TxConstants.Service.CFG_DATA_TX_CLIENT_TIMEOUT, TimeUnit.HOURS.toMillis(1));
+ conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS, 2);
+ conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS, 4);
+
+ injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ Modules.override(new TransactionModules().getDistributedModules())
+ .with(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(TransactionStateStorage.class).to(SlowTransactionStorage.class).in(Scopes.SINGLETON);
+ }
+ }),
+ new TransactionClientModule()
+ );
+
+ zkClientService = injector.getInstance(ZKClientService.class);
+ zkClientService.startAndWait();
+
+ // start a tx server
+ txService = injector.getInstance(TransactionService.class);
+ storage = injector.getInstance(TransactionStateStorage.class);
+ try {
+ LOG.info("Starting transaction service");
+ txService.startAndWait();
+ } catch (Exception e) {
+ LOG.error("Failed to start service: ", e);
+ }
+ }
+
+ @Before
+ public void reset() throws Exception {
+ getClient().resetState();
+ }
+
+ @AfterClass
+ public static void stop() throws Exception {
+ txService.stopAndWait();
+ storage.stopAndWait();
+ zkClientService.stopAndWait();
+ zkServer.stopAndWait();
+ }
+
+ public TransactionSystemClient getClient() throws Exception {
+ return injector.getInstance(TransactionSystemClient.class);
+ }
+
+ @Test
+ public void testThriftServerStop() throws Exception {
+ int nThreads = NUM_CLIENTS;
+ ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
+ for (int i = 0; i < nThreads; ++i) {
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TransactionSystemClient txClient = getClient();
+ CLIENTS_DONE_LATCH.countDown();
+ txClient.startShort();
+ } catch (Exception e) {
+ // Exception expected
+ }
+ }
+ });
+ }
+
+ // Wait till all clients finish sending reqeust to transaction manager
+ CLIENTS_DONE_LATCH.await();
+ TimeUnit.SECONDS.sleep(1);
+
+ // Expire zookeeper session, which causes Thrift server to stop.
+ expireZkSession(zkClientService);
+ waitForThriftTermination();
+
+ // Stop Zookeeper client so that it does not re-connect to Zookeeper and start Thrift sever again.
+ zkClientService.stopAndWait();
+ STORAGE_WAIT_LATCH.countDown();
+ TimeUnit.SECONDS.sleep(1);
+
+ // Make sure Thrift server stopped.
+ Assert.assertEquals(Service.State.TERMINATED, txService.thriftRPCServerState());
+ }
+
+ private void expireZkSession(ZKClientService zkClientService) throws Exception {
+ ZooKeeper zooKeeper = zkClientService.getZooKeeperSupplier().get();
+ final SettableFuture<?> connectFuture = SettableFuture.create();
+ Watcher watcher = new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getState() == Event.KeeperState.SyncConnected) {
+ connectFuture.set(null);
+ }
+ }
+ };
+
+ // Create another Zookeeper session with the same sessionId so that the original one expires.
+ final ZooKeeper dupZookeeper =
+ new ZooKeeper(zkClientService.getConnectString(), zooKeeper.getSessionTimeout(), watcher,
+ zooKeeper.getSessionId(), zooKeeper.getSessionPasswd());
+ connectFuture.get(30, TimeUnit.SECONDS);
+ Assert.assertEquals("Failed to re-create current session", dupZookeeper.getState(), ZooKeeper.States.CONNECTED);
+ dupZookeeper.close();
+ }
+
+ private void waitForThriftTermination() throws InterruptedException {
+ int count = 0;
+ while (txService.thriftRPCServerState() != Service.State.TERMINATED && count++ < 200) {
+ TimeUnit.MILLISECONDS.sleep(50);
+ }
+ }
+
+ private static class SlowTransactionStorage extends InMemoryTransactionStateStorage {
+ @Override
+ public TransactionLog createLog(long timestamp) throws IOException {
+ return new SlowTransactionLog(timestamp);
+ }
+ }
+
+ private static class SlowTransactionLog extends InMemoryTransactionStateStorage.InMemoryTransactionLog {
+ public SlowTransactionLog(long timestamp) {
+ super(timestamp);
+ }
+
+ @Override
+ public void append(TransactionEdit edit) throws IOException {
+ try {
+ STORAGE_WAIT_LATCH.await();
+ } catch (InterruptedException e) {
+ LOG.error("Got exception: ", e);
+ }
+ super.append(edit);
+ }
+
+ @Override
+ public void append(List<TransactionEdit> edits) throws IOException {
+ try {
+ STORAGE_WAIT_LATCH.await();
+ } catch (InterruptedException e) {
+ LOG.error("Got exception: ", e);
+ }
+ super.append(edits);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/org/apache/tephra/hbase/AbstractTransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/hbase/AbstractTransactionVisibilityFilterTest.java b/tephra-core/src/test/java/org/apache/tephra/hbase/AbstractTransactionVisibilityFilterTest.java
new file mode 100644
index 0000000..4828adf
--- /dev/null
+++ b/tephra-core/src/test/java/org/apache/tephra/hbase/AbstractTransactionVisibilityFilterTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.util.ConfigurationFactory;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.List;
+
+/**
+ * Common test class for TransactionVisibilityFilter implementations.
+ */
+public abstract class AbstractTransactionVisibilityFilterTest {
+
+ protected static final byte[] FAM = new byte[] {'f'};
+ protected static final byte[] FAM2 = new byte[] {'f', '2'};
+ protected static final byte[] FAM3 = new byte[] {'f', '3'};
+ protected static final byte[] COL = new byte[] {'c'};
+ protected static final List<byte[]> EMPTY_CHANGESET = Lists.newArrayListWithCapacity(0);
+
+ protected TransactionManager txManager;
+
+ @Before
+ public void setup() throws Exception {
+ Configuration conf = new ConfigurationFactory().get();
+ conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ txManager = new TransactionManager(conf);
+ txManager.startAndWait();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ txManager.stopAndWait();
+ }
+}
[06/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionFilters.java
deleted file mode 100644
index a818524..0000000
--- a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionFilters.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10.coprocessor;
-
-import co.cask.tephra.Transaction;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Factory class for providing {@link Filter} instances.
- */
-public class TransactionFilters {
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
- }
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessor.java
deleted file mode 100644
index 43329f0..0000000
--- a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessor.java
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCodec;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreScanner;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-
-/**
- * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
- * for transactions:
- * <ul>
- * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
- * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
- * or expired due to TTL.</li>
- * </ul>
- *
- * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
- * or on all user tables by adding the following to hbase-site.xml:
- * {@code
- * <property>
- * <name>hbase.coprocessor.region.classes</name>
- * <value>co.cask.tephra.hbase10.coprocessor.TransactionProcessor</value>
- * </property>
- * }
- * </p>
- *
- * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
- * as an attribute:
- * {@code
- * Transaction t = ...;
- * Get get = new Get(...);
- * TransactionCodec codec = new TransactionCodec();
- * codec.addToOperation(get, t);
- * }
- * </p>
- */
-public class TransactionProcessor extends BaseRegionObserver {
- private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
-
- private TransactionStateCache cache;
- private final TransactionCodec txCodec;
- protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
- protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
-
- public TransactionProcessor() {
- this.txCodec = new TransactionCodec();
- }
-
- /* RegionObserver implementation */
-
- @Override
- public void start(CoprocessorEnvironment e) throws IOException {
- if (e instanceof RegionCoprocessorEnvironment) {
- RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
- Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
- this.cache = cacheSupplier.get();
-
- HTableDescriptor tableDesc = env.getRegion().getTableDesc();
- for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
- String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
- long ttl = 0;
- if (columnTTL != null) {
- try {
- ttl = Long.parseLong(columnTTL);
- LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
- } catch (NumberFormatException nfe) {
- LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
- ", value = " + columnTTL);
- }
- }
- ttlByFamily.put(columnDesc.getName(), ttl);
- }
-
- this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
- TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
- this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
- if (readNonTxnData) {
- LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
- }
- }
- }
-
- protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
- return new TransactionStateCacheSupplier(env.getConfiguration());
- }
-
- @Override
- public void stop(CoprocessorEnvironment e) throws IOException {
- // nothing to do
- }
-
- @Override
- public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
- throws IOException {
- Transaction tx = getFromOperation(get);
- if (tx != null) {
- projectFamilyDeletes(get);
- get.setMaxVersions();
- get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
- get.setFilter(newFilter);
- }
- }
-
- @Override
- public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
- Durability durability) throws IOException {
- // Translate deletes into our own delete tombstones
- // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
- // us to rollback the changes (by a real delete) if the transaction fails
-
- // Deletes that are part of a transaction rollback do not need special handling.
- // They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
- return;
- }
-
- // Other deletes are client-initiated and need to be translated into our own tombstones
- // TODO: this should delegate to the DeleteStrategy implementation.
- Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
- for (byte[] family : delete.getFamilyCellMap().keySet()) {
- List<Cell> familyCells = delete.getFamilyCellMap().get(family);
- if (isFamilyDelete(familyCells)) {
- deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- } else {
- int cellSize = familyCells.size();
- for (int i = 0; i < cellSize; i++) {
- Cell cell = familyCells.get(i);
- deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
- }
- e.getEnvironment().getRegion().put(deleteMarkers);
- // skip normal delete handling
- e.bypass();
- }
-
- private boolean isFamilyDelete(List<Cell> familyCells) {
- return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
- }
-
- @Override
- public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
- throws IOException {
- Transaction tx = getFromOperation(scan);
- if (tx != null) {
- projectFamilyDeletes(scan);
- scan.setMaxVersions();
- scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
- scan.setFilter(newFilter);
- }
- return s;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any scan operation.
- * @param scan The original scan request
- * @return The modified scan request with the family delete qualifiers represented
- */
- private Scan projectFamilyDeletes(Scan scan) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return scan;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any get operation.
- * @param get The original get request
- * @return The modified get request with the family delete qualifiers represented
- */
- private Get projectFamilyDeletes(Get get) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return get;
- }
-
- @Override
- public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- KeyValueScanner memstoreScanner, InternalScanner scanner)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
- Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
- HConstants.OLDEST_TIMESTAMP);
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
- CompactionRequest request)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners,
- scanType, earliestPutTs);
- }
-
- protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
- TransactionVisibilityState snapshot, Store store,
- List<? extends KeyValueScanner> scanners, ScanType type,
- long earliestPutTs) throws IOException {
- if (snapshot == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
- ", no current transaction state found, defaulting to normal " + action + " scanner");
- }
- return null;
- }
-
- // construct a dummy transaction from the latest snapshot
- Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
- Scan scan = new Scan();
- // need to see all versions, since we filter out excludes and applications may rely on multiple versions
- scan.setMaxVersions();
- scan.setFilter(
- new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
- snapshot.getInvalid(),
- getTransactionFilter(dummyTx, type, null)));
-
- return new StoreScanner(store, store.getScanInfo(), scan, scanners,
- type, store.getSmallestReadPoint(), earliestPutTs);
- }
-
- private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
- byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
- if (encoded != null) {
- return txCodec.decode(encoded);
- }
- return null;
- }
-
- /**
- * Derived classes can override this method to customize the filter used to return data visible for the current
- * transaction.
- *
- * @param tx the current transaction to apply
- * @param type the type of scan being performed
- */
- protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
- return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
- }
-
- /**
- * Filter used to include cells visible to in-progress transactions on flush and commit.
- */
- static class IncludeInProgressFilter extends FilterBase {
- private final long visibilityUpperBound;
- private final Set<Long> invalidIds;
- private final Filter txFilter;
-
- public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
- this.visibilityUpperBound = upperBound;
- this.invalidIds = Sets.newHashSet(invalids);
- this.txFilter = transactionFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- // include all cells visible to in-progress transactions, except for those already marked as invalid
- long ts = cell.getTimestamp();
- if (ts > visibilityUpperBound) {
- // include everything that could still be in-progress except invalids
- if (invalidIds.contains(ts)) {
- return ReturnCode.SKIP;
- }
- return ReturnCode.INCLUDE;
- }
- return txFilter.filterKeyValue(cell);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java
deleted file mode 100644
index 0d85ad5..0000000
--- a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilter.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Applies filtering of data based on transactional visibility (HBase 1.0+ specific version).
- * Note: this is intended for server-side use only, as additional properties need to be set on
- * any {@code Scan} or {@code Get} operation performed.
- */
-public class TransactionVisibilityFilter extends FilterBase {
- private final Transaction tx;
- // oldest visible timestamp by column family, used to apply TTL when reading
- private final Map<ImmutableBytesWritable, Long> oldestTsByFamily;
- // if false, empty values will be interpreted as deletes
- private final boolean allowEmptyValues;
- // whether or not we can remove delete markers
- // these can only be safely removed when we are traversing all storefiles
- private final boolean clearDeletes;
- // optional sub-filter to apply to visible cells
- private final Filter cellFilter;
- // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
- private final ImmutableBytesWritable currentFamily = new ImmutableBytesWritable(HConstants.EMPTY_BYTE_ARRAY);
-
- private long currentOldestTs;
-
- private DeleteTracker deleteTracker = new DeleteTracker();
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- this(tx, ttlByFamily, allowEmptyValues, scanType, null);
- }
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- this.tx = tx;
- this.oldestTsByFamily = Maps.newTreeMap();
- for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
- long familyTTL = ttlEntry.getValue();
- oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
- familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
- }
- this.allowEmptyValues = allowEmptyValues;
- this.clearDeletes =
- scanType == ScanType.COMPACT_DROP_DELETES ||
- (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
- this.cellFilter = cellFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (!CellUtil.matchingFamily(cell, currentFamily.get(), currentFamily.getOffset(), currentFamily.getLength())) {
- // column family changed
- currentFamily.set(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
- Long familyOldestTs = oldestTsByFamily.get(currentFamily);
- currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
- deleteTracker.reset();
- }
- // need to apply TTL for the column family here
- long kvTimestamp = cell.getTimestamp();
- if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
- // passed TTL for this column, seek to next
- return ReturnCode.NEXT_COL;
- } else if (tx.isVisible(kvTimestamp)) {
- // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
- // cell is visible
- // visibility SNAPSHOT_ALL needs all matches
- return runSubFilter(ReturnCode.INCLUDE, cell);
- }
- if (DeleteTracker.isFamilyDelete(cell)) {
- deleteTracker.addFamilyDelete(cell);
- if (clearDeletes) {
- return ReturnCode.NEXT_COL;
- } else {
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // check if masked by family delete
- if (deleteTracker.isDeleted(cell)) {
- return ReturnCode.NEXT_COL;
- }
- // check for column delete
- if (isColumnDelete(cell)) {
- if (clearDeletes) {
- // skip "deleted" cell
- return ReturnCode.NEXT_COL;
- } else {
- // keep the marker but skip any remaining versions
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- } else {
- return ReturnCode.SKIP;
- }
- }
-
- private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
- if (cellFilter != null) {
- ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
- return determineReturnCode(txFilterCode, subFilterCode);
- }
- return txFilterCode;
- }
-
- /**
- * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
- * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
- * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
- * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
- *
- * @param txFilterCode return code from TransactionVisibilityFilter
- * @param subFilterCode return code from sub-filter
- * @return final return code
- */
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- // Return the more restrictive of the two filter responses
- switch (subFilterCode) {
- case INCLUDE:
- return txFilterCode;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
-
- @Override
- public boolean filterRow() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRow();
- }
- return super.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- // Convert Tephra deletes back into HBase deletes
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
- if (DeleteTracker.isFamilyDelete(cell)) {
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
- KeyValue.Type.DeleteFamily);
- } else if (isColumnDelete(cell)) {
- // Note: in some cases KeyValue.Type.Delete is used in Delete object,
- // and in some other cases KeyValue.Type.DeleteColumn is used.
- // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
- // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
- // work in both cases.
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
- cell.getTimestamp(), KeyValue.Type.DeleteColumn);
- }
- }
- return cell;
- }
-
- @Override
- public void reset() throws IOException {
- deleteTracker.reset();
- if (cellFilter != null) {
- cellFilter.reset();
- }
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRowKey(buffer, offset, length);
- }
- return super.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterAllRemaining();
- }
- return super.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- if (cellFilter != null) {
- cellFilter.filterRowCells(kvs);
- } else {
- super.filterRowCells(kvs);
- }
- }
-
- @Override
- public boolean hasFilterRow() {
- if (cellFilter != null) {
- return cellFilter.hasFilterRow();
- }
- return super.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextKeyHint(currentKV);
- }
- return super.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextCellHint(currentKV);
- }
- return super.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- if (cellFilter != null) {
- return cellFilter.isFamilyEssential(name);
- }
- return super.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return super.toByteArray();
- }
-
- private boolean isColumnDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
- }
-
- private static final class DeleteTracker {
- private long familyDeleteTs;
-
- public static boolean isFamilyDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
- CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
- CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
- }
-
- public void addFamilyDelete(Cell delete) {
- this.familyDeleteTs = delete.getTimestamp();
- }
-
- public boolean isDeleted(Cell cell) {
- return cell.getTimestamp() <= familyDeleteTs;
- }
-
- public void reset() {
- this.familyDeleteTs = 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/HBase10ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/HBase10ConfigurationProvider.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/HBase10ConfigurationProvider.java
new file mode 100644
index 0000000..104c707
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/HBase10ConfigurationProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.tephra.util.ConfigurationProvider;
+
+/**
+ * HBase 1.0 version of {@link ConfigurationProvider}.
+ */
+public class HBase10ConfigurationProvider extends ConfigurationProvider {
+ @Override
+ public Configuration get() {
+ return HBaseConfiguration.create();
+ }
+
+ @Override
+ public Configuration get(Configuration baseConf) {
+ return HBaseConfiguration.create(baseConf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/SecondaryIndexTable.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/SecondaryIndexTable.java
new file mode 100644
index 0000000..1124971
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/SecondaryIndexTable.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase10;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.apache.tephra.distributed.TransactionServiceClient;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A Transactional SecondaryIndexTable.
+ */
+public class SecondaryIndexTable {
+ private byte[] secondaryIndex;
+ private TransactionAwareHTable transactionAwareHTable;
+ private TransactionAwareHTable secondaryIndexTable;
+ private TransactionContext transactionContext;
+ private final TableName secondaryIndexTableName;
+ private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
+ private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
+ private static final byte[] DELIMITER = new byte[] {0};
+
+ public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
+ byte[] secondaryIndex) {
+ secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
+ HTable secondaryIndexHTable = null;
+ HBaseAdmin hBaseAdmin = null;
+ try {
+ hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
+ if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
+ hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
+ }
+ secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ } finally {
+ try {
+ hBaseAdmin.close();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ }
+
+ this.secondaryIndex = secondaryIndex;
+ this.transactionAwareHTable = new TransactionAwareHTable(hTable);
+ this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
+ this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
+ secondaryIndexTable);
+ }
+
+ public Result get(Get get) throws IOException {
+ return get(Collections.singletonList(get))[0];
+ }
+
+ public Result[] get(List<Get> gets) throws IOException {
+ try {
+ transactionContext.start();
+ Result[] result = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return result;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public Result[] getByIndex(byte[] value) throws IOException {
+ try {
+ transactionContext.start();
+ Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
+ scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
+ ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
+
+ ArrayList<Get> gets = new ArrayList<Get>();
+ for (Result result : indexScanner) {
+ for (Cell cell : result.listCells()) {
+ gets.add(new Get(cell.getValue()));
+ }
+ }
+ Result[] results = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return results;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public void put(Put put) throws IOException {
+ put(Collections.singletonList(put));
+ }
+
+
+ public void put(List<Put> puts) throws IOException {
+ try {
+ transactionContext.start();
+ ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
+ for (Put put : puts) {
+ List<Put> indexPuts = new ArrayList<Put>();
+ Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
+ for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
+ for (KeyValue value : family.getValue()) {
+ if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
+ secondaryIndex, 0, secondaryIndex.length)) {
+ byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
+ Bytes.add(value.getValue(), DELIMITER,
+ value.getRow()));
+ Put indexPut = new Put(secondaryRow);
+ indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
+ indexPuts.add(indexPut);
+ }
+ }
+ }
+ secondaryIndexPuts.addAll(indexPuts);
+ }
+ transactionAwareHTable.put(puts);
+ secondaryIndexTable.put(secondaryIndexPuts);
+ transactionContext.finish();
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java
new file mode 100644
index 0000000..3d0fb20
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java
@@ -0,0 +1,674 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase10;
+
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.tephra.AbstractTransactionAwareTable;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionAware;
+import org.apache.tephra.TxConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+/**
+ * A Transaction Aware HTable implementation for HBase 1.0. Operations are committed as usual,
+ * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
+ * was started.
+ */
+public class TransactionAwareHTable extends AbstractTransactionAwareTable
+ implements HTableInterface, TransactionAware {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
+ private final HTableInterface hTable;
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ */
+ public TransactionAwareHTable(HTableInterface hTable) {
+ this(hTable, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
+ this(hTable, conflictLevel, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
+ this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
+ boolean allowNonTransactional) {
+ super(conflictLevel, allowNonTransactional);
+ this.hTable = hTable;
+ }
+
+ /* AbstractTransactionAwareTable implementation */
+
+ @Override
+ protected byte[] getTableKey() {
+ return getTableName();
+ }
+
+ @Override
+ protected boolean doCommit() throws IOException {
+ hTable.flushCommits();
+ return true;
+ }
+
+ @Override
+ protected boolean doRollback() throws Exception {
+ try {
+ // pre-size arraylist of deletes
+ int size = 0;
+ for (Set<ActionChange> cs : changeSets.values()) {
+ size += cs.size();
+ }
+ List<Delete> rollbackDeletes = new ArrayList<>(size);
+ for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
+ long transactionTimestamp = entry.getKey();
+ for (ActionChange change : entry.getValue()) {
+ byte[] row = change.getRow();
+ byte[] family = change.getFamily();
+ byte[] qualifier = change.getQualifier();
+ Delete rollbackDelete = new Delete(row);
+ rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ switch (conflictLevel) {
+ case ROW:
+ case NONE:
+ // issue family delete for the tx write pointer
+ rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
+ break;
+ case COLUMN:
+ if (family != null && qualifier == null) {
+ rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
+ } else if (family != null && qualifier != null) {
+ rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
+ }
+ rollbackDeletes.add(rollbackDelete);
+ }
+ }
+ hTable.delete(rollbackDeletes);
+ return true;
+ } finally {
+ try {
+ hTable.flushCommits();
+ } catch (Exception e) {
+ LOG.error("Could not flush HTable commits", e);
+ }
+ tx = null;
+ changeSets.clear();
+ }
+ }
+
+ /* HTableInterface implementation */
+
+ @Override
+ public byte[] getTableName() {
+ return hTable.getTableName();
+ }
+
+ @Override
+ public TableName getName() {
+ return hTable.getName();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hTable.getConfiguration();
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return hTable.getTableDescriptor();
+ }
+
+ @Override
+ public boolean exists(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.exists(transactionalizeAction(get));
+ }
+
+ @Override
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.exists(transactionalizedGets);
+ }
+
+ @Override
+ public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batch(transactionalizeActions(actions), results);
+ }
+
+ @Override
+ public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batch(transactionalizeActions(actions));
+ }
+
+ @Override
+ public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
+ IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batchCallback(transactionalizeActions(actions), results, callback);
+ }
+
+ @Override
+ public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
+ InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batchCallback(transactionalizeActions(actions), callback);
+ }
+
+ @Override
+ public Result get(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.get(transactionalizeAction(get));
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.get(transactionalizedGets);
+ }
+
+ @Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.getRowOrBefore(row, family);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan scan) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addColumn(family, qualifier);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public void put(Put put) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Put txPut = transactionalizeAction(put);
+ hTable.put(txPut);
+ }
+
+ @Override
+ public void put(List<Put> puts) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
+ for (Put put : puts) {
+ Put txPut = transactionalizeAction(put);
+ transactionalizedPuts.add(txPut);
+ }
+ hTable.put(transactionalizedPuts);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndPut(row, family, qualifier, value, put);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public void delete(Delete delete) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.delete(transactionalizeAction(delete));
+ }
+
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
+ for (Delete delete : deletes) {
+ Delete txDelete = transactionalizeAction(delete);
+ transactionalizedDeletes.add(txDelete);
+ }
+ hTable.delete(transactionalizedDeletes);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndDelete(row, family, qualifier, value, delete);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
+ byte[] bytes3, Delete delete) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndDelete(bytes, bytes1, bytes2, compareOp, bytes3, delete);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
+ byte[] bytes3, Put put) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndPut(bytes, bytes1, bytes2, compareOp, bytes3, put);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean[] existsAll(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.existsAll(transactionalizedGets);
+ }
+
+ @Override
+ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
+ CompareFilter.CompareOp compareOp, byte[] value, RowMutations rowMutations)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndMutate(row, family, qualifier, compareOp, value, rowMutations);
+ }
+
+ throw new UnsupportedOperationException("checkAndMutate operation is not supported transactionally");
+ }
+
+ @Override
+ public void mutateRow(RowMutations rm) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ RowMutations transactionalMutations = new RowMutations();
+ for (Mutation mutation : rm.getMutations()) {
+ if (mutation instanceof Put) {
+ transactionalMutations.add(transactionalizeAction((Put) mutation));
+ } else if (mutation instanceof Delete) {
+ transactionalMutations.add(transactionalizeAction((Delete) mutation));
+ }
+ }
+ hTable.mutateRow(transactionalMutations);
+ }
+
+ @Override
+ public Result append(Append append) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.append(append);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public Result increment(Increment increment) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.increment(increment);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean isAutoFlush() {
+ return hTable.isAutoFlush();
+ }
+
+ @Override
+ public void flushCommits() throws IOException {
+ hTable.flushCommits();
+ }
+
+ @Override
+ public void close() throws IOException {
+ hTable.close();
+ }
+
+ @Override
+ public CoprocessorRpcChannel coprocessorService(byte[] row) {
+ return hTable.coprocessorService(row);
+ }
+
+ @Override
+ public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable)
+ throws ServiceException, Throwable {
+ return hTable.coprocessorService(service, startKey, endKey, callable);
+ }
+
+ @Override
+ public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable, Batch.Callback<R> callback)
+ throws ServiceException, Throwable {
+ hTable.coprocessorService(service, startKey, endKey, callable, callback);
+ }
+
+ @Override
+ public <R extends Message> Map<byte[], R> batchCoprocessorService(
+ MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey,
+ R responsePrototype) throws ServiceException, Throwable {
+ return hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
+ }
+
+ @Override
+ public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
+ Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
+ throws ServiceException, Throwable {
+ hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush) {
+ setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ hTable.setAutoFlush(autoFlush, clearBufferOnFail);
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ hTable.setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public long getWriteBufferSize() {
+ return hTable.getWriteBufferSize();
+ }
+
+ @Override
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ hTable.setWriteBufferSize(writeBufferSize);
+ }
+
+ // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
+
+ private Get transactionalizeAction(Get get) throws IOException {
+ addToOperation(get, tx);
+ return get;
+ }
+
+ private Scan transactionalizeAction(Scan scan) throws IOException {
+ addToOperation(scan, tx);
+ return scan;
+ }
+
+ private Put transactionalizeAction(Put put) throws IOException {
+ Put txPut = new Put(put.getRow(), tx.getWritePointer());
+ Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
+ if (!familyMap.isEmpty()) {
+ for (Map.Entry<byte[], List<Cell>> family : familyMap) {
+ List<Cell> familyValues = family.getValue();
+ if (!familyValues.isEmpty()) {
+ for (Cell value : familyValues) {
+ txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
+ addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
+ txPut.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txPut.setDurability(put.getDurability());
+ addToOperation(txPut, tx);
+ return txPut;
+ }
+
+ private Delete transactionalizeAction(Delete delete) throws IOException {
+ long transactionTimestamp = tx.getWritePointer();
+
+ byte[] deleteRow = delete.getRow();
+ Delete txDelete = new Delete(deleteRow, transactionTimestamp);
+
+ Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
+ if (familyToDelete.isEmpty()) {
+ // perform a row delete if we are using row-level conflict detection
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // Row delete leaves delete markers in all column families of the table
+ // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
+ for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
+ // no need to identify individual columns deleted
+ addToChangeSet(deleteRow, columnDescriptor.getName(), null);
+ }
+ } else {
+ Result result = get(new Get(delete.getRow()));
+ // Delete everything
+ NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
+ for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
+ }
+ }
+ }
+ } else {
+ for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
+ byte[] family = familyEntry.getKey();
+ List<Cell> entries = familyEntry.getValue();
+ boolean isFamilyDelete = false;
+ if (entries.size() == 1) {
+ Cell cell = entries.get(0);
+ isFamilyDelete = CellUtil.isDeleteFamily(cell);
+ }
+ if (isFamilyDelete) {
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // no need to identify individual columns deleted
+ txDelete.deleteFamily(family);
+ addToChangeSet(deleteRow, family, null);
+ } else {
+ Result result = get(new Get(delete.getRow()).addFamily(family));
+ // Delete entire family
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, family, column.getKey());
+ }
+ }
+ } else {
+ for (Cell value : entries) {
+ txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
+ addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ txDelete.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txDelete.setDurability(delete.getDurability());
+ return txDelete;
+ }
+
+ private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
+ List<Row> transactionalizedActions = new ArrayList<>(actions.size());
+ for (Row action : actions) {
+ if (action instanceof Get) {
+ transactionalizedActions.add(transactionalizeAction((Get) action));
+ } else if (action instanceof Put) {
+ transactionalizedActions.add(transactionalizeAction((Put) action));
+ } else if (action instanceof Delete) {
+ transactionalizedActions.add(transactionalizeAction((Delete) action));
+ } else {
+ transactionalizedActions.add(action);
+ }
+ }
+ return transactionalizedActions;
+ }
+
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilter.java
new file mode 100644
index 0000000..bef7275
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/CellSkipFilter.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
+ * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
+ * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
+ * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
+ * Please see TEPHRA-169 for more details.
+ */
+public class CellSkipFilter extends FilterBase {
+ private final Filter filter;
+ // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
+ private KeyValue skipColumn = null;
+
+ public CellSkipFilter(Filter filter) {
+ this.filter = filter;
+ }
+
+ /**
+ * Determines whether the current cell should be skipped. The cell will be skipped
+ * if the previous keyvalue had the same key as the current cell. This means filter already responded
+ * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
+ * @param cell the {@link Cell} to be tested for skipping
+ * @return true is current cell should be skipped, false otherwise
+ */
+ private boolean skipCellVersion(Cell cell) {
+ return skipColumn != null
+ && CellUtil.matchingRow(cell, skipColumn.getRowArray(), skipColumn.getRowOffset(),
+ skipColumn.getRowLength())
+ && CellUtil.matchingFamily(cell, skipColumn.getFamilyArray(), skipColumn.getFamilyOffset(),
+ skipColumn.getFamilyLength())
+ && CellUtil.matchingQualifier(cell, skipColumn.getQualifierArray(), skipColumn.getQualifierOffset(),
+ skipColumn.getQualifierLength());
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (skipCellVersion(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+
+ ReturnCode code = filter.filterKeyValue(cell);
+ if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
+ // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
+ skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(),
+ cell.getFamilyLength(), cell.getQualifierArray(),
+ cell.getQualifierOffset(), cell.getQualifierLength());
+ } else {
+ skipColumn = null;
+ }
+ return code;
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ return filter.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ return filter.transformCell(cell);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ filter.reset();
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ return filter.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ return filter.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ filter.filterRowCells(kvs);
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ return filter.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ return filter.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ return filter.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ return filter.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return filter.toByteArray();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionFilters.java
new file mode 100644
index 0000000..91c4713
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionFilters.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10.coprocessor;
+
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.tephra.Transaction;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Factory class for providing {@link Filter} instances.
+ */
+public class TransactionFilters {
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+ }
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+ }
+}
[51/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
Rename package to org.apache.tephra
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/9c693743
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/9c693743
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/9c693743
Branch: refs/heads/master
Commit: 9c693743e65f8a055afba4d8c866460ab1e93894
Parents: a85b013
Author: poorna <po...@cask.co>
Authored: Fri May 6 16:01:56 2016 -0700
Committer: poorna <po...@cask.co>
Committed: Fri May 6 16:01:58 2016 -0700
----------------------------------------------------------------------
README.rst | 26 +-
bin/tephra | 2 +-
conf/logback.xml | 2 +-
.../main/java/co/cask/tephra/Transaction.java | 296 -
.../java/co/cask/tephra/TransactionAware.java | 112 -
.../java/co/cask/tephra/TransactionAwares.java | 111 -
.../tephra/TransactionConflictException.java | 32 -
.../tephra/TransactionFailureException.java | 32 -
.../TransactionNotInProgressException.java | 28 -
.../java/co/cask/tephra/TransactionType.java | 33 -
.../java/org/apache/tephra/Transaction.java | 296 +
.../org/apache/tephra/TransactionAware.java | 112 +
.../org/apache/tephra/TransactionAwares.java | 111 +
.../tephra/TransactionConflictException.java | 32 +
.../tephra/TransactionFailureException.java | 32 +
.../TransactionNotInProgressException.java | 28 +
.../java/org/apache/tephra/TransactionType.java | 33 +
.../tephra/AbstractTransactionAwareTable.java | 228 -
.../tephra/AbstractTransactionExecutor.java | 129 -
.../src/main/java/co/cask/tephra/ChangeId.java | 77 -
.../cask/tephra/DefaultTransactionExecutor.java | 147 -
.../tephra/InvalidTruncateTimeException.java | 29 -
.../java/co/cask/tephra/NoRetryStrategy.java | 33 -
.../co/cask/tephra/RetryOnConflictStrategy.java | 41 -
.../java/co/cask/tephra/RetryStrategies.java | 40 -
.../main/java/co/cask/tephra/RetryStrategy.java | 34 -
.../java/co/cask/tephra/TransactionAdmin.java | 158 -
.../java/co/cask/tephra/TransactionCodec.java | 57 -
.../java/co/cask/tephra/TransactionContext.java | 323 -
...ransactionCouldNotTakeSnapshotException.java | 28 -
.../co/cask/tephra/TransactionExecutor.java | 141 -
.../cask/tephra/TransactionExecutorFactory.java | 28 -
.../java/co/cask/tephra/TransactionManager.java | 1398 ---
.../co/cask/tephra/TransactionServiceMain.java | 149 -
.../co/cask/tephra/TransactionSystemClient.java | 142 -
.../main/java/co/cask/tephra/TxConstants.java | 343 -
.../coprocessor/TransactionStateCache.java | 176 -
.../TransactionStateCacheSupplier.java | 54 -
.../cask/tephra/coprocessor/package-info.java | 22 -
.../distributed/AbstractClientProvider.java | 218 -
.../distributed/CloseableThriftClient.java | 44 -
.../co/cask/tephra/distributed/ElasticPool.java | 163 -
.../distributed/PooledClientProvider.java | 133 -
.../co/cask/tephra/distributed/RetryNTimes.java | 73 -
.../cask/tephra/distributed/RetryStrategy.java | 42 -
.../distributed/RetryStrategyProvider.java | 40 -
.../tephra/distributed/RetryWithBackoff.java | 108 -
.../distributed/SingleUseClientProvider.java | 63 -
.../distributed/ThreadLocalClientProvider.java | 72 -
.../distributed/ThriftClientProvider.java | 67 -
.../distributed/TransactionConverterUtils.java | 91 -
.../tephra/distributed/TransactionService.java | 156 -
.../distributed/TransactionServiceClient.java | 473 -
.../TransactionServiceThriftClient.java | 244 -
.../TransactionServiceThriftHandler.java | 184 -
.../cask/tephra/distributed/package-info.java | 23 -
.../tephra/distributed/thrift/TBoolean.java | 402 -
.../thrift/TInvalidTruncateTimeException.java | 394 -
.../tephra/distributed/thrift/TTransaction.java | 1364 --
...ransactionCouldNotTakeSnapshotException.java | 404 -
.../TTransactionNotInProgressException.java | 404 -
.../distributed/thrift/TTransactionServer.java | 11072 -----------------
.../distributed/thrift/TTransactionType.java | 63 -
.../distributed/thrift/TVisibilityLevel.java | 62 -
.../tephra/inmemory/DetachedTxSystemClient.java | 143 -
.../inmemory/InMemoryTransactionService.java | 132 -
.../tephra/inmemory/InMemoryTxSystemClient.java | 136 -
.../tephra/inmemory/MinimalTxSystemClient.java | 112 -
.../co/cask/tephra/inmemory/package-info.java | 22 -
.../tephra/metrics/DefaultMetricsCollector.java | 117 -
.../cask/tephra/metrics/MetricsCollector.java | 54 -
.../cask/tephra/metrics/TxMetricsCollector.java | 66 -
.../main/java/co/cask/tephra/package-info.java | 22 -
.../tephra/persist/AbstractTransactionLog.java | 251 -
.../AbstractTransactionStateStorage.java | 43 -
.../cask/tephra/persist/CommitMarkerCodec.java | 118 -
.../cask/tephra/persist/HDFSTransactionLog.java | 147 -
.../HDFSTransactionLogReaderSupplier.java | 58 -
.../persist/HDFSTransactionLogReaderV1.java | 72 -
.../persist/HDFSTransactionLogReaderV2.java | 114 -
.../persist/HDFSTransactionStateStorage.java | 492 -
.../java/co/cask/tephra/persist/HDFSUtil.java | 212 -
.../tephra/persist/LocalFileTransactionLog.java | 135 -
.../LocalFileTransactionStateStorage.java | 346 -
.../persist/NoOpTransactionStateStorage.java | 152 -
.../co/cask/tephra/persist/TransactionEdit.java | 360 -
.../tephra/persist/TransactionEditCodecs.java | 313 -
.../co/cask/tephra/persist/TransactionLog.java | 40 -
.../tephra/persist/TransactionLogReader.java | 42 -
.../tephra/persist/TransactionLogWriter.java | 52 -
.../tephra/persist/TransactionSnapshot.java | 199 -
.../tephra/persist/TransactionStateStorage.java | 103 -
.../persist/TransactionVisibilityState.java | 63 -
.../co/cask/tephra/persist/package-info.java | 22 -
.../co/cask/tephra/rpc/RPCServiceHandler.java | 28 -
.../co/cask/tephra/rpc/ThriftRPCServer.java | 277 -
.../java/co/cask/tephra/rpc/package-info.java | 21 -
.../co/cask/tephra/runtime/ConfigModule.java | 39 -
.../cask/tephra/runtime/DiscoveryModules.java | 110 -
.../tephra/runtime/TransactionClientModule.java | 79 -
.../runtime/TransactionDistributedModule.java | 57 -
.../runtime/TransactionInMemoryModule.java | 58 -
.../tephra/runtime/TransactionLocalModule.java | 57 -
.../cask/tephra/runtime/TransactionModules.java | 41 -
.../TransactionStateStorageProvider.java | 55 -
.../java/co/cask/tephra/runtime/ZKModule.java | 69 -
.../co/cask/tephra/snapshot/BinaryDecoder.java | 110 -
.../co/cask/tephra/snapshot/BinaryEncoder.java | 89 -
.../tephra/snapshot/DefaultSnapshotCodec.java | 208 -
.../co/cask/tephra/snapshot/SnapshotCodec.java | 58 -
.../tephra/snapshot/SnapshotCodecProvider.java | 155 -
.../cask/tephra/snapshot/SnapshotCodecV2.java | 82 -
.../cask/tephra/snapshot/SnapshotCodecV3.java | 34 -
.../cask/tephra/snapshot/SnapshotCodecV4.java | 100 -
.../co/cask/tephra/snapshot/package-info.java | 22 -
.../cask/tephra/util/ConfigurationFactory.java | 75 -
.../cask/tephra/util/ConfigurationProvider.java | 32 -
.../java/co/cask/tephra/util/HBaseVersion.java | 207 -
.../util/HBaseVersionSpecificFactory.java | 74 -
.../main/java/co/cask/tephra/util/TxUtils.java | 152 -
.../tephra/visibility/DefaultFenceWait.java | 63 -
.../co/cask/tephra/visibility/FenceWait.java | 46 -
.../co/cask/tephra/visibility/ReadFence.java | 79 -
.../cask/tephra/visibility/VisibilityFence.java | 140 -
.../co/cask/tephra/visibility/WriteFence.java | 87 -
.../co/cask/tephra/zookeeper/BasicACLData.java | 49 -
.../tephra/zookeeper/BasicNodeChildren.java | 67 -
.../co/cask/tephra/zookeeper/BasicNodeData.java | 68 -
.../tephra/zookeeper/TephraZKClientService.java | 627 -
.../tephra/AbstractTransactionAwareTable.java | 228 +
.../tephra/AbstractTransactionExecutor.java | 129 +
.../main/java/org/apache/tephra/ChangeId.java | 77 +
.../tephra/DefaultTransactionExecutor.java | 147 +
.../tephra/InvalidTruncateTimeException.java | 29 +
.../java/org/apache/tephra/NoRetryStrategy.java | 33 +
.../apache/tephra/RetryOnConflictStrategy.java | 41 +
.../java/org/apache/tephra/RetryStrategies.java | 40 +
.../java/org/apache/tephra/RetryStrategy.java | 34 +
.../org/apache/tephra/TransactionAdmin.java | 158 +
.../org/apache/tephra/TransactionCodec.java | 57 +
.../org/apache/tephra/TransactionContext.java | 323 +
...ransactionCouldNotTakeSnapshotException.java | 28 +
.../org/apache/tephra/TransactionExecutor.java | 141 +
.../tephra/TransactionExecutorFactory.java | 28 +
.../org/apache/tephra/TransactionManager.java | 1398 +++
.../apache/tephra/TransactionServiceMain.java | 149 +
.../apache/tephra/TransactionSystemClient.java | 142 +
.../java/org/apache/tephra/TxConstants.java | 343 +
.../coprocessor/TransactionStateCache.java | 176 +
.../TransactionStateCacheSupplier.java | 54 +
.../apache/tephra/coprocessor/package-info.java | 22 +
.../distributed/AbstractClientProvider.java | 218 +
.../distributed/CloseableThriftClient.java | 44 +
.../apache/tephra/distributed/ElasticPool.java | 163 +
.../distributed/PooledClientProvider.java | 133 +
.../apache/tephra/distributed/RetryNTimes.java | 73 +
.../tephra/distributed/RetryStrategy.java | 42 +
.../distributed/RetryStrategyProvider.java | 40 +
.../tephra/distributed/RetryWithBackoff.java | 108 +
.../distributed/SingleUseClientProvider.java | 63 +
.../distributed/ThreadLocalClientProvider.java | 72 +
.../distributed/ThriftClientProvider.java | 67 +
.../distributed/TransactionConverterUtils.java | 91 +
.../tephra/distributed/TransactionService.java | 156 +
.../distributed/TransactionServiceClient.java | 473 +
.../TransactionServiceThriftClient.java | 244 +
.../TransactionServiceThriftHandler.java | 184 +
.../apache/tephra/distributed/package-info.java | 23 +
.../tephra/distributed/thrift/TBoolean.java | 392 +
.../thrift/TInvalidTruncateTimeException.java | 394 +
.../tephra/distributed/thrift/TTransaction.java | 1364 ++
...ransactionCouldNotTakeSnapshotException.java | 394 +
.../TTransactionNotInProgressException.java | 394 +
.../distributed/thrift/TTransactionServer.java | 11072 +++++++++++++++++
.../distributed/thrift/TTransactionType.java | 59 +
.../distributed/thrift/TVisibilityLevel.java | 62 +
.../tephra/inmemory/DetachedTxSystemClient.java | 143 +
.../inmemory/InMemoryTransactionService.java | 132 +
.../tephra/inmemory/InMemoryTxSystemClient.java | 136 +
.../tephra/inmemory/MinimalTxSystemClient.java | 112 +
.../apache/tephra/inmemory/package-info.java | 22 +
.../tephra/metrics/DefaultMetricsCollector.java | 117 +
.../apache/tephra/metrics/MetricsCollector.java | 53 +
.../tephra/metrics/TxMetricsCollector.java | 66 +
.../java/org/apache/tephra/package-info.java | 22 +
.../tephra/persist/AbstractTransactionLog.java | 251 +
.../AbstractTransactionStateStorage.java | 43 +
.../tephra/persist/CommitMarkerCodec.java | 118 +
.../tephra/persist/HDFSTransactionLog.java | 147 +
.../HDFSTransactionLogReaderSupplier.java | 58 +
.../persist/HDFSTransactionLogReaderV1.java | 72 +
.../persist/HDFSTransactionLogReaderV2.java | 114 +
.../persist/HDFSTransactionStateStorage.java | 492 +
.../org/apache/tephra/persist/HDFSUtil.java | 212 +
.../tephra/persist/LocalFileTransactionLog.java | 135 +
.../LocalFileTransactionStateStorage.java | 346 +
.../persist/NoOpTransactionStateStorage.java | 152 +
.../apache/tephra/persist/TransactionEdit.java | 360 +
.../tephra/persist/TransactionEditCodecs.java | 313 +
.../apache/tephra/persist/TransactionLog.java | 40 +
.../tephra/persist/TransactionLogReader.java | 42 +
.../tephra/persist/TransactionLogWriter.java | 52 +
.../tephra/persist/TransactionSnapshot.java | 199 +
.../tephra/persist/TransactionStateStorage.java | 103 +
.../persist/TransactionVisibilityState.java | 63 +
.../org/apache/tephra/persist/package-info.java | 22 +
.../apache/tephra/rpc/RPCServiceHandler.java | 28 +
.../org/apache/tephra/rpc/ThriftRPCServer.java | 277 +
.../org/apache/tephra/rpc/package-info.java | 21 +
.../org/apache/tephra/runtime/ConfigModule.java | 39 +
.../apache/tephra/runtime/DiscoveryModules.java | 110 +
.../tephra/runtime/TransactionClientModule.java | 79 +
.../runtime/TransactionDistributedModule.java | 57 +
.../runtime/TransactionInMemoryModule.java | 57 +
.../tephra/runtime/TransactionLocalModule.java | 57 +
.../tephra/runtime/TransactionModules.java | 41 +
.../TransactionStateStorageProvider.java | 55 +
.../org/apache/tephra/runtime/ZKModule.java | 69 +
.../apache/tephra/snapshot/BinaryDecoder.java | 110 +
.../apache/tephra/snapshot/BinaryEncoder.java | 89 +
.../tephra/snapshot/DefaultSnapshotCodec.java | 208 +
.../apache/tephra/snapshot/SnapshotCodec.java | 58 +
.../tephra/snapshot/SnapshotCodecProvider.java | 155 +
.../apache/tephra/snapshot/SnapshotCodecV2.java | 83 +
.../apache/tephra/snapshot/SnapshotCodecV3.java | 36 +
.../apache/tephra/snapshot/SnapshotCodecV4.java | 101 +
.../apache/tephra/snapshot/package-info.java | 22 +
.../tephra/util/ConfigurationFactory.java | 75 +
.../tephra/util/ConfigurationProvider.java | 32 +
.../org/apache/tephra/util/HBaseVersion.java | 207 +
.../util/HBaseVersionSpecificFactory.java | 74 +
.../java/org/apache/tephra/util/TxUtils.java | 152 +
.../tephra/visibility/DefaultFenceWait.java | 63 +
.../org/apache/tephra/visibility/FenceWait.java | 46 +
.../org/apache/tephra/visibility/ReadFence.java | 79 +
.../tephra/visibility/VisibilityFence.java | 140 +
.../apache/tephra/visibility/WriteFence.java | 87 +
.../apache/tephra/zookeeper/BasicACLData.java | 49 +
.../tephra/zookeeper/BasicNodeChildren.java | 67 +
.../apache/tephra/zookeeper/BasicNodeData.java | 68 +
.../tephra/zookeeper/TephraZKClientService.java | 626 +
tephra-core/src/main/thrift/transaction.thrift | 2 +-
.../tephra/ThriftTransactionSystemTest.java | 119 -
.../co/cask/tephra/TransactionAdminTest.java | 179 -
.../co/cask/tephra/TransactionContextTest.java | 676 -
.../co/cask/tephra/TransactionExecutorTest.java | 590 -
.../co/cask/tephra/TransactionManagerTest.java | 346 -
.../cask/tephra/TransactionServiceMainTest.java | 77 -
.../co/cask/tephra/TransactionSystemTest.java | 320 -
.../java/co/cask/tephra/TransactionTest.java | 159 -
.../tephra/distributed/ElasticPoolTest.java | 130 -
.../distributed/PooledClientProviderTest.java | 187 -
.../ThriftTransactionServerTest.java | 238 -
...AbstractTransactionVisibilityFilterTest.java | 56 -
.../AbstractTransactionStateStorageTest.java | 555 -
.../tephra/persist/CommitMarkerCodecTest.java | 168 -
.../tephra/persist/HDFSTransactionLogTest.java | 198 -
.../HDFSTransactionStateStorageTest.java | 72 -
.../InMemoryTransactionStateStorage.java | 200 -
.../LocalTransactionStateStorageTest.java | 220 -
.../tephra/persist/TransactionEditTest.java | 124 -
.../cask/tephra/snapshot/SnapshotCodecTest.java | 368 -
.../util/AbstractConfigurationProviderTest.java | 47 -
.../co/cask/tephra/util/HBaseVersionTest.java | 110 -
.../cask/tephra/util/TransactionEditUtil.java | 86 -
.../java/co/cask/tephra/util/TxUtilsTest.java | 35 -
.../tephra/visibility/VisibilityFenceTest.java | 351 -
.../tephra/ThriftTransactionSystemTest.java | 119 +
.../org/apache/tephra/TransactionAdminTest.java | 179 +
.../apache/tephra/TransactionContextTest.java | 676 +
.../apache/tephra/TransactionExecutorTest.java | 590 +
.../apache/tephra/TransactionManagerTest.java | 346 +
.../tephra/TransactionServiceMainTest.java | 77 +
.../apache/tephra/TransactionSystemTest.java | 320 +
.../java/org/apache/tephra/TransactionTest.java | 159 +
.../tephra/distributed/ElasticPoolTest.java | 130 +
.../distributed/PooledClientProviderTest.java | 187 +
.../ThriftTransactionServerTest.java | 238 +
...AbstractTransactionVisibilityFilterTest.java | 56 +
.../AbstractTransactionStateStorageTest.java | 555 +
.../tephra/persist/CommitMarkerCodecTest.java | 168 +
.../tephra/persist/HDFSTransactionLogTest.java | 198 +
.../HDFSTransactionStateStorageTest.java | 72 +
.../InMemoryTransactionStateStorage.java | 200 +
.../LocalTransactionStateStorageTest.java | 220 +
.../tephra/persist/TransactionEditTest.java | 124 +
.../tephra/snapshot/SnapshotCodecTest.java | 368 +
.../util/AbstractConfigurationProviderTest.java | 47 +
.../apache/tephra/util/HBaseVersionTest.java | 110 +
.../apache/tephra/util/TransactionEditUtil.java | 86 +
.../org/apache/tephra/util/TxUtilsTest.java | 35 +
.../tephra/visibility/VisibilityFenceTest.java | 351 +
.../co/cask/tephra/examples/BalanceBooks.java | 357 -
.../co/cask/tephra/examples/package-info.java | 40 -
.../apache/tephra/examples/BalanceBooks.java | 357 +
.../apache/tephra/examples/package-info.java | 40 +
.../cask/tephra/examples/BalanceBooksTest.java | 131 -
.../tephra/examples/BalanceBooksTest.java | 131 +
.../hbase96/HBase96ConfigurationProvider.java | 38 -
.../tephra/hbase96/SecondaryIndexTable.java | 170 -
.../tephra/hbase96/TransactionAwareHTable.java | 615 -
.../hbase96/coprocessor/CellSkipFilter.java | 134 -
.../hbase96/coprocessor/TransactionFilters.java | 62 -
.../coprocessor/TransactionProcessor.java | 349 -
.../TransactionVisibilityFilter.java | 313 -
.../hbase96/HBase96ConfigurationProvider.java | 38 +
.../tephra/hbase96/SecondaryIndexTable.java | 170 +
.../tephra/hbase96/TransactionAwareHTable.java | 615 +
.../hbase96/coprocessor/CellSkipFilter.java | 134 +
.../hbase96/coprocessor/TransactionFilters.java | 62 +
.../coprocessor/TransactionProcessor.java | 349 +
.../TransactionVisibilityFilter.java | 313 +
.../HBase96ConfigurationProviderTest.java | 32 -
.../hbase96/TransactionAwareHTableTest.java | 1555 ---
.../hbase96/coprocessor/CellSkipFilterTest.java | 123 -
.../coprocessor/TransactionProcessorTest.java | 771 --
.../TransactionVisibilityFilterTest.java | 374 -
.../HBase96ConfigurationProviderTest.java | 32 +
.../hbase96/TransactionAwareHTableTest.java | 1555 +++
.../hbase96/coprocessor/CellSkipFilterTest.java | 123 +
.../coprocessor/TransactionProcessorTest.java | 771 ++
.../TransactionVisibilityFilterTest.java | 374 +
.../hbase98/HBase98ConfigurationProvider.java | 38 -
.../tephra/hbase98/SecondaryIndexTable.java | 170 -
.../tephra/hbase98/TransactionAwareHTable.java | 642 -
.../hbase98/coprocessor/CellSkipFilter.java | 134 -
.../hbase98/coprocessor/TransactionFilters.java | 62 -
.../coprocessor/TransactionProcessor.java | 349 -
.../TransactionVisibilityFilter.java | 313 -
.../hbase98/HBase98ConfigurationProvider.java | 38 +
.../tephra/hbase98/SecondaryIndexTable.java | 170 +
.../tephra/hbase98/TransactionAwareHTable.java | 642 +
.../hbase98/coprocessor/CellSkipFilter.java | 134 +
.../hbase98/coprocessor/TransactionFilters.java | 62 +
.../coprocessor/TransactionProcessor.java | 349 +
.../TransactionVisibilityFilter.java | 313 +
.../HBase98ConfigurationProviderTest.java | 32 -
.../hbase98/TransactionAwareHTableTest.java | 1552 ---
.../hbase98/coprocessor/CellSkipFilterTest.java | 123 -
.../coprocessor/TransactionProcessorTest.java | 814 --
.../TransactionVisibilityFilterTest.java | 373 -
.../HBase98ConfigurationProviderTest.java | 32 +
.../hbase98/TransactionAwareHTableTest.java | 1552 +++
.../hbase98/coprocessor/CellSkipFilterTest.java | 123 +
.../coprocessor/TransactionProcessorTest.java | 814 ++
.../TransactionVisibilityFilterTest.java | 373 +
.../HBase10ConfigurationProvider.java | 38 -
.../tephra/hbase10cdh/SecondaryIndexTable.java | 170 -
.../hbase10cdh/TransactionAwareHTable.java | 674 -
.../hbase10cdh/coprocessor/CellSkipFilter.java | 138 -
.../coprocessor/TransactionFilters.java | 62 -
.../coprocessor/TransactionProcessor.java | 349 -
.../TransactionVisibilityFilter.java | 313 -
.../HBase10ConfigurationProvider.java | 38 +
.../tephra/hbase10cdh/SecondaryIndexTable.java | 170 +
.../hbase10cdh/TransactionAwareHTable.java | 674 +
.../hbase10cdh/coprocessor/CellSkipFilter.java | 138 +
.../coprocessor/TransactionFilters.java | 62 +
.../coprocessor/TransactionProcessor.java | 349 +
.../TransactionVisibilityFilter.java | 313 +
.../HBase10ConfigurationProviderTest.java | 32 -
.../hbase10cdh/TransactionAwareHTableTest.java | 1552 ---
.../coprocessor/CellSkipFilterTest.java | 123 -
.../coprocessor/TransactionProcessorTest.java | 620 -
.../TransactionVisibilityFilterTest.java | 373 -
.../HBase10ConfigurationProviderTest.java | 32 +
.../hbase10cdh/TransactionAwareHTableTest.java | 1552 +++
.../coprocessor/CellSkipFilterTest.java | 123 +
.../coprocessor/TransactionProcessorTest.java | 620 +
.../TransactionVisibilityFilterTest.java | 373 +
.../hbase10/HBase10ConfigurationProvider.java | 38 -
.../tephra/hbase10/SecondaryIndexTable.java | 170 -
.../tephra/hbase10/TransactionAwareHTable.java | 674 -
.../hbase10/coprocessor/CellSkipFilter.java | 138 -
.../hbase10/coprocessor/TransactionFilters.java | 62 -
.../coprocessor/TransactionProcessor.java | 349 -
.../TransactionVisibilityFilter.java | 313 -
.../hbase10/HBase10ConfigurationProvider.java | 38 +
.../tephra/hbase10/SecondaryIndexTable.java | 170 +
.../tephra/hbase10/TransactionAwareHTable.java | 674 +
.../hbase10/coprocessor/CellSkipFilter.java | 138 +
.../hbase10/coprocessor/TransactionFilters.java | 62 +
.../coprocessor/TransactionProcessor.java | 349 +
.../TransactionVisibilityFilter.java | 313 +
.../HBase10ConfigurationProviderTest.java | 32 -
.../hbase10/TransactionAwareHTableTest.java | 1552 ---
.../hbase10/coprocessor/CellSkipFilterTest.java | 123 -
.../coprocessor/TransactionProcessorTest.java | 620 -
.../TransactionVisibilityFilterTest.java | 373 -
.../HBase10ConfigurationProviderTest.java | 32 +
.../hbase10/TransactionAwareHTableTest.java | 1552 +++
.../hbase10/coprocessor/CellSkipFilterTest.java | 123 +
.../coprocessor/TransactionProcessorTest.java | 620 +
.../TransactionVisibilityFilterTest.java | 373 +
.../hbase11/HBase11ConfigurationProvider.java | 38 -
.../tephra/hbase11/SecondaryIndexTable.java | 178 -
.../tephra/hbase11/TransactionAwareHTable.java | 674 -
.../hbase11/coprocessor/CellSkipFilter.java | 138 -
.../hbase11/coprocessor/TransactionFilters.java | 62 -
.../coprocessor/TransactionProcessor.java | 348 -
.../TransactionVisibilityFilter.java | 308 -
.../hbase11/HBase11ConfigurationProvider.java | 38 +
.../tephra/hbase11/SecondaryIndexTable.java | 178 +
.../tephra/hbase11/TransactionAwareHTable.java | 674 +
.../hbase11/coprocessor/CellSkipFilter.java | 138 +
.../hbase11/coprocessor/TransactionFilters.java | 62 +
.../coprocessor/TransactionProcessor.java | 348 +
.../TransactionVisibilityFilter.java | 308 +
.../HBase11ConfigurationProviderTest.java | 32 -
.../hbase11/TransactionAwareHTableTest.java | 1546 ---
.../hbase11/coprocessor/CellSkipFilterTest.java | 123 -
.../coprocessor/TransactionProcessorTest.java | 622 -
.../TransactionVisibilityFilterTest.java | 374 -
.../HBase11ConfigurationProviderTest.java | 32 +
.../hbase11/TransactionAwareHTableTest.java | 1546 +++
.../hbase11/coprocessor/CellSkipFilterTest.java | 123 +
.../coprocessor/TransactionProcessorTest.java | 622 +
.../TransactionVisibilityFilterTest.java | 374 +
418 files changed, 57024 insertions(+), 57057 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/README.rst
----------------------------------------------------------------------
diff --git a/README.rst b/README.rst
index af2bde1..e1221b4 100644
--- a/README.rst
+++ b/README.rst
@@ -142,12 +142,12 @@ For any client applications, add the following dependencies to any Apache Maven
build system's equivalent configuration), in order to make use of Tephra classes::
<dependency>
- <groupId>co.cask.tephra</groupId>
+ <groupId>org.apache.tephra</groupId>
<artifactId>tephra-api</artifactId>
<version>0.7.1</version>
</dependency>
<dependency>
- <groupId>co.cask.tephra</groupId>
+ <groupId>org.apache.tephra</groupId>
<artifactId>tephra-core</artifactId>
<version>0.7.1</version>
</dependency>
@@ -158,7 +158,7 @@ appropriate HBase compatibility library.
For HBase 0.96.x::
<dependency>
- <groupId>co.cask.tephra</groupId>
+ <groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-0.96</artifactId>
<version>0.7.1</version>
</dependency>
@@ -166,7 +166,7 @@ For HBase 0.96.x::
For HBase 0.98.x::
<dependency>
- <groupId>co.cask.tephra</groupId>
+ <groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-0.98</artifactId>
<version>0.7.1</version>
</dependency>
@@ -174,7 +174,7 @@ For HBase 0.98.x::
For HBase 1.0.x::
<dependency>
- <groupId>co.cask.tephra</groupId>
+ <groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.0</artifactId>
<version>0.7.1</version>
</dependency>
@@ -183,7 +183,7 @@ If you are running the CDH 5.4, 5.5, or 5.6 version of HBase 1.0.x (this version
with Apache HBase 1.0.x)::
<dependency>
- <groupId>co.cask.tephra</groupId>
+ <groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.0-cdh</artifactId>
<version>0.7.1</version>
</dependency>
@@ -191,7 +191,7 @@ with Apache HBase 1.0.x)::
For HBase 1.1.x or CDH 5.7 version of HBase 1.2.x::
<dependency>
- <groupId>co.cask.tephra</groupId>
+ <groupId>org.apache.tephra</groupId>
<artifactId>tephra-hbase-compat-1.1</artifactId>
<version>0.7.1</version>
</dependency>
@@ -302,35 +302,35 @@ For HBase 0.96.x::
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>co.cask.tephra.hbase96.coprocessor.TransactionProcessor</value>
+ <value>org.apache.tephra.hbase96.coprocessor.TransactionProcessor</value>
</property>
For HBase 0.98.x::
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>co.cask.tephra.hbase98.coprocessor.TransactionProcessor</value>
+ <value>org.apache.tephra.hbase98.coprocessor.TransactionProcessor</value>
</property>
For HBase 1.0.x::
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>co.cask.tephra.hbase10.coprocessor.TransactionProcessor</value>
+ <value>org.apache.tephra.hbase10.coprocessor.TransactionProcessor</value>
</property>
For the CDH 5.4, 5.5, or 5.6 version of HBase 1.0.x::
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>co.cask.tephra.hbase10cdh.coprocessor.TransactionProcessor</value>
+ <value>org.apache.tephra.hbase10cdh.coprocessor.TransactionProcessor</value>
</property>
For HBase 1.1.x or CDH 5.7 version of HBase 1.2.x::
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>co.cask.tephra.hbase11.coprocessor.TransactionProcessor</value>
+ <value>org.apache.tephra.hbase11.coprocessor.TransactionProcessor</value>
</property>
You may configure the ``TransactionProcessor`` to be loaded only on HBase tables that you will
@@ -622,7 +622,7 @@ Known Issues and Limitations
changes, its transaction ID is added to a list of excluded transactions. Data from invalidated
transactions will be dropped by the ``TransactionProcessor`` coprocessor on HBase region flush
and compaction operations. Currently, however, transaction IDs can only be manually removed
- from the list of excluded transaction IDs, using the ``co.cask.tephra.TransactionAdmin`` tool.
+ from the list of excluded transaction IDs, using the ``org.apache.tephra.TransactionAdmin`` tool.
How to Contribute
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/bin/tephra
----------------------------------------------------------------------
diff --git a/bin/tephra b/bin/tephra
index 834117b..303d189 100644
--- a/bin/tephra
+++ b/bin/tephra
@@ -169,7 +169,7 @@ start() {
echo "`date` Starting $APP service on `hostname`" >> $loglog
echo "`ulimit -a`" >> $loglog 2>&1
- export MAIN_CLASS="co.cask.tephra.TransactionServiceMain"
+ export MAIN_CLASS="org.apache.tephra.TransactionServiceMain"
echo "Running class $MAIN_CLASS"
echo "Command: " "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX $MAIN_CLASS >>$loglog
nohup nice -n $NICENESS "$JAVA" $OPTS -cp $CLASSPATH $JAVA_HEAPMAX ${MAIN_CLASS} </dev/null >>$loglog 2>&1 &
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/conf/logback.xml
----------------------------------------------------------------------
diff --git a/conf/logback.xml b/conf/logback.xml
index 6a0f5d5..0bc5998 100644
--- a/conf/logback.xml
+++ b/conf/logback.xml
@@ -30,7 +30,7 @@
<logger name="org.apache.hadoop" level="WARN"/>
<logger name="org.apache.hive" level="WARN"/>
<logger name="org.quartz.core" level="WARN"/>
- <logger name="co.cask.tephra" level="INFO"/>
+ <logger name="org.apache.tephra" level="INFO"/>
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/co/cask/tephra/Transaction.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/co/cask/tephra/Transaction.java b/tephra-api/src/main/java/co/cask/tephra/Transaction.java
deleted file mode 100644
index 753a064..0000000
--- a/tephra-api/src/main/java/co/cask/tephra/Transaction.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import java.util.Arrays;
-
-/**
- * Transaction details
- */
-// NOTE: this class should have minimal dependencies as it is used in HBase CPs and other places where minimal classes
-// are available
-public class Transaction {
- private final long readPointer;
- private final long txId;
- private final long writePointer;
- private final long[] invalids;
- private final long[] inProgress;
- private final long firstShortInProgress;
- private final TransactionType type;
- private final long[] checkpointWritePointers;
-
- private VisibilityLevel visibilityLevel = VisibilityLevel.SNAPSHOT;
-
- private static final long[] NO_EXCLUDES = { };
- public static final long NO_TX_IN_PROGRESS = Long.MAX_VALUE;
-
- public static final Transaction ALL_VISIBLE_LATEST =
- new Transaction(Long.MAX_VALUE, Long.MAX_VALUE, NO_EXCLUDES, NO_EXCLUDES, NO_TX_IN_PROGRESS, TransactionType.SHORT);
-
- /**
- * Defines the possible visibility levels for read operations.
- *
- * <p>
- * <ul>
- * <li><code>SNAPSHOT</code> - uses the transaction's read snapshot, plus includes all write pointers from the
- * current transaction</li>
- * <li><code>SNAPSHOT_EXCLUDE_CURRENT</code> - uses the transaction's read snapshot, plus includes all write
- * pointers from the current transaction, <strong>except</strong> the current write pointer
- * (see {@link #getWritePointer()})</li>
- * <li><code>SNAPSHOT_ALL</code> - uses the transaction's read snapshot, plus includes all write pointers from the
- * current transaction. This visibility level will lead to raw fetch operations, where all versions
- * (including deletes) visible to current transaction are returned.</li>
- * </ul>
- * </p>
- *
- * <p>The default value used is {@code SNAPSHOT}.</p>
- *
- * @see #setVisibility(VisibilityLevel)
- */
- public enum VisibilityLevel {
- SNAPSHOT,
- SNAPSHOT_EXCLUDE_CURRENT,
- SNAPSHOT_ALL
- }
-
- /**
- * Creates a new short transaction.
- * @param readPointer read pointer for transaction
- * @param txId unique identifier for the transaction
- * @param invalids list of invalid transactions to exclude while reading
- * @param inProgress list of in-progress transactions to exclude while reading
- * @param firstShortInProgress earliest in-progress short transaction
- */
- public Transaction(long readPointer, long txId, long[] invalids, long[] inProgress,
- long firstShortInProgress) {
- this(readPointer, txId, invalids, inProgress, firstShortInProgress, TransactionType.SHORT);
- }
-
- /**
- * Creates a new transaction.
- * @param readPointer read pointer for transaction
- * @param txId unique identifier for the transaction
- * @param invalids list of invalid transactions to exclude while reading
- * @param inProgress list of in-progress transactions to exclude while reading
- * @param firstShortInProgress earliest in-progress short transaction
- * @param type transaction type
- */
- public Transaction(long readPointer, long txId, long[] invalids, long[] inProgress,
- long firstShortInProgress, TransactionType type) {
- this(readPointer, txId, txId, invalids, inProgress, firstShortInProgress, type, new long[0],
- VisibilityLevel.SNAPSHOT);
- }
-
- /**
- * Creates a new transaction.
- * @param readPointer read pointer for transaction
- * @param txId unique identifier for the transaction
- * @param writePointer the current pointer to be used for any writes.
- * For new transactions, this will be the same as {@code txId}. For checkpointed
- * transactions, this will be the most recent write pointer issued.
- * @param invalids list of invalid transactions to exclude while reading
- * @param inProgress list of in-progress transactions to exclude while reading
- * @param firstShortInProgress earliest in-progress short transaction
- * @param type transaction type
- * @param checkpointPointers the list of writer pointers added from checkpoints on the transaction
- * @param visibilityLevel the visibility level to use for transactional reads
- */
- public Transaction(long readPointer, long txId, long writePointer, long[] invalids, long[] inProgress,
- long firstShortInProgress, TransactionType type, long[] checkpointPointers,
- VisibilityLevel visibilityLevel) {
- this.readPointer = readPointer;
- this.txId = txId;
- this.writePointer = writePointer;
- this.invalids = invalids;
- this.inProgress = inProgress;
- this.firstShortInProgress = firstShortInProgress;
- this.type = type;
- this.checkpointWritePointers = checkpointPointers;
- this.visibilityLevel = visibilityLevel;
- }
-
- /**
- * Creates a new transaction for a checkpoint operation, copying all members from the original transaction,
- * with the updated checkpoint write pointers.
- *
- * @param toCopy the original transaction containing the state to copy
- * @param writePointer the new write pointer to use for the transaction
- * @param checkpointPointers the list of write pointers added from checkpoints on the transaction
- */
- public Transaction(Transaction toCopy, long writePointer, long[] checkpointPointers) {
- this(toCopy.getReadPointer(), toCopy.getTransactionId(), writePointer, toCopy.getInvalids(),
- toCopy.getInProgress(), toCopy.getFirstShortInProgress(), toCopy.getType(), checkpointPointers,
- toCopy.getVisibilityLevel());
- }
-
- public long getReadPointer() {
- return readPointer;
- }
-
- /**
- * Returns the initial write pointer assigned to the transaction. This will remain the same for the life of the
- * transaction, and uniquely identifies it with the transaction service. This value should be provided
- * to identify the transaction when calling any transaction lifecycle methods on the transaction service.
- */
- public long getTransactionId() {
- return txId;
- }
-
- /**
- * Returns the write pointer to be used in persisting any changes. After a checkpoint is performed, this will differ
- * from {@link #getTransactionId()}. This method should always be used when setting the timestamp for writes
- * in order to ensure that the correct value is used.
- */
- public long getWritePointer() {
- return writePointer;
- }
-
- public long[] getInvalids() {
- return invalids;
- }
-
- public long[] getInProgress() {
- return inProgress;
- }
-
- public long getFirstInProgress() {
- return inProgress.length == 0 ? NO_TX_IN_PROGRESS : inProgress[0];
- }
-
- public TransactionType getType() {
- return type;
- }
-
- /**
- * @return transaction id {@code X} such that any of the transactions newer than {@code X} may be invisible to this<p>
- * NOTE: the returned tx id can be invalid.
- */
- public long getVisibilityUpperBound() {
- // NOTE: in some cases when we do not provide visibility guarantee, we set readPointer to MAX value, but
- // at same time we don't want that to case cleanup everything as this is used for tx janitor + ttl to see
- // what can be cleaned up. When non-tx mode is implemented better, we should not need this check
- return inProgress.length == 0 ? Math.min(txId - 1, readPointer) : inProgress[0] - 1;
- }
-
- public long getFirstShortInProgress() {
- return firstShortInProgress;
- }
-
- /**
- * Returns true if the given version corresponds to a transaction that was in-progress at the time this transaction
- * started.
- */
- public boolean isInProgress(long version) {
- return Arrays.binarySearch(inProgress, version) >= 0;
- }
-
- /**
- * Returns true if the given version is present in one of the arrays of excluded versions (in-progress and
- * invalid transactions).
- */
- public boolean isExcluded(long version) {
- return Arrays.binarySearch(inProgress, version) >= 0
- || Arrays.binarySearch(invalids, version) >= 0;
- }
-
- /**
- * Returns true if the the given version corresponds to one of the checkpoint versions in the current
- * transaction.
- */
- public boolean isCheckpoint(long version) {
- return Arrays.binarySearch(checkpointWritePointers, version) >= 0;
- }
-
- /**
- * Returns whether or not the given version should be visible to the current transaction. A version will be visible
- * if it was successfully committed prior to the current transaction starting, or was written by the current
- * transaction (using either the current write pointer or the write pointer from a prior checkpoint).
- *
- * @param version the data version to check for visibility
- * @return true if the version is visible, false if it should be hidden (filtered)
- *
- * @see #setVisibility(VisibilityLevel) to control whether the current write pointer is visible.
- */
- public boolean isVisible(long version) {
- // either it was committed before or the change belongs to current tx
- return (version <= getReadPointer() && !isExcluded(version)) ||
- (isCurrentWrite(version) &&
- (visibilityLevel != VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT || writePointer != version));
- }
-
- /**
- * Returns whether the given version was written by the current transaction.
- *
- * @param version the data version
- * @return true if version was written by current transaction, false otherwise.
- */
- public boolean isCurrentWrite(long version) {
- return writePointer == version || txId == version || isCheckpoint(version);
- }
-
- /**
- * Sets the visibility level for read operations.
- */
- public void setVisibility(VisibilityLevel level) {
- this.visibilityLevel = level;
- }
-
- /**
- * Returns the currently set visibility level.
- */
- public VisibilityLevel getVisibilityLevel() {
- return visibilityLevel;
- }
-
- public boolean hasExcludes() {
- return invalids.length > 0 || inProgress.length > 0;
- }
-
-
- public int excludesSize() {
- return invalids.length + inProgress.length;
- }
-
- /**
- * Returns any prior write pointers used in the current transaction. A new write pointer is issued when the
- * {@code TransactionContext.checkpoint(Transaction)} operation is called, and the prior write pointer is added
- * to the array of checkpoint write pointers.
-` */
- public long[] getCheckpointWritePointers() {
- return checkpointWritePointers;
- }
-
- @Override
- public String toString() {
- return new StringBuilder(100)
- .append(Transaction.class.getSimpleName())
- .append('{')
- .append("readPointer: ").append(readPointer)
- .append(", transactionId: ").append(txId)
- .append(", writePointer: ").append(writePointer)
- .append(", invalids: ").append(Arrays.toString(invalids))
- .append(", inProgress: ").append(Arrays.toString(inProgress))
- .append(", firstShortInProgress: ").append(firstShortInProgress)
- .append(", type: ").append(type)
- .append(", checkpointWritePointers: ").append(Arrays.toString(checkpointWritePointers))
- .append(", visibilityLevel: ").append(visibilityLevel)
- .append('}')
- .toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/co/cask/tephra/TransactionAware.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/co/cask/tephra/TransactionAware.java b/tephra-api/src/main/java/co/cask/tephra/TransactionAware.java
deleted file mode 100644
index 524d841..0000000
--- a/tephra-api/src/main/java/co/cask/tephra/TransactionAware.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import java.util.Collection;
-
-/**
- * Interface to be implemented by a component that interacts with transaction logic.
- * <p/>
- * The client code that uses transaction logic looks like this:
- * <pre>
- * TransactionAware dataSet = // ... // dataSet is one example of component that interacts with tx logic
- *
- * Transaction tx = txClient.start();
- * dataSet.startTx(tx); // notifying about new transaction
- * dataSet.write(...);
- * // ... do other operations on dataSet
- * Collection<byte[]> changes = dataSet.getTxChanges();
- * boolean rollback = true;
- * if (txClient.canCommit(changes)) { // checking conflicts before commit, if none, commit tx
- * if (dataSet.commitTx()) { // try persisting changes
- * if (txClient.commit(tx)) { // if OK, make tx visible; if not - tx stays invisible to others
- * dataSet.postTxCommit(); // notifying dataset about tx commit success via callback
- * rollback = false;
- * }
- * }
- * }
- *
- * if (rollback) { // if there are conflicts (or cannot commit), try rollback changes
- * if (dataSet.rollbackTx()) { // try undo changes
- * txClient.abort(tx); // if OK, make tx visible; if not - tx stays invisible to others
- * }
- * }
- *
- * </pre>
- */
-// todo: use custom exception class?
-// todo: review exception handling where it is used
-// todo: add flush()? nah - flush is same as commitTx() actually
-// todo: add onCommitted() - so that e.g. hbase table can do *actual* deletes at this point
-public interface TransactionAware {
- /**
- * Called when new transaction has started. This may reset any state which has been left behind by the previous
- * transaction.
- *
- * @param tx transaction info
- */
- // todo: rename to onTxStart()
- void startTx(Transaction tx);
-
- /**
- * Called when the state of the current transaction has been updated. This should replace any reference to the
- * current {@link Transaction} held by this {@code TransactionAware}, but should <strong>not</strong> reset
- * any state (such as the write change sets) that is currently maintained.
- *
- * @param tx the updated transaction
- */
- void updateTx(Transaction tx);
-
- /**
- * @return changes made by current transaction to be used for conflicts detection before commit.
- */
- Collection<byte[]> getTxChanges();
-
- /**
- * Called before transaction has been committed.
- * Can be used e.g. to flush changes cached in-memory to persistent store.
- * @return true if transaction can be committed, otherwise false.
- */
- // todo: rename to beforeTxCommit()
- boolean commitTx() throws Exception;
-
- /**
- * Called after transaction has been committed.
- * Can be used e.g. evict entries from a cache etc. Because this is called after the transaction is committed,
- * the success or failure of the transaction cannot depend on it. Hence this method returns nothing and it is not
- * expected to throw exceptions.
- * @throws RuntimeException in case of serious failure that should not be ignored.
- */
- void postTxCommit();
-
- /**
- * Called during transaction rollback (for whatever reason: conflicts, errors, etc.).
- * @return true if all changes made during transaction were rolled back, false otherwise (e.g. if more cleanup needed
- * or changes cannot be undone). True also means that this transaction can be made visible to others without
- * breaking consistency of the data: since all changes were undone there's "nothing to see".
- */
- // todo: rename to onTxRollback()
- boolean rollbackTx() throws Exception;
-
- /**
- * Used for error reporting.
- */
- // todo: use toString() instead everywhere
- String getTransactionAwareName();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/co/cask/tephra/TransactionAwares.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/co/cask/tephra/TransactionAwares.java b/tephra-api/src/main/java/co/cask/tephra/TransactionAwares.java
deleted file mode 100644
index 044efc1..0000000
--- a/tephra-api/src/main/java/co/cask/tephra/TransactionAwares.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Utility class that provides handy methods for working with {@link TransactionAware} classes and their instances.
- */
-public final class TransactionAwares {
- private TransactionAwares() {}
-
- /**
- * Create composite transaction aware object that delegates transaction logic to given collection of
- * {@link TransactionAware}s
- * @param transactionAwares collection of {@link TransactionAware}s
- * @return instance of {@link TransactionAware}
- */
- public static TransactionAware of(Collection<TransactionAware> transactionAwares) {
- // this is most common case, trying to optimize
- if (transactionAwares.size() == 1) {
- return transactionAwares.iterator().next();
- }
-
- TransactionAwareCollection result = new TransactionAwareCollection();
- result.addAll(transactionAwares);
-
- return result;
- }
-
- private static class TransactionAwareCollection extends ArrayList<TransactionAware> implements TransactionAware {
-
- @Override
- public void startTx(Transaction tx) {
- for (TransactionAware txAware : this) {
- txAware.startTx(tx);
- }
- }
-
- @Override
- public void updateTx(Transaction tx) {
- for (TransactionAware txAware : this) {
- txAware.updateTx(tx);
- }
- }
-
- @Override
- public Collection<byte[]> getTxChanges() {
- List<byte[]> changes = new ArrayList<byte[]>();
- for (TransactionAware txAware : this) {
- changes.addAll(txAware.getTxChanges());
- }
-
- return changes;
- }
-
- @Override
- public boolean commitTx() throws Exception {
- boolean success = true;
- for (TransactionAware txAware : this) {
- success = success && txAware.commitTx();
- }
- return success;
- }
-
- @Override
- public void postTxCommit() {
- for (TransactionAware txAware : this) {
- txAware.postTxCommit();
- }
- }
-
- @Override
- public boolean rollbackTx() throws Exception {
- boolean success = true;
- for (TransactionAware txAware : this) {
- success = success && txAware.rollbackTx();
- }
- return success;
- }
-
- @Override
- public String getTransactionAwareName() {
- // todo: will go away, see comment at TransactionAware
- StringBuilder sb = new StringBuilder("{");
- for (TransactionAware txAware : this) {
- sb.append(txAware.getTransactionAwareName()).append(",");
- }
- sb.replace(sb.length() - 1, sb.length() - 1, "}");
- return sb.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/co/cask/tephra/TransactionConflictException.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/co/cask/tephra/TransactionConflictException.java b/tephra-api/src/main/java/co/cask/tephra/TransactionConflictException.java
deleted file mode 100644
index 4b0733d..0000000
--- a/tephra-api/src/main/java/co/cask/tephra/TransactionConflictException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Thrown to indicate transaction conflict occurred when trying to commit a transaction.
- */
-public class TransactionConflictException extends TransactionFailureException {
- public TransactionConflictException(String message) {
- super(message);
- }
-
- public TransactionConflictException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/co/cask/tephra/TransactionFailureException.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/co/cask/tephra/TransactionFailureException.java b/tephra-api/src/main/java/co/cask/tephra/TransactionFailureException.java
deleted file mode 100644
index 1ce3071..0000000
--- a/tephra-api/src/main/java/co/cask/tephra/TransactionFailureException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Thrown to indicate execution of a transaction failed.
- */
-public class TransactionFailureException extends Exception {
- public TransactionFailureException(String message) {
- super(message);
- }
-
- public TransactionFailureException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/co/cask/tephra/TransactionNotInProgressException.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/co/cask/tephra/TransactionNotInProgressException.java b/tephra-api/src/main/java/co/cask/tephra/TransactionNotInProgressException.java
deleted file mode 100644
index 814421a..0000000
--- a/tephra-api/src/main/java/co/cask/tephra/TransactionNotInProgressException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Thrown when transaction has timed out.
- */
-public class TransactionNotInProgressException extends TransactionFailureException {
- public TransactionNotInProgressException(String message) {
- super(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/co/cask/tephra/TransactionType.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/co/cask/tephra/TransactionType.java b/tephra-api/src/main/java/co/cask/tephra/TransactionType.java
deleted file mode 100644
index 1235525..0000000
--- a/tephra-api/src/main/java/co/cask/tephra/TransactionType.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-/**
- * Type of transaction.
- */
-public enum TransactionType {
- /**
- * Short transactions detect conflicts during commit.
- */
- SHORT,
- /**
- * Long running transactions do not detect conflicts during commit.
- */
- LONG
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/org/apache/tephra/Transaction.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/org/apache/tephra/Transaction.java b/tephra-api/src/main/java/org/apache/tephra/Transaction.java
new file mode 100644
index 0000000..287423a
--- /dev/null
+++ b/tephra-api/src/main/java/org/apache/tephra/Transaction.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import java.util.Arrays;
+
+/**
+ * Transaction details
+ */
+// NOTE: this class should have minimal dependencies as it is used in HBase CPs and other places where minimal classes
+// are available
+public class Transaction {
+ private final long readPointer;
+ private final long txId;
+ private final long writePointer;
+ private final long[] invalids;
+ private final long[] inProgress;
+ private final long firstShortInProgress;
+ private final TransactionType type;
+ private final long[] checkpointWritePointers;
+
+ private VisibilityLevel visibilityLevel = VisibilityLevel.SNAPSHOT;
+
+ private static final long[] NO_EXCLUDES = { };
+ public static final long NO_TX_IN_PROGRESS = Long.MAX_VALUE;
+
+ public static final Transaction ALL_VISIBLE_LATEST =
+ new Transaction(Long.MAX_VALUE, Long.MAX_VALUE, NO_EXCLUDES, NO_EXCLUDES, NO_TX_IN_PROGRESS, TransactionType.SHORT);
+
+ /**
+ * Defines the possible visibility levels for read operations.
+ *
+ * <p>
+ * <ul>
+ * <li><code>SNAPSHOT</code> - uses the transaction's read snapshot, plus includes all write pointers from the
+ * current transaction</li>
+ * <li><code>SNAPSHOT_EXCLUDE_CURRENT</code> - uses the transaction's read snapshot, plus includes all write
+ * pointers from the current transaction, <strong>except</strong> the current write pointer
+ * (see {@link #getWritePointer()})</li>
+ * <li><code>SNAPSHOT_ALL</code> - uses the transaction's read snapshot, plus includes all write pointers from the
+ * current transaction. This visibility level will lead to raw fetch operations, where all versions
+ * (including deletes) visible to current transaction are returned.</li>
+ * </ul>
+ * </p>
+ *
+ * <p>The default value used is {@code SNAPSHOT}.</p>
+ *
+ * @see #setVisibility(VisibilityLevel)
+ */
+ public enum VisibilityLevel {
+ SNAPSHOT,
+ SNAPSHOT_EXCLUDE_CURRENT,
+ SNAPSHOT_ALL
+ }
+
+ /**
+ * Creates a new short transaction.
+ * @param readPointer read pointer for transaction
+ * @param txId unique identifier for the transaction
+ * @param invalids list of invalid transactions to exclude while reading
+ * @param inProgress list of in-progress transactions to exclude while reading
+ * @param firstShortInProgress earliest in-progress short transaction
+ */
+ public Transaction(long readPointer, long txId, long[] invalids, long[] inProgress,
+ long firstShortInProgress) {
+ this(readPointer, txId, invalids, inProgress, firstShortInProgress, TransactionType.SHORT);
+ }
+
+ /**
+ * Creates a new transaction.
+ * @param readPointer read pointer for transaction
+ * @param txId unique identifier for the transaction
+ * @param invalids list of invalid transactions to exclude while reading
+ * @param inProgress list of in-progress transactions to exclude while reading
+ * @param firstShortInProgress earliest in-progress short transaction
+ * @param type transaction type
+ */
+ public Transaction(long readPointer, long txId, long[] invalids, long[] inProgress,
+ long firstShortInProgress, TransactionType type) {
+ this(readPointer, txId, txId, invalids, inProgress, firstShortInProgress, type, new long[0],
+ VisibilityLevel.SNAPSHOT);
+ }
+
+ /**
+ * Creates a new transaction.
+ * @param readPointer read pointer for transaction
+ * @param txId unique identifier for the transaction
+ * @param writePointer the current pointer to be used for any writes.
+ * For new transactions, this will be the same as {@code txId}. For checkpointed
+ * transactions, this will be the most recent write pointer issued.
+ * @param invalids list of invalid transactions to exclude while reading
+ * @param inProgress list of in-progress transactions to exclude while reading
+ * @param firstShortInProgress earliest in-progress short transaction
+ * @param type transaction type
+ * @param checkpointPointers the list of writer pointers added from checkpoints on the transaction
+ * @param visibilityLevel the visibility level to use for transactional reads
+ */
+ public Transaction(long readPointer, long txId, long writePointer, long[] invalids, long[] inProgress,
+ long firstShortInProgress, TransactionType type, long[] checkpointPointers,
+ VisibilityLevel visibilityLevel) {
+ this.readPointer = readPointer;
+ this.txId = txId;
+ this.writePointer = writePointer;
+ this.invalids = invalids;
+ this.inProgress = inProgress;
+ this.firstShortInProgress = firstShortInProgress;
+ this.type = type;
+ this.checkpointWritePointers = checkpointPointers;
+ this.visibilityLevel = visibilityLevel;
+ }
+
+ /**
+ * Creates a new transaction for a checkpoint operation, copying all members from the original transaction,
+ * with the updated checkpoint write pointers.
+ *
+ * @param toCopy the original transaction containing the state to copy
+ * @param writePointer the new write pointer to use for the transaction
+ * @param checkpointPointers the list of write pointers added from checkpoints on the transaction
+ */
+ public Transaction(Transaction toCopy, long writePointer, long[] checkpointPointers) {
+ this(toCopy.getReadPointer(), toCopy.getTransactionId(), writePointer, toCopy.getInvalids(),
+ toCopy.getInProgress(), toCopy.getFirstShortInProgress(), toCopy.getType(), checkpointPointers,
+ toCopy.getVisibilityLevel());
+ }
+
+ public long getReadPointer() {
+ return readPointer;
+ }
+
+ /**
+ * Returns the initial write pointer assigned to the transaction. This will remain the same for the life of the
+ * transaction, and uniquely identifies it with the transaction service. This value should be provided
+ * to identify the transaction when calling any transaction lifecycle methods on the transaction service.
+ */
+ public long getTransactionId() {
+ return txId;
+ }
+
+ /**
+ * Returns the write pointer to be used in persisting any changes. After a checkpoint is performed, this will differ
+ * from {@link #getTransactionId()}. This method should always be used when setting the timestamp for writes
+ * in order to ensure that the correct value is used.
+ */
+ public long getWritePointer() {
+ return writePointer;
+ }
+
+ public long[] getInvalids() {
+ return invalids;
+ }
+
+ public long[] getInProgress() {
+ return inProgress;
+ }
+
+ public long getFirstInProgress() {
+ return inProgress.length == 0 ? NO_TX_IN_PROGRESS : inProgress[0];
+ }
+
+ public TransactionType getType() {
+ return type;
+ }
+
+ /**
+ * @return transaction id {@code X} such that any of the transactions newer than {@code X} may be invisible to this<p>
+ * NOTE: the returned tx id can be invalid.
+ */
+ public long getVisibilityUpperBound() {
+ // NOTE: in some cases when we do not provide visibility guarantee, we set readPointer to MAX value, but
+ // at same time we don't want that to case cleanup everything as this is used for tx janitor + ttl to see
+ // what can be cleaned up. When non-tx mode is implemented better, we should not need this check
+ return inProgress.length == 0 ? Math.min(txId - 1, readPointer) : inProgress[0] - 1;
+ }
+
+ public long getFirstShortInProgress() {
+ return firstShortInProgress;
+ }
+
+ /**
+ * Returns true if the given version corresponds to a transaction that was in-progress at the time this transaction
+ * started.
+ */
+ public boolean isInProgress(long version) {
+ return Arrays.binarySearch(inProgress, version) >= 0;
+ }
+
+ /**
+ * Returns true if the given version is present in one of the arrays of excluded versions (in-progress and
+ * invalid transactions).
+ */
+ public boolean isExcluded(long version) {
+ return Arrays.binarySearch(inProgress, version) >= 0
+ || Arrays.binarySearch(invalids, version) >= 0;
+ }
+
+ /**
+ * Returns true if the the given version corresponds to one of the checkpoint versions in the current
+ * transaction.
+ */
+ public boolean isCheckpoint(long version) {
+ return Arrays.binarySearch(checkpointWritePointers, version) >= 0;
+ }
+
+ /**
+ * Returns whether or not the given version should be visible to the current transaction. A version will be visible
+ * if it was successfully committed prior to the current transaction starting, or was written by the current
+ * transaction (using either the current write pointer or the write pointer from a prior checkpoint).
+ *
+ * @param version the data version to check for visibility
+ * @return true if the version is visible, false if it should be hidden (filtered)
+ *
+ * @see #setVisibility(VisibilityLevel) to control whether the current write pointer is visible.
+ */
+ public boolean isVisible(long version) {
+ // either it was committed before or the change belongs to current tx
+ return (version <= getReadPointer() && !isExcluded(version)) ||
+ (isCurrentWrite(version) &&
+ (visibilityLevel != VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT || writePointer != version));
+ }
+
+ /**
+ * Returns whether the given version was written by the current transaction.
+ *
+ * @param version the data version
+ * @return true if version was written by current transaction, false otherwise.
+ */
+ public boolean isCurrentWrite(long version) {
+ return writePointer == version || txId == version || isCheckpoint(version);
+ }
+
+ /**
+ * Sets the visibility level for read operations.
+ */
+ public void setVisibility(VisibilityLevel level) {
+ this.visibilityLevel = level;
+ }
+
+ /**
+ * Returns the currently set visibility level.
+ */
+ public VisibilityLevel getVisibilityLevel() {
+ return visibilityLevel;
+ }
+
+ public boolean hasExcludes() {
+ return invalids.length > 0 || inProgress.length > 0;
+ }
+
+
+ public int excludesSize() {
+ return invalids.length + inProgress.length;
+ }
+
+ /**
+ * Returns any prior write pointers used in the current transaction. A new write pointer is issued when the
+ * {@code TransactionContext.checkpoint(Transaction)} operation is called, and the prior write pointer is added
+ * to the array of checkpoint write pointers.
+` */
+ public long[] getCheckpointWritePointers() {
+ return checkpointWritePointers;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder(100)
+ .append(Transaction.class.getSimpleName())
+ .append('{')
+ .append("readPointer: ").append(readPointer)
+ .append(", transactionId: ").append(txId)
+ .append(", writePointer: ").append(writePointer)
+ .append(", invalids: ").append(Arrays.toString(invalids))
+ .append(", inProgress: ").append(Arrays.toString(inProgress))
+ .append(", firstShortInProgress: ").append(firstShortInProgress)
+ .append(", type: ").append(type)
+ .append(", checkpointWritePointers: ").append(Arrays.toString(checkpointWritePointers))
+ .append(", visibilityLevel: ").append(visibilityLevel)
+ .append('}')
+ .toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/org/apache/tephra/TransactionAware.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/org/apache/tephra/TransactionAware.java b/tephra-api/src/main/java/org/apache/tephra/TransactionAware.java
new file mode 100644
index 0000000..a580914
--- /dev/null
+++ b/tephra-api/src/main/java/org/apache/tephra/TransactionAware.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import java.util.Collection;
+
+/**
+ * Interface to be implemented by a component that interacts with transaction logic.
+ * <p/>
+ * The client code that uses transaction logic looks like this:
+ * <pre>
+ * TransactionAware dataSet = // ... // dataSet is one example of component that interacts with tx logic
+ *
+ * Transaction tx = txClient.start();
+ * dataSet.startTx(tx); // notifying about new transaction
+ * dataSet.write(...);
+ * // ... do other operations on dataSet
+ * Collection<byte[]> changes = dataSet.getTxChanges();
+ * boolean rollback = true;
+ * if (txClient.canCommit(changes)) { // checking conflicts before commit, if none, commit tx
+ * if (dataSet.commitTx()) { // try persisting changes
+ * if (txClient.commit(tx)) { // if OK, make tx visible; if not - tx stays invisible to others
+ * dataSet.postTxCommit(); // notifying dataset about tx commit success via callback
+ * rollback = false;
+ * }
+ * }
+ * }
+ *
+ * if (rollback) { // if there are conflicts (or cannot commit), try rollback changes
+ * if (dataSet.rollbackTx()) { // try undo changes
+ * txClient.abort(tx); // if OK, make tx visible; if not - tx stays invisible to others
+ * }
+ * }
+ *
+ * </pre>
+ */
+// todo: use custom exception class?
+// todo: review exception handling where it is used
+// todo: add flush()? nah - flush is same as commitTx() actually
+// todo: add onCommitted() - so that e.g. hbase table can do *actual* deletes at this point
+public interface TransactionAware {
+ /**
+ * Called when new transaction has started. This may reset any state which has been left behind by the previous
+ * transaction.
+ *
+ * @param tx transaction info
+ */
+ // todo: rename to onTxStart()
+ void startTx(Transaction tx);
+
+ /**
+ * Called when the state of the current transaction has been updated. This should replace any reference to the
+ * current {@link Transaction} held by this {@code TransactionAware}, but should <strong>not</strong> reset
+ * any state (such as the write change sets) that is currently maintained.
+ *
+ * @param tx the updated transaction
+ */
+ void updateTx(Transaction tx);
+
+ /**
+ * @return changes made by current transaction to be used for conflicts detection before commit.
+ */
+ Collection<byte[]> getTxChanges();
+
+ /**
+ * Called before transaction has been committed.
+ * Can be used e.g. to flush changes cached in-memory to persistent store.
+ * @return true if transaction can be committed, otherwise false.
+ */
+ // todo: rename to beforeTxCommit()
+ boolean commitTx() throws Exception;
+
+ /**
+ * Called after transaction has been committed.
+ * Can be used e.g. evict entries from a cache etc. Because this is called after the transaction is committed,
+ * the success or failure of the transaction cannot depend on it. Hence this method returns nothing and it is not
+ * expected to throw exceptions.
+ * @throws RuntimeException in case of serious failure that should not be ignored.
+ */
+ void postTxCommit();
+
+ /**
+ * Called during transaction rollback (for whatever reason: conflicts, errors, etc.).
+ * @return true if all changes made during transaction were rolled back, false otherwise (e.g. if more cleanup needed
+ * or changes cannot be undone). True also means that this transaction can be made visible to others without
+ * breaking consistency of the data: since all changes were undone there's "nothing to see".
+ */
+ // todo: rename to onTxRollback()
+ boolean rollbackTx() throws Exception;
+
+ /**
+ * Used for error reporting.
+ */
+ // todo: use toString() instead everywhere
+ String getTransactionAwareName();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/org/apache/tephra/TransactionAwares.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/org/apache/tephra/TransactionAwares.java b/tephra-api/src/main/java/org/apache/tephra/TransactionAwares.java
new file mode 100644
index 0000000..40059ea
--- /dev/null
+++ b/tephra-api/src/main/java/org/apache/tephra/TransactionAwares.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Utility class that provides handy methods for working with {@link TransactionAware} classes and their instances.
+ */
+public final class TransactionAwares {
+ private TransactionAwares() {}
+
+ /**
+ * Create composite transaction aware object that delegates transaction logic to given collection of
+ * {@link TransactionAware}s
+ * @param transactionAwares collection of {@link TransactionAware}s
+ * @return instance of {@link TransactionAware}
+ */
+ public static TransactionAware of(Collection<TransactionAware> transactionAwares) {
+ // this is most common case, trying to optimize
+ if (transactionAwares.size() == 1) {
+ return transactionAwares.iterator().next();
+ }
+
+ TransactionAwareCollection result = new TransactionAwareCollection();
+ result.addAll(transactionAwares);
+
+ return result;
+ }
+
+ private static class TransactionAwareCollection extends ArrayList<TransactionAware> implements TransactionAware {
+
+ @Override
+ public void startTx(Transaction tx) {
+ for (TransactionAware txAware : this) {
+ txAware.startTx(tx);
+ }
+ }
+
+ @Override
+ public void updateTx(Transaction tx) {
+ for (TransactionAware txAware : this) {
+ txAware.updateTx(tx);
+ }
+ }
+
+ @Override
+ public Collection<byte[]> getTxChanges() {
+ List<byte[]> changes = new ArrayList<byte[]>();
+ for (TransactionAware txAware : this) {
+ changes.addAll(txAware.getTxChanges());
+ }
+
+ return changes;
+ }
+
+ @Override
+ public boolean commitTx() throws Exception {
+ boolean success = true;
+ for (TransactionAware txAware : this) {
+ success = success && txAware.commitTx();
+ }
+ return success;
+ }
+
+ @Override
+ public void postTxCommit() {
+ for (TransactionAware txAware : this) {
+ txAware.postTxCommit();
+ }
+ }
+
+ @Override
+ public boolean rollbackTx() throws Exception {
+ boolean success = true;
+ for (TransactionAware txAware : this) {
+ success = success && txAware.rollbackTx();
+ }
+ return success;
+ }
+
+ @Override
+ public String getTransactionAwareName() {
+ // todo: will go away, see comment at TransactionAware
+ StringBuilder sb = new StringBuilder("{");
+ for (TransactionAware txAware : this) {
+ sb.append(txAware.getTransactionAwareName()).append(",");
+ }
+ sb.replace(sb.length() - 1, sb.length() - 1, "}");
+ return sb.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/org/apache/tephra/TransactionConflictException.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/org/apache/tephra/TransactionConflictException.java b/tephra-api/src/main/java/org/apache/tephra/TransactionConflictException.java
new file mode 100644
index 0000000..d07ed04
--- /dev/null
+++ b/tephra-api/src/main/java/org/apache/tephra/TransactionConflictException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Thrown to indicate transaction conflict occurred when trying to commit a transaction.
+ */
+public class TransactionConflictException extends TransactionFailureException {
+ public TransactionConflictException(String message) {
+ super(message);
+ }
+
+ public TransactionConflictException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/org/apache/tephra/TransactionFailureException.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/org/apache/tephra/TransactionFailureException.java b/tephra-api/src/main/java/org/apache/tephra/TransactionFailureException.java
new file mode 100644
index 0000000..88932a4
--- /dev/null
+++ b/tephra-api/src/main/java/org/apache/tephra/TransactionFailureException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Thrown to indicate execution of a transaction failed.
+ */
+public class TransactionFailureException extends Exception {
+ public TransactionFailureException(String message) {
+ super(message);
+ }
+
+ public TransactionFailureException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/org/apache/tephra/TransactionNotInProgressException.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/org/apache/tephra/TransactionNotInProgressException.java b/tephra-api/src/main/java/org/apache/tephra/TransactionNotInProgressException.java
new file mode 100644
index 0000000..9f765ec
--- /dev/null
+++ b/tephra-api/src/main/java/org/apache/tephra/TransactionNotInProgressException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Thrown when transaction has timed out.
+ */
+public class TransactionNotInProgressException extends TransactionFailureException {
+ public TransactionNotInProgressException(String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-api/src/main/java/org/apache/tephra/TransactionType.java
----------------------------------------------------------------------
diff --git a/tephra-api/src/main/java/org/apache/tephra/TransactionType.java b/tephra-api/src/main/java/org/apache/tephra/TransactionType.java
new file mode 100644
index 0000000..7febe49
--- /dev/null
+++ b/tephra-api/src/main/java/org/apache/tephra/TransactionType.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Type of transaction.
+ */
+public enum TransactionType {
+ /**
+ * Short transactions detect conflicts during commit.
+ */
+ SHORT,
+ /**
+ * Long running transactions do not detect conflicts during commit.
+ */
+ LONG
+}
[32/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/rpc/ThriftRPCServer.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/rpc/ThriftRPCServer.java b/tephra-core/src/main/java/org/apache/tephra/rpc/ThriftRPCServer.java
new file mode 100644
index 0000000..738d6c8
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/rpc/ThriftRPCServer.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.rpc;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.AbstractExecutionThreadService;
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadedSelectorServerWithFix;
+import org.apache.thrift.transport.TFramedTransport;
+import org.apache.thrift.transport.TNonblockingServerSocket;
+import org.apache.twill.common.Threads;
+import org.apache.twill.internal.utils.Networks;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.InetSocketAddress;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @param <T> The type of service handler interface.
+ * @param <I> The type of the thrift service.
+ */
+public final class ThriftRPCServer<T extends RPCServiceHandler, I> extends AbstractExecutionThreadService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ThriftRPCServer.class);
+
+ private final String name;
+ private final int ioThreads;
+ private final int workerThreads;
+ private final int maxReadBufferBytes;
+ private final T serviceHandler;
+ private final TProcessor processor;
+
+ private InetSocketAddress bindAddress;
+ private ExecutorService executor;
+ private TServer server;
+
+ /**
+ * Creates a {@link Builder} for creating instance of {@link ThriftRPCServer}.
+ * @param serviceType Class of the thrift service.
+ * @param <I> Type of the thrift service.
+ * @return A {@link Builder}.
+ */
+ public static <I> Builder<I> builder(Class<I> serviceType) {
+ return new Builder<I>(serviceType);
+ }
+
+ /**
+ * Builder for creating instance of ThriftRPCServer. By default, the instance created will bind to
+ * random port and with 2 io threads and worker threads equals to min(2, number of cpu cores - 2).
+ */
+ public static final class Builder<I> {
+ private final Class<I> serviceType;
+ private String name;
+ private InetSocketAddress bindAddress = new InetSocketAddress(0);
+ private int ioThreads = 2;
+ private int workerThreads = Runtime.getRuntime().availableProcessors() - 2;
+ // 16Mb
+ private int maxReadBufferBytes = 16 * 1024 * 1024;
+
+ private Builder(Class<I> serviceType) {
+ this.serviceType = serviceType;
+ this.name = serviceType.getSimpleName();
+ }
+
+ public Builder<I> setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder<I> setHost(String host) {
+ this.bindAddress = new InetSocketAddress(host, bindAddress.getPort());
+ return this;
+ }
+
+ public Builder<I> setPort(int port) {
+ this.bindAddress = new InetSocketAddress(bindAddress.getHostName(), port);
+ return this;
+ }
+
+ public Builder<I> setIOThreads(int count) {
+ this.ioThreads = count;
+ return this;
+ }
+
+ public Builder<I> setWorkerThreads(int count) {
+ this.workerThreads = count;
+ return this;
+ }
+
+ public Builder<I> setMaxReadBufferBytes(int maxReadBufferBytes) {
+ this.maxReadBufferBytes = maxReadBufferBytes;
+ return this;
+ }
+
+ public <T extends RPCServiceHandler> ThriftRPCServer<T, I> build(T serviceHandler) {
+ return new ThriftRPCServer<T, I>(bindAddress, ioThreads, workerThreads, maxReadBufferBytes,
+ serviceHandler, serviceType, name);
+ }
+ }
+
+ /**
+ * Creates a ThriftRPCServer with the given paramters.
+ *
+ * @param bindAddress The socket address for the server to listen on. If {@code null}, it'll be binded to random
+ * port on localhost.
+ * @param ioThreads Number of io threads.
+ * @param workerThreads Number of worker threads.
+ * @param serviceHandler Handler for handling client requests.
+ */
+ @SuppressWarnings("unchecked")
+ private ThriftRPCServer(InetSocketAddress bindAddress, int ioThreads,
+ int workerThreads, int maxReadBufferBytes,
+ T serviceHandler, Class<I> serviceType, String name) {
+ Preconditions.checkArgument(ioThreads > 0, "IO threads must be > 0.");
+ Preconditions.checkArgument(workerThreads > 0, "Worker threads must be > 0.");
+
+ this.bindAddress = bindAddress;
+ this.ioThreads = ioThreads;
+ this.workerThreads = workerThreads;
+ this.maxReadBufferBytes = maxReadBufferBytes;
+ this.serviceHandler = serviceHandler;
+ this.name = name;
+ this.processor = createProcessor((Class<T>) serviceHandler.getClass(), serviceType);
+ }
+
+ public InetSocketAddress getBindAddress() {
+ return bindAddress;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ // Determines the address and port to listen on
+ InetSocketAddress listenOn = bindAddress;
+ if (listenOn == null || listenOn.getPort() <= 0) {
+ int port = Networks.getRandomPort();
+ if (listenOn == null) {
+ listenOn = new InetSocketAddress("localhost", port);
+ } else {
+ listenOn = new InetSocketAddress(listenOn.getAddress(), port);
+ }
+ }
+ bindAddress = listenOn;
+
+ executor = new ThreadPoolExecutor(0, workerThreads,
+ 60L, TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>(),
+ Threads.createDaemonThreadFactory(String.format("%s-rpc-%%d", name)),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+ serviceHandler.init();
+
+ TThreadedSelectorServerWithFix.Args args =
+ new TThreadedSelectorServerWithFix.Args(new TNonblockingServerSocket(listenOn))
+ .selectorThreads(ioThreads)
+ .protocolFactory(new TBinaryProtocol.Factory())
+ .transportFactory(new TFramedTransport.Factory())
+ .processor(processor)
+ .executorService(executor);
+
+ // ENG-443 - Set the max read buffer size. This is important as this will
+ // prevent the server from throwing OOME if telnetd to the port
+ // it's running on.
+ args.maxReadBufferBytes = maxReadBufferBytes;
+ server = new TThreadedSelectorServerWithFix(args);
+ LOG.info("Starting RPC server for {}", name);
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ serviceHandler.destroy();
+ executor.shutdownNow();
+ LOG.info("RPC server for {} stopped.", name);
+ }
+
+ @Override
+ protected void triggerShutdown() {
+ LOG.info("Request to stop RPC server for {}", name);
+ server.stop();
+ }
+
+ @Override
+ protected void run() throws Exception {
+ LOG.info("Running RPC server for {}", name);
+ server.serve();
+ LOG.info("Done running RPC server for {}", name);
+ }
+
+ @SuppressWarnings("unchecked")
+ private TProcessor createProcessor(final Class<T> handlerType, Class<I> serviceType) {
+ // Pick the Iface inner interface and the Processor class
+ Class<? extends TProcessor> processorType = null;
+ Class<?> ifaceType = null;
+ for (Class<?> clz : serviceType.getDeclaredClasses()) {
+ if (TProcessor.class.isAssignableFrom(clz)) {
+ processorType = (Class<? extends TProcessor>) clz;
+ } else if (clz.isInterface() && "Iface".equals(clz.getSimpleName())) {
+ ifaceType = clz;
+ }
+ }
+
+ Preconditions.checkArgument(processorType != null,
+ "Missing TProcessor, %s is not a valid thrift service.", serviceType.getName());
+ Preconditions.checkArgument(ifaceType != null,
+ "Missing Iface, %s is not a valid thrift service.", serviceType.getName());
+
+ // If handler already implements the Iface, simply delegate
+ if (ifaceType.isAssignableFrom(handlerType)) {
+ return createProxyProcessor(handlerType, processorType, ifaceType);
+ }
+
+ throw new IllegalArgumentException("Unsupported handler type.");
+ }
+
+ private TProcessor createProxyProcessor(final Class<T> handlerType,
+ Class<? extends TProcessor> processorType, Class<?> ifaceType) {
+
+ try {
+ // Map from Iface method to handlerType method to save reflection lookup
+ ImmutableMap.Builder<Method, Method> builder = ImmutableMap.builder();
+ for (Method method : ifaceType.getMethods()) {
+ Method handlerMethod = handlerType.getMethod(method.getName(), method.getParameterTypes());
+ if (!handlerMethod.isAccessible()) {
+ handlerMethod.setAccessible(true);
+ }
+ builder.put(method, handlerMethod);
+ }
+ final Map<Method, Method> methods = builder.build();
+
+ Object proxy = Proxy.newProxyInstance(ifaceType.getClassLoader(),
+ new Class[]{ifaceType}, new InvocationHandler() {
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ try {
+ return methods.get(method).invoke(serviceHandler, args);
+ } catch (InvocationTargetException e) {
+ if (e.getCause() != null) {
+ throw e.getCause();
+ } else {
+ throw e;
+ }
+ }
+ }
+ });
+
+ return processorType.getConstructor(ifaceType).newInstance(proxy);
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/rpc/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/rpc/package-info.java b/tephra-core/src/main/java/org/apache/tephra/rpc/package-info.java
new file mode 100644
index 0000000..96a5fe5
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/rpc/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains class for writing RPC server and client in simple manner.
+ */
+package org.apache.tephra.rpc;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/ConfigModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/ConfigModule.java b/tephra-core/src/main/java/org/apache/tephra/runtime/ConfigModule.java
new file mode 100644
index 0000000..b8ae111
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/ConfigModule.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.inject.AbstractModule;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Provides Guice bindings for {@link Configuration}.
+ */
+public final class ConfigModule extends AbstractModule {
+
+ private final Configuration configuration;
+
+ public ConfigModule(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ protected void configure() {
+ bind(Configuration.class).toInstance(configuration);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/DiscoveryModules.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/DiscoveryModules.java b/tephra-core/src/main/java/org/apache/tephra/runtime/DiscoveryModules.java
new file mode 100644
index 0000000..b4e2d1b
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/DiscoveryModules.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Module;
+import com.google.inject.PrivateModule;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.discovery.Discoverable;
+import org.apache.twill.discovery.DiscoveryService;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+import org.apache.twill.discovery.InMemoryDiscoveryService;
+import org.apache.twill.discovery.ServiceDiscovered;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.zookeeper.ZKClientService;
+
+/**
+ * Provides access to Google Guice modules for in-memory, single-node, and distributed operation for
+ * {@link DiscoveryService} and {@link DiscoveryServiceClient}.
+ */
+public final class DiscoveryModules {
+
+ public Module getInMemoryModules() {
+ return new InMemoryDiscoveryModule();
+ }
+
+ public Module getSingleNodeModules() {
+ return new InMemoryDiscoveryModule();
+ }
+
+ public Module getDistributedModules() {
+ return new ZKDiscoveryModule();
+ }
+
+ private static final class InMemoryDiscoveryModule extends AbstractModule {
+
+ // ensuring to be singleton across JVM
+ private static final InMemoryDiscoveryService IN_MEMORY_DISCOVERY_SERVICE = new InMemoryDiscoveryService();
+
+ @Override
+ protected void configure() {
+ InMemoryDiscoveryService discovery = IN_MEMORY_DISCOVERY_SERVICE;
+ bind(DiscoveryService.class).toInstance(discovery);
+ bind(DiscoveryServiceClient.class).toInstance(discovery);
+ }
+ }
+
+ private static final class ZKDiscoveryModule extends PrivateModule {
+
+ @Override
+ protected void configure() {
+ expose(DiscoveryService.class);
+ expose(DiscoveryServiceClient.class);
+ }
+
+ @Provides
+ @Singleton
+ private ZKDiscoveryService providesZKDiscoveryService(ZKClientService zkClient) {
+ return new ZKDiscoveryService(zkClient);
+ }
+
+ @Provides
+ @Singleton
+ private DiscoveryService providesDiscoveryService(final ZKClientService zkClient,
+ final ZKDiscoveryService delegate) {
+ return new DiscoveryService() {
+ @Override
+ public Cancellable register(Discoverable discoverable) {
+ if (!zkClient.isRunning()) {
+ zkClient.startAndWait();
+ }
+ return delegate.register(discoverable);
+ }
+ };
+ }
+
+ @Provides
+ @Singleton
+ private DiscoveryServiceClient providesDiscoveryServiceClient(final ZKClientService zkClient,
+ final ZKDiscoveryService delegate) {
+ return new DiscoveryServiceClient() {
+ @Override
+ public ServiceDiscovered discover(String s) {
+ if (!zkClient.isRunning()) {
+ zkClient.startAndWait();
+ }
+ return delegate.discover(s);
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionClientModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionClientModule.java b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionClientModule.java
new file mode 100644
index 0000000..6689d05
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionClientModule.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.distributed.PooledClientProvider;
+import org.apache.tephra.distributed.ThreadLocalClientProvider;
+import org.apache.tephra.distributed.ThriftClientProvider;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+
+/**
+ * Provides Guice binding for {@link ThriftClientProvider}.
+ */
+public class TransactionClientModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(ThriftClientProvider.class).toProvider(ThriftClientProviderSupplier.class);
+ }
+
+ /**
+ * Provides implementation of {@link ThriftClientProvider}
+ * based on configuration.
+ */
+ @Singleton
+ private static final class ThriftClientProviderSupplier implements Provider<ThriftClientProvider> {
+
+ private final Configuration cConf;
+ private DiscoveryServiceClient discoveryServiceClient;
+
+ @Inject
+ ThriftClientProviderSupplier(Configuration cConf) {
+ this.cConf = cConf;
+ }
+
+ @Inject(optional = true)
+ void setDiscoveryServiceClient(DiscoveryServiceClient discoveryServiceClient) {
+ this.discoveryServiceClient = discoveryServiceClient;
+ }
+
+ @Override
+ public ThriftClientProvider get() {
+ // configure the client provider
+ String provider = cConf.get(TxConstants.Service.CFG_DATA_TX_CLIENT_PROVIDER,
+ TxConstants.Service.DEFAULT_DATA_TX_CLIENT_PROVIDER);
+ ThriftClientProvider clientProvider;
+ if ("pool".equals(provider)) {
+ clientProvider = new PooledClientProvider(cConf, discoveryServiceClient);
+ } else if ("thread-local".equals(provider)) {
+ clientProvider = new ThreadLocalClientProvider(cConf, discoveryServiceClient);
+ } else {
+ String message = "Unknown Transaction Service Client Provider '" + provider + "'.";
+ throw new IllegalArgumentException(message);
+ }
+ return clientProvider;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionDistributedModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionDistributedModule.java b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionDistributedModule.java
new file mode 100644
index 0000000..aaf3534
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionDistributedModule.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Singleton;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.google.inject.name.Names;
+import org.apache.tephra.DefaultTransactionExecutor;
+import org.apache.tephra.TransactionExecutor;
+import org.apache.tephra.TransactionExecutorFactory;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.distributed.TransactionServiceClient;
+import org.apache.tephra.metrics.DefaultMetricsCollector;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.apache.tephra.persist.HDFSTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+
+/**
+ * Guice bindings for running in distributed mode on a cluster.
+ */
+final class TransactionDistributedModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(SnapshotCodecProvider.class).in(Singleton.class);
+ bind(TransactionStateStorage.class).annotatedWith(Names.named("persist"))
+ .to(HDFSTransactionStateStorage.class).in(Singleton.class);
+ bind(TransactionStateStorage.class).toProvider(TransactionStateStorageProvider.class).in(Singleton.class);
+
+ bind(TransactionManager.class).in(Singleton.class);
+ bind(TransactionSystemClient.class).to(TransactionServiceClient.class).in(Singleton.class);
+ bind(MetricsCollector.class).to(DefaultMetricsCollector.class).in(Singleton.class);
+
+ install(new FactoryModuleBuilder()
+ .implement(TransactionExecutor.class, DefaultTransactionExecutor.class)
+ .build(TransactionExecutorFactory.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionInMemoryModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionInMemoryModule.java b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionInMemoryModule.java
new file mode 100644
index 0000000..de7678a
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionInMemoryModule.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Singleton;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import org.apache.tephra.DefaultTransactionExecutor;
+import org.apache.tephra.TransactionExecutor;
+import org.apache.tephra.TransactionExecutorFactory;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.NoOpTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+
+/**
+ * Guice bindings for running completely in-memory (no persistence). This should only be used for
+ * test classes, as the transaction state cannot be recovered in the case of a failure.
+ */
+public class TransactionInMemoryModule extends AbstractModule {
+ public TransactionInMemoryModule() {
+ }
+
+ @Override
+ protected void configure() {
+ bind(SnapshotCodecProvider.class).in(Singleton.class);
+ bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class).in(Singleton.class);
+ bind(TransactionManager.class).in(Singleton.class);
+ bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
+ // no metrics output for in-memory
+ bind(MetricsCollector.class).to(TxMetricsCollector.class);
+
+ install(new FactoryModuleBuilder()
+ .implement(TransactionExecutor.class, DefaultTransactionExecutor.class)
+ .build(TransactionExecutorFactory.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionLocalModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionLocalModule.java b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionLocalModule.java
new file mode 100644
index 0000000..7d0b663
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionLocalModule.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Singleton;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.google.inject.name.Names;
+import org.apache.tephra.DefaultTransactionExecutor;
+import org.apache.tephra.TransactionExecutor;
+import org.apache.tephra.TransactionExecutorFactory;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.DefaultMetricsCollector;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.apache.tephra.persist.LocalFileTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+
+/**
+ * Guice bindings for running in single-node mode (persistence to local disk and in-memory client).
+ */
+final class TransactionLocalModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(SnapshotCodecProvider.class).in(Singleton.class);
+ bind(TransactionStateStorage.class).annotatedWith(Names.named("persist"))
+ .to(LocalFileTransactionStateStorage.class).in(Singleton.class);
+ bind(TransactionStateStorage.class).toProvider(TransactionStateStorageProvider.class).in(Singleton.class);
+
+ bind(TransactionManager.class).in(Singleton.class);
+ bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
+ bind(MetricsCollector.class).to(DefaultMetricsCollector.class);
+
+ install(new FactoryModuleBuilder()
+ .implement(TransactionExecutor.class, DefaultTransactionExecutor.class)
+ .build(TransactionExecutorFactory.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionModules.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionModules.java b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionModules.java
new file mode 100644
index 0000000..a3fe1c1
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionModules.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.inject.Module;
+
+/**
+ * Provides access to Google Guice modules for in-memory, single-node, and distributed operation.
+ */
+public class TransactionModules {
+ public TransactionModules() {
+ }
+
+ public Module getInMemoryModules() {
+ return new TransactionInMemoryModule();
+ }
+
+ public Module getSingleNodeModules() {
+ return new TransactionLocalModule();
+ }
+
+ public Module getDistributedModules() {
+ return new TransactionDistributedModule();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionStateStorageProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionStateStorageProvider.java b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionStateStorageProvider.java
new file mode 100644
index 0000000..5456553
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/TransactionStateStorageProvider.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.runtime;
+
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import com.google.inject.name.Names;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.persist.NoOpTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+
+/**
+ * A provider for {@link TransactionStateStorage} that provides different
+ * {@link TransactionStateStorage} implementation based on configuration.
+ */
+@Singleton
+public final class TransactionStateStorageProvider implements Provider<TransactionStateStorage> {
+
+ private final Configuration cConf;
+ private final Injector injector;
+
+ @Inject
+ TransactionStateStorageProvider(Configuration cConf, Injector injector) {
+ this.cConf = cConf;
+ this.injector = injector;
+ }
+
+ @Override
+ public TransactionStateStorage get() {
+ if (cConf.getBoolean(TxConstants.Manager.CFG_DO_PERSIST, true)) {
+ return injector.getInstance(Key.get(TransactionStateStorage.class, Names.named("persist")));
+ } else {
+ return injector.getInstance(NoOpTransactionStateStorage.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/runtime/ZKModule.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/runtime/ZKModule.java b/tephra-core/src/main/java/org/apache/tephra/runtime/ZKModule.java
new file mode 100644
index 0000000..da3e019
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/runtime/ZKModule.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.runtime;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.zookeeper.TephraZKClientService;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Provides Guice binding to {@link ZKClient} and {@link ZKClientService}.
+ */
+public class ZKModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ /**
+ * ZKClientService is provided by the provider method
+ * {@link #provideZKClientService(org.apache.hadoop.conf.Configuration)}.
+ */
+ bind(ZKClient.class).to(ZKClientService.class);
+ }
+
+ @Provides
+ @Singleton
+ private ZKClientService provideZKClientService(Configuration conf) {
+ String zkStr = conf.get(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM);
+ if (zkStr == null) {
+ // Default to HBase one.
+ zkStr = conf.get(TxConstants.HBase.ZOOKEEPER_QUORUM);
+ }
+
+ int timeOut = conf.getInt(TxConstants.HBase.ZK_SESSION_TIMEOUT, TxConstants.HBase.DEFAULT_ZK_SESSION_TIMEOUT);
+ ZKClientService zkClientService = new TephraZKClientService(zkStr, timeOut, null,
+ ArrayListMultimap.<String, byte[]>create());
+ return ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(zkClientService, RetryStrategies.exponentialDelay(500, 2000, TimeUnit.MILLISECONDS)
+ )
+ )
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryDecoder.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryDecoder.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryDecoder.java
new file mode 100644
index 0000000..a1cd6dd
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryDecoder.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An decoder to help read snapshots in binary format.
+ */
+public final class BinaryDecoder {
+
+ private final InputStream input;
+
+ /**
+ * @param input Stream to read from.
+ */
+ public BinaryDecoder(InputStream input) {
+ this.input = input;
+ }
+
+ /**
+ * Read one int from the input.
+ * @return the read number
+ * @throws java.io.IOException If there is IO error.
+ * @throws java.io.EOFException If end of file reached.
+ */
+ public int readInt() throws IOException {
+ int val = 0;
+ int shift = 0;
+ int b = readByte();
+ while (b > 0x7f) {
+ val ^= (b & 0x7f) << shift;
+ shift += 7;
+ b = readByte();
+ }
+ val ^= b << shift;
+ return (val >>> 1) ^ -(val & 1);
+ }
+
+ /**
+ * Read one long int from the input.
+ * @return the read number
+ * @throws java.io.IOException If there is IO error.
+ * @throws java.io.EOFException If end of file reached.
+ */
+ public long readLong() throws IOException {
+ long val = 0;
+ int shift = 0;
+ int b = readByte();
+ while (b > 0x7f) {
+ val ^= (long) (b & 0x7f) << shift;
+ shift += 7;
+ b = readByte();
+ }
+ val ^= (long) b << shift;
+ return (val >>> 1) ^ -(val & 1);
+ }
+
+ /**
+ * Read a byte sequence. First read an int to indicate how many bytes to read, then that many bytes.
+ * @return the read bytes as a byte array
+ * @throws java.io.IOException If there is IO error.
+ * @throws java.io.EOFException If end of file reached.
+ */
+ public byte[] readBytes() throws IOException {
+ int toRead = readInt();
+ byte[] bytes = new byte[toRead];
+ while (toRead > 0) {
+ int byteRead = input.read(bytes, bytes.length - toRead, toRead);
+ if (byteRead == -1) {
+ throw new EOFException();
+ }
+ toRead -= byteRead;
+ }
+ return bytes;
+ }
+
+ /**
+ * Reads a single byte value.
+ *
+ * @return The byte value read.
+ * @throws java.io.IOException If there is IO error.
+ * @throws java.io.EOFException If end of file reached.
+ */
+ private int readByte() throws IOException {
+ int b = input.read();
+ if (b == -1) {
+ throw new EOFException();
+ }
+ return b;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryEncoder.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryEncoder.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryEncoder.java
new file mode 100644
index 0000000..c4ce0a8
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/BinaryEncoder.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An encoder to help encode snapshots in binary format.
+ */
+public final class BinaryEncoder {
+
+ private final OutputStream output;
+
+ /**
+ * @param output stream to write to
+ */
+ public BinaryEncoder(OutputStream output) {
+ this.output = output;
+ }
+
+ /**
+ * write a single int value.
+ * @throws java.io.IOException If there is IO error.
+ */
+ public BinaryEncoder writeInt(int i) throws IOException {
+ // Compute the zig-zag value. First double the value and flip the bit if the input is negative.
+ int val = (i << 1) ^ (i >> 31);
+
+ if ((val & ~0x7f) != 0) {
+ output.write(0x80 | val & 0x7f);
+ val >>>= 7;
+ while (val > 0x7f) {
+ output.write(0x80 | val & 0x7f);
+ val >>>= 7;
+ }
+ }
+ output.write(val);
+
+ return this;
+ }
+
+ /**
+ * write a single long int value.
+ * @throws java.io.IOException If there is IO error.
+ */
+ public BinaryEncoder writeLong(long l) throws IOException {
+ // Compute the zig-zag value. First double the value and flip the bit if the input is negative.
+ long val = (l << 1) ^ (l >> 63);
+
+ if ((val & ~0x7f) != 0) {
+ output.write((int) (0x80 | val & 0x7f));
+ val >>>= 7;
+ while (val > 0x7f) {
+ output.write((int) (0x80 | val & 0x7f));
+ val >>>= 7;
+ }
+ }
+ output.write((int) val);
+
+ return this;
+ }
+
+ /**
+ * write a sequence of bytes. First writes the number of bytes as an int, then the bytes themselves.
+ * @throws java.io.IOException If there is IO error.
+ */
+ public BinaryEncoder writeBytes(byte[] bytes) throws IOException {
+ writeLong(bytes.length);
+ output.write(bytes, 0, bytes.length);
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/DefaultSnapshotCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/DefaultSnapshotCodec.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/DefaultSnapshotCodec.java
new file mode 100644
index 0000000..4a94c74
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/DefaultSnapshotCodec.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Handles serialization/deserialization of a {@link TransactionSnapshot}
+ * and its elements to {@code byte[]}.
+ * @deprecated This codec is now deprecated and is replaced by {@link SnapshotCodecV2}.
+ */
+@Deprecated
+public class DefaultSnapshotCodec implements SnapshotCodec {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultSnapshotCodec.class);
+
+ @Override
+ public int getVersion() {
+ return 1;
+ }
+
+ @Override
+ public void encode(OutputStream out, TransactionSnapshot snapshot) {
+ try {
+ BinaryEncoder encoder = new BinaryEncoder(out);
+
+ encoder.writeLong(snapshot.getTimestamp());
+ encoder.writeLong(snapshot.getReadPointer());
+ encoder.writeLong(snapshot.getWritePointer());
+ encodeInvalid(encoder, snapshot.getInvalid());
+ encodeInProgress(encoder, snapshot.getInProgress());
+ encodeChangeSets(encoder, snapshot.getCommittingChangeSets());
+ encodeChangeSets(encoder, snapshot.getCommittedChangeSets());
+
+ } catch (IOException e) {
+ LOG.error("Unable to serialize transaction state: ", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public TransactionSnapshot decode(InputStream in) {
+ BinaryDecoder decoder = new BinaryDecoder(in);
+
+ try {
+ TransactionVisibilityState minTxSnapshot = decodeTransactionVisibilityState(in);
+ NavigableMap<Long, Set<ChangeId>> committing = decodeChangeSets(decoder);
+ NavigableMap<Long, Set<ChangeId>> committed = decodeChangeSets(decoder);
+ return new TransactionSnapshot(minTxSnapshot.getTimestamp(), minTxSnapshot.getReadPointer(),
+ minTxSnapshot.getWritePointer(), minTxSnapshot.getInvalid(),
+ minTxSnapshot.getInProgress(), committing, committed);
+ } catch (IOException e) {
+ LOG.error("Unable to deserialize transaction state: ", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public TransactionVisibilityState decodeTransactionVisibilityState(InputStream in) {
+ BinaryDecoder decoder = new BinaryDecoder(in);
+ try {
+ long timestamp = decoder.readLong();
+ long readPointer = decoder.readLong();
+ long writePointer = decoder.readLong();
+ Collection<Long> invalid = decodeInvalid(decoder);
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress = decodeInProgress(decoder);
+ return new TransactionSnapshot(timestamp, readPointer, writePointer, invalid, inProgress);
+ } catch (IOException e) {
+ LOG.error("Unable to deserialize transaction state: ", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private void encodeInvalid(BinaryEncoder encoder, Collection<Long> invalid) throws IOException {
+ if (!invalid.isEmpty()) {
+ encoder.writeInt(invalid.size());
+ for (long invalidTx : invalid) {
+ encoder.writeLong(invalidTx);
+ }
+ }
+ encoder.writeInt(0); // zero denotes end of list as per AVRO spec
+ }
+
+ private Collection<Long> decodeInvalid(BinaryDecoder decoder) throws IOException {
+ int size = decoder.readInt();
+ Collection<Long> invalid = Lists.newArrayListWithCapacity(size);
+ while (size != 0) { // zero denotes end of list as per AVRO spec
+ for (int remaining = size; remaining > 0; --remaining) {
+ invalid.add(decoder.readLong());
+ }
+ size = decoder.readInt();
+ }
+ return invalid;
+ }
+
+ protected void encodeInProgress(BinaryEncoder encoder, Map<Long, TransactionManager.InProgressTx> inProgress)
+ throws IOException {
+
+ if (!inProgress.isEmpty()) {
+ encoder.writeInt(inProgress.size());
+ for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
+ encoder.writeLong(entry.getKey()); // tx id
+ encoder.writeLong(entry.getValue().getExpiration());
+ encoder.writeLong(entry.getValue().getVisibilityUpperBound());
+ }
+ }
+ encoder.writeInt(0); // zero denotes end of list as per AVRO spec
+ }
+
+ protected NavigableMap<Long, TransactionManager.InProgressTx> decodeInProgress(BinaryDecoder decoder)
+ throws IOException {
+
+ int size = decoder.readInt();
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
+ while (size != 0) { // zero denotes end of list as per AVRO spec
+ for (int remaining = size; remaining > 0; --remaining) {
+ long txId = decoder.readLong();
+ long expiration = decoder.readLong();
+ long visibilityUpperBound = decoder.readLong();
+ inProgress.put(txId,
+ new TransactionManager.InProgressTx(visibilityUpperBound, expiration));
+ }
+ size = decoder.readInt();
+ }
+ return inProgress;
+ }
+
+ private void encodeChangeSets(BinaryEncoder encoder, Map<Long, Set<ChangeId>> changes) throws IOException {
+ if (!changes.isEmpty()) {
+ encoder.writeInt(changes.size());
+ for (Map.Entry<Long, Set<ChangeId>> entry : changes.entrySet()) {
+ encoder.writeLong(entry.getKey());
+ encodeChanges(encoder, entry.getValue());
+ }
+ }
+ encoder.writeInt(0); // zero denotes end of list as per AVRO spec
+ }
+
+ private NavigableMap<Long, Set<ChangeId>> decodeChangeSets(BinaryDecoder decoder) throws IOException {
+ int size = decoder.readInt();
+ NavigableMap<Long, Set<ChangeId>> changeSets = new TreeMap<Long, Set<ChangeId>>();
+ while (size != 0) { // zero denotes end of list as per AVRO spec
+ for (int remaining = size; remaining > 0; --remaining) {
+ changeSets.put(decoder.readLong(), decodeChanges(decoder));
+ }
+ size = decoder.readInt();
+ }
+ return changeSets;
+ }
+
+ private void encodeChanges(BinaryEncoder encoder, Set<ChangeId> changes) throws IOException {
+ if (!changes.isEmpty()) {
+ encoder.writeInt(changes.size());
+ for (ChangeId change : changes) {
+ encoder.writeBytes(change.getKey());
+ }
+ }
+ encoder.writeInt(0); // zero denotes end of list as per AVRO spec
+ }
+
+ private Set<ChangeId> decodeChanges(BinaryDecoder decoder) throws IOException {
+ int size = decoder.readInt();
+ HashSet<ChangeId> changes = Sets.newHashSetWithExpectedSize(size);
+ while (size != 0) { // zero denotes end of list as per AVRO spec
+ for (int remaining = size; remaining > 0; --remaining) {
+ changes.add(new ChangeId(decoder.readBytes()));
+ }
+ size = decoder.readInt();
+ }
+ // todo is there an immutable hash set?
+ return changes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodec.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodec.java
new file mode 100644
index 0000000..e2c5c16
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodec.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionVisibilityState;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Interface to decode and encode a transaction snapshot. Each codec implements one version of the encoding.
+ * It need not include the version when encoding the snapshot.
+ */
+public interface SnapshotCodec {
+
+ /**
+ * @return the version of the encoding implemented by the codec.
+ */
+ int getVersion();
+
+ /**
+ * Encode a transaction snapshot into an output stream.
+ * @param out the output stream to write to
+ * @param snapshot the snapshot to encode
+ */
+ void encode(OutputStream out, TransactionSnapshot snapshot);
+
+ /**
+ * Decode a transaction snapshot from an input stream.
+ * @param in the input stream to read from
+ * @return the decoded snapshot
+ */
+ TransactionSnapshot decode(InputStream in);
+
+ /**
+ * Decode transaction visibility state from an input stream.
+ * @param in the input stream to read from
+ * @return {@link TransactionVisibilityState}
+ */
+ TransactionVisibilityState decodeTransactionVisibilityState(InputStream in);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecProvider.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecProvider.java
new file mode 100644
index 0000000..3756846
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecProvider.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.inject.Inject;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.SortedMap;
+import javax.annotation.Nonnull;
+
+/**
+ * Maintains the codecs for all known versions of the transaction snapshot encoding.
+ */
+public class SnapshotCodecProvider implements SnapshotCodec {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SnapshotCodecProvider.class);
+
+ private final SortedMap<Integer, SnapshotCodec> codecs = Maps.newTreeMap();
+
+ @Inject
+ public SnapshotCodecProvider(Configuration configuration) {
+ initialize(configuration);
+ }
+
+ /**
+ * Register all codec specified in the configuration with this provider.
+ * There can only be one codec for a given version.
+ */
+ private void initialize(Configuration configuration) {
+ String[] codecClassNames = configuration.getTrimmedStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ List<Class> codecClasses = Lists.newArrayList();
+ if (codecClassNames != null) {
+ for (String clsName : codecClassNames) {
+ try {
+ codecClasses.add(Class.forName(clsName));
+ } catch (ClassNotFoundException cnfe) {
+ LOG.warn("Unable to load class configured for " + TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES
+ + ": " + clsName, cnfe);
+ }
+ }
+ }
+
+ if (codecClasses.size() == 0) {
+ codecClasses.addAll(Arrays.asList(TxConstants.Persist.DEFAULT_TX_SNAPHOT_CODEC_CLASSES));
+ }
+ for (Class<?> codecClass : codecClasses) {
+ try {
+ SnapshotCodec codec = (SnapshotCodec) (codecClass.newInstance());
+ codecs.put(codec.getVersion(), codec);
+ LOG.debug("Using snapshot codec {} for snapshots of version {}", codecClass.getName(), codec.getVersion());
+ } catch (Exception e) {
+ LOG.warn("Error instantiating snapshot codec {}. Skipping.", codecClass.getName(), e);
+ }
+ }
+ }
+
+ /**
+ * Retrieve the codec for a particular version of the encoding.
+ * @param version the version of interest
+ * @return the corresponding codec
+ * @throws java.lang.IllegalArgumentException if the version is not known
+ */
+ @Nonnull
+ @VisibleForTesting
+ SnapshotCodec getCodecForVersion(int version) {
+ SnapshotCodec codec = codecs.get(version);
+ if (codec == null) {
+ throw new IllegalArgumentException(String.format("Version %d of snapshot encoding is not supported", version));
+ }
+ return codec;
+ }
+
+ /**
+ * Retrieve the current snapshot codec, that is, the codec with the highest known version.
+ * @return the current codec
+ * @throws java.lang.IllegalStateException if no codecs are registered
+ */
+ private SnapshotCodec getCurrentCodec() {
+ if (codecs.isEmpty()) {
+ throw new IllegalStateException(String.format("No codecs are registered."));
+ }
+ return codecs.get(codecs.lastKey());
+ }
+
+ // Return the appropriate codec for the version in InputStream
+ private SnapshotCodec getCodec(InputStream in) {
+ BinaryDecoder decoder = new BinaryDecoder(in);
+ int persistedVersion;
+ try {
+ persistedVersion = decoder.readInt();
+ } catch (IOException e) {
+ LOG.error("Unable to read transaction state version: ", e);
+ throw Throwables.propagate(e);
+ }
+ return getCodecForVersion(persistedVersion);
+ }
+
+ @Override
+ public int getVersion() {
+ return getCurrentCodec().getVersion();
+ }
+
+ @Override
+ public TransactionSnapshot decode(InputStream in) {
+ return getCodec(in).decode(in);
+ }
+
+ @Override
+ public TransactionVisibilityState decodeTransactionVisibilityState(InputStream in) {
+ return getCodec(in).decodeTransactionVisibilityState(in);
+ }
+
+ @Override
+ public void encode(OutputStream out, TransactionSnapshot snapshot) {
+ SnapshotCodec codec = getCurrentCodec();
+ try {
+ new BinaryEncoder(out).writeInt(codec.getVersion());
+ } catch (IOException e) {
+ LOG.error("Unable to write transaction state version: ", e);
+ throw Throwables.propagate(e);
+ }
+ codec.encode(out, snapshot);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV2.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV2.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV2.java
new file mode 100644
index 0000000..ccf026d
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV2.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import com.google.common.collect.Maps;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.persist.TransactionSnapshot;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+/**
+ * Handles serialization/deserialization of a {@link TransactionSnapshot}
+ * and its elements to {@code byte[]}.
+ */
+public class SnapshotCodecV2 extends DefaultSnapshotCodec {
+ @Override
+ public int getVersion() {
+ return 2;
+ }
+
+ @Override
+ protected void encodeInProgress(BinaryEncoder encoder, Map<Long, TransactionManager.InProgressTx> inProgress)
+ throws IOException {
+
+ if (!inProgress.isEmpty()) {
+ encoder.writeInt(inProgress.size());
+ for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
+ encoder.writeLong(entry.getKey()); // tx id
+ encoder.writeLong(entry.getValue().getExpiration());
+ encoder.writeLong(entry.getValue().getVisibilityUpperBound());
+ encoder.writeInt(entry.getValue().getType().ordinal());
+ }
+ }
+ encoder.writeInt(0); // zero denotes end of list as per AVRO spec
+ }
+
+ @Override
+ protected NavigableMap<Long, TransactionManager.InProgressTx> decodeInProgress(BinaryDecoder decoder)
+ throws IOException {
+
+ int size = decoder.readInt();
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
+ while (size != 0) { // zero denotes end of list as per AVRO spec
+ for (int remaining = size; remaining > 0; --remaining) {
+ long txId = decoder.readLong();
+ long expiration = decoder.readLong();
+ long visibilityUpperBound = decoder.readLong();
+ int txTypeIdx = decoder.readInt();
+ TransactionType txType;
+ try {
+ txType = TransactionType.values()[txTypeIdx];
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new IOException("Type enum ordinal value is out of range: " + txTypeIdx);
+ }
+ inProgress.put(txId,
+ new TransactionManager.InProgressTx(visibilityUpperBound, expiration, txType,
+ new LongArrayList()));
+ }
+ size = decoder.readInt();
+ }
+ return inProgress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV3.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV3.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV3.java
new file mode 100644
index 0000000..1b9e2b3
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV3.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import org.apache.tephra.persist.TransactionSnapshot;
+
+/**
+ * Handles serialization/deserialization of a {@link TransactionSnapshot}
+ * and its elements to {@code byte[]}.
+ *
+ * <p>The serialization/deserialization of this codec is the same as that performed by {@link SnapshotCodecV2},
+ * but a new version number is used to allow easy migration from projects using deprecated codecs with
+ * conflicting version numbers.</p>
+ */
+public class SnapshotCodecV3 extends SnapshotCodecV2 {
+ @Override
+ public int getVersion() {
+ return 3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV4.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV4.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV4.java
new file mode 100644
index 0000000..cadaa8e
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/SnapshotCodecV4.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.snapshot;
+
+import com.google.common.collect.Maps;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.persist.TransactionSnapshot;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+/**
+ * Handles serialization/deserialization of a {@link TransactionSnapshot}
+ * and its elements to {@code byte[]}.
+ *
+ */
+public class SnapshotCodecV4 extends SnapshotCodecV2 {
+ @Override
+ public int getVersion() {
+ return 4;
+ }
+
+ @Override
+ protected void encodeInProgress(BinaryEncoder encoder, Map<Long, TransactionManager.InProgressTx> inProgress)
+ throws IOException {
+
+ if (!inProgress.isEmpty()) {
+ encoder.writeInt(inProgress.size());
+ for (Map.Entry<Long, TransactionManager.InProgressTx> entry : inProgress.entrySet()) {
+ encoder.writeLong(entry.getKey()); // tx id
+ encoder.writeLong(entry.getValue().getExpiration());
+ encoder.writeLong(entry.getValue().getVisibilityUpperBound());
+ encoder.writeInt(entry.getValue().getType().ordinal());
+ // write checkpoint tx IDs
+ LongArrayList checkpointPointers = entry.getValue().getCheckpointWritePointers();
+ if (checkpointPointers != null && !checkpointPointers.isEmpty()) {
+ encoder.writeInt(checkpointPointers.size());
+ for (int i = 0; i < checkpointPointers.size(); i++) {
+ encoder.writeLong(checkpointPointers.getLong(i));
+ }
+ }
+ encoder.writeInt(0);
+ }
+ }
+ encoder.writeInt(0); // zero denotes end of list as per AVRO spec
+ }
+
+ @Override
+ protected NavigableMap<Long, TransactionManager.InProgressTx> decodeInProgress(BinaryDecoder decoder)
+ throws IOException {
+
+ int size = decoder.readInt();
+ NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
+ while (size != 0) { // zero denotes end of list as per AVRO spec
+ for (int remaining = size; remaining > 0; --remaining) {
+ long txId = decoder.readLong();
+ long expiration = decoder.readLong();
+ long visibilityUpperBound = decoder.readLong();
+ int txTypeIdx = decoder.readInt();
+ TransactionType txType;
+ try {
+ txType = TransactionType.values()[txTypeIdx];
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new IOException("Type enum ordinal value is out of range: " + txTypeIdx);
+ }
+ // read checkpoint tx IDs
+ int checkpointPointerSize = decoder.readInt();
+ LongArrayList checkpointPointers = new LongArrayList(checkpointPointerSize);
+ while (checkpointPointerSize != 0) {
+ for (int checkpointRemaining = checkpointPointerSize; checkpointRemaining > 0; --checkpointRemaining) {
+ checkpointPointers.add(decoder.readLong());
+ }
+ checkpointPointerSize = decoder.readInt();
+ }
+ inProgress.put(txId,
+ new TransactionManager.InProgressTx(visibilityUpperBound, expiration, txType, checkpointPointers));
+ }
+ size = decoder.readInt();
+ }
+ return inProgress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/snapshot/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/snapshot/package-info.java b/tephra-core/src/main/java/org/apache/tephra/snapshot/package-info.java
new file mode 100644
index 0000000..cf1a276
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/snapshot/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains interfaces and implementations for encoding and decoding transaction snapshots.
+ */
+package org.apache.tephra.snapshot;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationFactory.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationFactory.java b/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationFactory.java
new file mode 100644
index 0000000..a13668c
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationFactory.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import com.google.inject.Provider;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Provides {@code org.apache.hadoop.conf.Configuration} instances, constructed by the correct version used
+ * for the runtime.
+ */
+public class ConfigurationFactory implements Provider<Configuration> {
+ private static class ConfigurationProviderFactory extends HBaseVersionSpecificFactory<ConfigurationProvider> {
+ @Override
+ protected String getHBase96Classname() {
+ return "org.apache.tephra.hbase96.HBase96ConfigurationProvider";
+ }
+
+ @Override
+ protected String getHBase98Classname() {
+ return "org.apache.tephra.hbase98.HBase98ConfigurationProvider";
+ }
+
+ @Override
+ protected String getHBase10Classname() {
+ return "org.apache.tephra.hbase10.HBase10ConfigurationProvider";
+ }
+
+ @Override
+ protected String getHBase11Classname() {
+ return "org.apache.tephra.hbase11.HBase11ConfigurationProvider";
+ }
+
+ @Override
+ protected String getHBase10CDHClassname() {
+ return "org.apache.tephra.hbase10cdh.HBase10ConfigurationProvider";
+ }
+ }
+
+ private final ConfigurationProvider provider = new ConfigurationProviderFactory().get();
+
+ /**
+ * Returns a new {@link org.apache.hadoop.conf.Configuration} instance from the HBase version-specific factory.
+ */
+ @Override
+ public Configuration get() {
+ return provider.get();
+ }
+
+ /**
+ * Returns a new {@link org.apache.hadoop.conf.Configuration} instance from the HBase version-specific factory.
+ *
+ * @param baseConf additional configuration properties to merge on to the classpath configuration
+ * @return the merged configuration
+ */
+ public Configuration get(Configuration baseConf) {
+ return provider.get(baseConf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationProvider.java b/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationProvider.java
new file mode 100644
index 0000000..e133c69
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/util/ConfigurationProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import com.google.inject.Provider;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Provides {@code Configuration} instances, constructed by the HBase version on which we are running.
+ */
+public abstract class ConfigurationProvider implements Provider<Configuration> {
+ @Override
+ public abstract Configuration get();
+
+ public abstract Configuration get(Configuration baseConf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersion.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersion.java b/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersion.java
new file mode 100644
index 0000000..687e46d
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/util/HBaseVersion.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Method;
+import java.text.ParseException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Detects the currently loaded HBase version. It is assumed that only one HBase version is loaded at a time,
+ * since using more than one HBase version within the same process will require classloader isolation anyway.
+ */
+public class HBaseVersion {
+ private static final String HBASE_94_VERSION = "0.94";
+ private static final String HBASE_96_VERSION = "0.96";
+ private static final String HBASE_98_VERSION = "0.98";
+ private static final String HBASE_10_VERSION = "1.0";
+ private static final String HBASE_11_VERSION = "1.1";
+ private static final String HBASE_12_VERSION = "1.2";
+ private static final String CDH_CLASSIFIER = "cdh";
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseVersion.class);
+
+ /**
+ * Represents the major version of the HBase library that is currently loaded.
+ */
+ public enum Version {
+ HBASE_94("0.94"),
+ HBASE_96("0.96"),
+ HBASE_98("0.98"),
+ HBASE_10("1.0"),
+ HBASE_10_CDH("1.0-cdh"),
+ HBASE_11("1.1"),
+ HBASE_12_CDH("1.2-cdh"),
+ UNKNOWN("unknown");
+
+ final String majorVersion;
+
+ Version(String majorVersion) {
+ this.majorVersion = majorVersion;
+ }
+
+ public String getMajorVersion() {
+ return majorVersion;
+ }
+ }
+
+ private static Version currentVersion;
+ private static String versionString;
+ static {
+ try {
+ Class versionInfoClass = Class.forName("org.apache.hadoop.hbase.util.VersionInfo");
+ Method versionMethod = versionInfoClass.getMethod("getVersion");
+ versionString = (String) versionMethod.invoke(null);
+ if (versionString.startsWith(HBASE_94_VERSION)) {
+ currentVersion = Version.HBASE_94;
+ } else if (versionString.startsWith(HBASE_96_VERSION)) {
+ currentVersion = Version.HBASE_96;
+ } else if (versionString.startsWith(HBASE_98_VERSION)) {
+ currentVersion = Version.HBASE_98;
+ } else if (versionString.startsWith(HBASE_10_VERSION)) {
+ VersionNumber ver = VersionNumber.create(versionString);
+ if (ver.getClassifier() != null && ver.getClassifier().startsWith(CDH_CLASSIFIER)) {
+ currentVersion = Version.HBASE_10_CDH;
+ } else {
+ currentVersion = Version.HBASE_10;
+ }
+ } else if (versionString.startsWith(HBASE_11_VERSION)) {
+ currentVersion = Version.HBASE_11;
+ } else if (versionString.startsWith(HBASE_12_VERSION)) {
+ VersionNumber ver = VersionNumber.create(versionString);
+ if (ver.getClassifier() != null && ver.getClassifier().startsWith(CDH_CLASSIFIER)) {
+ currentVersion = Version.HBASE_12_CDH;
+ } else {
+ // CDH 5.7 comes with HBase version 1.2.0-CDH5.7.0. However currently there is no
+ // other hadoop distribution that uses HBase 1.2, so the version is set here to UNKNOWN.
+ currentVersion = Version.UNKNOWN;
+ }
+ } else {
+ currentVersion = Version.UNKNOWN;
+ }
+ } catch (Throwable e) {
+ // must be a class loading exception, HBase is not there
+ LOG.error("Unable to determine HBase version from string '{}', are HBase classes available?", versionString);
+ LOG.error("Exception was: ", e);
+ currentVersion = Version.UNKNOWN;
+ }
+ }
+
+ /**
+ * Returns the major version of the currently loaded HBase library.
+ */
+ public static Version get() {
+ return currentVersion;
+ }
+
+ /**
+ * Returns the full version string for the currently loaded HBase library.
+ */
+ public static String getVersionString() {
+ return versionString;
+ }
+
+ /**
+ * Prints out the HBase {@link Version} enum value for the current version of HBase on the classpath.
+ */
+ public static void main(String[] args) {
+ boolean verbose = args.length == 1 && "-v".equals(args[0]);
+ Version version = HBaseVersion.get();
+ System.out.println(version.getMajorVersion());
+ if (verbose) {
+ System.out.println("versionString=" + getVersionString());
+ }
+ }
+
+ /**
+ * Utility class to parse apart version number components. The version string provided is expected to be in
+ * the format: major[.minor[.patch[.last]][-classifier][-SNAPSHOT]
+ *
+ * <p>Only the major version number is actually required.</p>
+ */
+ public static class VersionNumber {
+ private static final Pattern PATTERN =
+ Pattern.compile("(\\d+)(\\.(\\d+))?(\\.(\\d+))?(\\.(\\d+))?(\\-(?!SNAPSHOT)([^\\-]+))?(\\-SNAPSHOT)?");
+
+ private Integer major;
+ private Integer minor;
+ private Integer patch;
+ private Integer last;
+ private String classifier;
+ private boolean snapshot;
+
+ private VersionNumber(Integer major, Integer minor, Integer patch, Integer last,
+ String classifier, boolean snapshot) {
+ this.major = major;
+ this.minor = minor;
+ this.patch = patch;
+ this.last = last;
+ this.classifier = classifier;
+ this.snapshot = snapshot;
+ }
+
+ public Integer getMajor() {
+ return major;
+ }
+
+ public Integer getMinor() {
+ return minor;
+ }
+
+ public Integer getPatch() {
+ return patch;
+ }
+
+ public Integer getLast() {
+ return last;
+ }
+
+ public String getClassifier() {
+ return classifier;
+ }
+
+ public boolean isSnapshot() {
+ return snapshot;
+ }
+
+ public static VersionNumber create(String versionString) throws ParseException {
+ Matcher matcher = PATTERN.matcher(versionString);
+ if (matcher.matches()) {
+ String majorString = matcher.group(1);
+ String minorString = matcher.group(3);
+ String patchString = matcher.group(5);
+ String last = matcher.group(7);
+ String classifier = matcher.group(9);
+ String snapshotString = matcher.group(10);
+ return new VersionNumber(new Integer(majorString),
+ minorString != null ? new Integer(minorString) : null,
+ patchString != null ? new Integer(patchString) : null,
+ last != null ? new Integer(last) : null,
+ classifier,
+ "-SNAPSHOT".equals(snapshotString));
+ }
+ throw new ParseException(
+ "Input string did not match expected pattern: major[.minor[.patch]][-classifier][-SNAPSHOT]", 0);
+ }
+ }
+}
[53/56] [abbrv] incubator-tephra git commit: Update operation
attribute key names
Posted by po...@apache.org.
Update operation attribute key names
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/d038cb30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/d038cb30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/d038cb30
Branch: refs/heads/master
Commit: d038cb300ac603ae103eb1ae060aa84e3e9d7eb6
Parents: 26ca8e1
Author: poorna <po...@cask.co>
Authored: Fri May 6 17:24:45 2016 -0700
Committer: poorna <po...@cask.co>
Committed: Fri May 6 17:24:45 2016 -0700
----------------------------------------------------------------------
tephra-core/src/main/java/org/apache/tephra/TxConstants.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/d038cb30/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TxConstants.java b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
index 7f7fc8c..d962486 100644
--- a/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
+++ b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
@@ -86,13 +86,13 @@ public class TxConstants {
/**
* Key used to set the serialized transaction as an attribute on Get and Scan operations.
*/
- public static final String TX_OPERATION_ATTRIBUTE_KEY = "cask.tx";
+ public static final String TX_OPERATION_ATTRIBUTE_KEY = "tephra.tx";
/**
* Key used to flag a delete operation as part of a transaction rollback. This is used so that the
* {@code TransactionProcessor} coprocessor loaded on a table can differentiate between deletes issued
* as part of a normal client operation versus those performed when rolling back a transaction.
*/
- public static final String TX_ROLLBACK_ATTRIBUTE_KEY = "cask.tx.rollback";
+ public static final String TX_ROLLBACK_ATTRIBUTE_KEY = "tephra.tx.rollback";
/**
* Column qualifier used for a special delete marker tombstone, which identifies an entire column family as deleted.
[54/56] [abbrv] incubator-tephra git commit: Update mailing list
information
Posted by po...@apache.org.
Update mailing list information
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/6cc3adbc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/6cc3adbc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/6cc3adbc
Branch: refs/heads/master
Commit: 6cc3adbcaa72bd7b44ec9ebdc6a269b1102898de
Parents: d038cb3
Author: poorna <po...@cask.co>
Authored: Fri May 6 17:46:30 2016 -0700
Committer: poorna <po...@cask.co>
Committed: Fri May 6 17:46:30 2016 -0700
----------------------------------------------------------------------
README.rst | 21 ++++++++++-----------
pom.xml | 4 ++--
2 files changed, 12 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/6cc3adbc/README.rst
----------------------------------------------------------------------
diff --git a/README.rst b/README.rst
index df097a4..da3621b 100644
--- a/README.rst
+++ b/README.rst
@@ -637,21 +637,20 @@ Bug Reports & Feature Requests
Bugs and tasks are tracked in a public JIRA `issue tracker <https://issues.cask.co/browse/TEPHRA>`__.
-Tephra User Groups and Mailing Lists
-....................................
+Apache Tephra User Groups and Mailing Lists
+...........................................
-- Tephra User Group: `tephra-user@googlegroups.com <https://groups.google.com/d/forum/tephra-user>`__
+- Apache Tephra Mailing List: `dev@tephra.incubator.apache.org <https://mail-archives.apache.org/mod_mbox/tephra-dev>`__
- The *tephra-user* mailing list is primarily for users using the product to develop
- applications. You can expect questions from users, release announcements, and any other
- discussions that we think will be helpful to the users.
+ The mailing list is for both users and developers of Apache Tephra.
+ You can expect questions from users, release announcements, and any other
+ discussions that we think will be helpful to the users. In addition, this list
+ will be used for all our design, architecture and technical discussions.
-- Tephra Developer Group and Development Discussions: `tephra-dev@googlegroups.com
- <https://groups.google.com/d/forum/tephra-dev>`__
+- Apache Tephra Commits Mailing List: `commits@tephra.incubator.apache.org
+ <https://mail-archives.apache.org/mod_mbox/tephra-commits>`__
- The *tephra-dev* mailing list is essentially for developers actively working on the
- product, and should be used for all our design, architecture and technical discussions
- moving forward. This mailing list will also receive all JIRA and GitHub notifications.
+ This mailing list will receive all Git commit notifications.
IRC
...
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/6cc3adbc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ca987c0..839d656 100644
--- a/pom.xml
+++ b/pom.xml
@@ -178,7 +178,7 @@
<post>dev@tephra.incubator.apache.org</post>
<subscribe>dev-subscribe@tephra.incubator.apache.org</subscribe>
<unsubscribe>dev-unsubscribe@tephra.incubator.apache.org</unsubscribe>
- <archive>http://mail-archives.apache.org/mod_mbox/tephra-dev/</archive>
+ <archive>https://mail-archives.apache.org/mod_mbox/tephra-dev/</archive>
<otherArchives>
<otherArchive>http://tephra-dev.markmail.org</otherArchive>
</otherArchives>
@@ -188,7 +188,7 @@
<post>commits@tephra.incubator.apache.org</post>
<subscribe>commits-subscribe@tephra.incubator.apache.org</subscribe>
<unsubscribe>commits-unsubscribe@tephra.incubator.apache.org</unsubscribe>
- <archive>http://mail-archives.apache.org/mod_mbox/tephra-commits/</archive>
+ <archive>https://mail-archives.apache.org/mod_mbox/tephra-commits/</archive>
<otherArchives>
<otherArchive>http://tephra-commits.markmail.org</otherArchive>
</otherArchives>
[46/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java
deleted file mode 100644
index e7119b6..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionCouldNotTakeSnapshotException.java
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TTransactionCouldNotTakeSnapshotException extends TException implements org.apache.thrift.TBase<TTransactionCouldNotTakeSnapshotException, TTransactionCouldNotTakeSnapshotException._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTransactionCouldNotTakeSnapshotException");
-
- private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TTransactionCouldNotTakeSnapshotExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TTransactionCouldNotTakeSnapshotExceptionTupleSchemeFactory());
- }
-
- public String message; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- MESSAGE((short)1, "message");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // MESSAGE
- return MESSAGE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTransactionCouldNotTakeSnapshotException.class, metaDataMap);
- }
-
- public TTransactionCouldNotTakeSnapshotException() {
- }
-
- public TTransactionCouldNotTakeSnapshotException(
- String message)
- {
- this();
- this.message = message;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TTransactionCouldNotTakeSnapshotException(TTransactionCouldNotTakeSnapshotException other) {
- if (other.isSetMessage()) {
- this.message = other.message;
- }
- }
-
- public TTransactionCouldNotTakeSnapshotException deepCopy() {
- return new TTransactionCouldNotTakeSnapshotException(this);
- }
-
- @Override
- public void clear() {
- this.message = null;
- }
-
- public String getMessage() {
- return this.message;
- }
-
- public TTransactionCouldNotTakeSnapshotException setMessage(String message) {
- this.message = message;
- return this;
- }
-
- public void unsetMessage() {
- this.message = null;
- }
-
- /** Returns true if field message is set (has been assigned a value) and false otherwise */
- public boolean isSetMessage() {
- return this.message != null;
- }
-
- public void setMessageIsSet(boolean value) {
- if (!value) {
- this.message = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case MESSAGE:
- if (value == null) {
- unsetMessage();
- } else {
- setMessage((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case MESSAGE:
- return getMessage();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case MESSAGE:
- return isSetMessage();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TTransactionCouldNotTakeSnapshotException)
- return this.equals((TTransactionCouldNotTakeSnapshotException)that);
- return false;
- }
-
- public boolean equals(TTransactionCouldNotTakeSnapshotException that) {
- if (that == null)
- return false;
-
- boolean this_present_message = true && this.isSetMessage();
- boolean that_present_message = true && that.isSetMessage();
- if (this_present_message || that_present_message) {
- if (!(this_present_message && that_present_message))
- return false;
- if (!this.message.equals(that.message))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(TTransactionCouldNotTakeSnapshotException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- TTransactionCouldNotTakeSnapshotException typedOther = (TTransactionCouldNotTakeSnapshotException)other;
-
- lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMessage()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TTransactionCouldNotTakeSnapshotException(");
- boolean first = true;
-
- sb.append("message:");
- if (this.message == null) {
- sb.append("null");
- } else {
- sb.append(this.message);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TTransactionCouldNotTakeSnapshotExceptionStandardSchemeFactory implements SchemeFactory {
- public TTransactionCouldNotTakeSnapshotExceptionStandardScheme getScheme() {
- return new TTransactionCouldNotTakeSnapshotExceptionStandardScheme();
- }
- }
-
- private static class TTransactionCouldNotTakeSnapshotExceptionStandardScheme extends StandardScheme<TTransactionCouldNotTakeSnapshotException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // MESSAGE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.message = iprot.readString();
- struct.setMessageIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.message != null) {
- oprot.writeFieldBegin(MESSAGE_FIELD_DESC);
- oprot.writeString(struct.message);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TTransactionCouldNotTakeSnapshotExceptionTupleSchemeFactory implements SchemeFactory {
- public TTransactionCouldNotTakeSnapshotExceptionTupleScheme getScheme() {
- return new TTransactionCouldNotTakeSnapshotExceptionTupleScheme();
- }
- }
-
- private static class TTransactionCouldNotTakeSnapshotExceptionTupleScheme extends TupleScheme<TTransactionCouldNotTakeSnapshotException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetMessage()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetMessage()) {
- oprot.writeString(struct.message);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TTransactionCouldNotTakeSnapshotException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.message = iprot.readString();
- struct.setMessageIsSet(true);
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionNotInProgressException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionNotInProgressException.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionNotInProgressException.java
deleted file mode 100644
index b74d990..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionNotInProgressException.java
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TTransactionNotInProgressException extends TException implements org.apache.thrift.TBase<TTransactionNotInProgressException, TTransactionNotInProgressException._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTransactionNotInProgressException");
-
- private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new TTransactionNotInProgressExceptionStandardSchemeFactory());
- schemes.put(TupleScheme.class, new TTransactionNotInProgressExceptionTupleSchemeFactory());
- }
-
- public String message; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- MESSAGE((short)1, "message");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // MESSAGE
- return MESSAGE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTransactionNotInProgressException.class, metaDataMap);
- }
-
- public TTransactionNotInProgressException() {
- }
-
- public TTransactionNotInProgressException(
- String message)
- {
- this();
- this.message = message;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public TTransactionNotInProgressException(TTransactionNotInProgressException other) {
- if (other.isSetMessage()) {
- this.message = other.message;
- }
- }
-
- public TTransactionNotInProgressException deepCopy() {
- return new TTransactionNotInProgressException(this);
- }
-
- @Override
- public void clear() {
- this.message = null;
- }
-
- public String getMessage() {
- return this.message;
- }
-
- public TTransactionNotInProgressException setMessage(String message) {
- this.message = message;
- return this;
- }
-
- public void unsetMessage() {
- this.message = null;
- }
-
- /** Returns true if field message is set (has been assigned a value) and false otherwise */
- public boolean isSetMessage() {
- return this.message != null;
- }
-
- public void setMessageIsSet(boolean value) {
- if (!value) {
- this.message = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case MESSAGE:
- if (value == null) {
- unsetMessage();
- } else {
- setMessage((String)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case MESSAGE:
- return getMessage();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case MESSAGE:
- return isSetMessage();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof TTransactionNotInProgressException)
- return this.equals((TTransactionNotInProgressException)that);
- return false;
- }
-
- public boolean equals(TTransactionNotInProgressException that) {
- if (that == null)
- return false;
-
- boolean this_present_message = true && this.isSetMessage();
- boolean that_present_message = true && that.isSetMessage();
- if (this_present_message || that_present_message) {
- if (!(this_present_message && that_present_message))
- return false;
- if (!this.message.equals(that.message))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(TTransactionNotInProgressException other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- TTransactionNotInProgressException typedOther = (TTransactionNotInProgressException)other;
-
- lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetMessage()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TTransactionNotInProgressException(");
- boolean first = true;
-
- sb.append("message:");
- if (this.message == null) {
- sb.append("null");
- } else {
- sb.append(this.message);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class TTransactionNotInProgressExceptionStandardSchemeFactory implements SchemeFactory {
- public TTransactionNotInProgressExceptionStandardScheme getScheme() {
- return new TTransactionNotInProgressExceptionStandardScheme();
- }
- }
-
- private static class TTransactionNotInProgressExceptionStandardScheme extends StandardScheme<TTransactionNotInProgressException> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // MESSAGE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.message = iprot.readString();
- struct.setMessageIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.message != null) {
- oprot.writeFieldBegin(MESSAGE_FIELD_DESC);
- oprot.writeString(struct.message);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class TTransactionNotInProgressExceptionTupleSchemeFactory implements SchemeFactory {
- public TTransactionNotInProgressExceptionTupleScheme getScheme() {
- return new TTransactionNotInProgressExceptionTupleScheme();
- }
- }
-
- private static class TTransactionNotInProgressExceptionTupleScheme extends TupleScheme<TTransactionNotInProgressException> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetMessage()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetMessage()) {
- oprot.writeString(struct.message);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, TTransactionNotInProgressException struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.message = iprot.readString();
- struct.setMessageIsSet(true);
- }
- }
- }
-
-}
-
[12/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilter.java
deleted file mode 100644
index 21d8f1f..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/CellSkipFilter.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
- * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
- * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
- * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
- * Please see TEPHRA-169 for more details.
- */
-public class CellSkipFilter extends FilterBase {
- private final Filter filter;
- // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
- private KeyValue skipColumn = null;
-
- public CellSkipFilter(Filter filter) {
- this.filter = filter;
- }
-
- /**
- * Determines whether the current cell should be skipped. The cell will be skipped
- * if the previous keyvalue had the same key as the current cell. This means filter already responded
- * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
- * @param cell the {@link Cell} to be tested for skipping
- * @return true is current cell should be skipped, false otherwise
- */
- private boolean skipCellVersion(Cell cell) {
- return skipColumn != null
- && CellUtil.matchingRow(cell, skipColumn.getRowArray(), skipColumn.getRowOffset(),
- skipColumn.getRowLength())
- && CellUtil.matchingFamily(cell, skipColumn.getFamilyArray(), skipColumn.getFamilyOffset(),
- skipColumn.getFamilyLength())
- && CellUtil.matchingQualifier(cell, skipColumn.getQualifierArray(), skipColumn.getQualifierOffset(),
- skipColumn.getQualifierLength());
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (skipCellVersion(cell)) {
- return ReturnCode.NEXT_COL;
- }
-
- ReturnCode code = filter.filterKeyValue(cell);
- if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
- // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
- skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
- cell.getFamilyArray(), cell.getFamilyOffset(),
- cell.getFamilyLength(), cell.getQualifierArray(),
- cell.getQualifierOffset(), cell.getQualifierLength());
- } else {
- skipColumn = null;
- }
- return code;
- }
-
- @Override
- public boolean filterRow() throws IOException {
- return filter.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- return filter.transformCell(cell);
- }
-
- @Override
- public void reset() throws IOException {
- filter.reset();
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- return filter.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- return filter.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- filter.filterRowCells(kvs);
- }
-
- @Override
- public boolean hasFilterRow() {
- return filter.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- return filter.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- return filter.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- return filter.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return filter.toByteArray();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionFilters.java
deleted file mode 100644
index 9c3400b..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionFilters.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh.coprocessor;
-
-import co.cask.tephra.Transaction;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Factory class for providing {@link Filter} instances.
- */
-public class TransactionFilters {
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
- }
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
deleted file mode 100644
index b41041d..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCodec;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreScanner;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-
-/**
- * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
- * for transactions:
- * <ul>
- * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
- * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
- * or expired due to TTL.</li>
- * </ul>
- *
- * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
- * or on all user tables by adding the following to hbase-site.xml:
- * {@code
- * <property>
- * <name>hbase.coprocessor.region.classes</name>
- * <value>TransactionProcessor</value>
- * </property>
- * }
- * </p>
- *
- * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
- * as an attribute:
- * {@code
- * Transaction t = ...;
- * Get get = new Get(...);
- * TransactionCodec codec = new TransactionCodec();
- * codec.addToOperation(get, t);
- * }
- * </p>
- */
-public class TransactionProcessor extends BaseRegionObserver {
- private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
-
- private TransactionStateCache cache;
- private final TransactionCodec txCodec;
- protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
- protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
-
- public TransactionProcessor() {
- this.txCodec = new TransactionCodec();
- }
-
- /* RegionObserver implementation */
-
- @Override
- public void start(CoprocessorEnvironment e) throws IOException {
- if (e instanceof RegionCoprocessorEnvironment) {
- RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
- Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
- this.cache = cacheSupplier.get();
-
- HTableDescriptor tableDesc = env.getRegion().getTableDesc();
- for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
- String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
- long ttl = 0;
- if (columnTTL != null) {
- try {
- ttl = Long.parseLong(columnTTL);
- LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
- } catch (NumberFormatException nfe) {
- LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
- ", value = " + columnTTL);
- }
- }
- ttlByFamily.put(columnDesc.getName(), ttl);
- }
-
- this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
- TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
- this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
- if (readNonTxnData) {
- LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
- }
- }
- }
-
- protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
- return new TransactionStateCacheSupplier(env.getConfiguration());
- }
-
- @Override
- public void stop(CoprocessorEnvironment e) throws IOException {
- // nothing to do
- }
-
- @Override
- public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
- throws IOException {
- Transaction tx = getFromOperation(get);
- if (tx != null) {
- projectFamilyDeletes(get);
- get.setMaxVersions();
- get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
- get.setFilter(newFilter);
- }
- }
-
- @Override
- public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
- Durability durability) throws IOException {
- // Translate deletes into our own delete tombstones
- // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
- // us to rollback the changes (by a real delete) if the transaction fails
-
- // Deletes that are part of a transaction rollback do not need special handling.
- // They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
- return;
- }
-
- // Other deletes are client-initiated and need to be translated into our own tombstones
- // TODO: this should delegate to the DeleteStrategy implementation.
- Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
- for (byte[] family : delete.getFamilyCellMap().keySet()) {
- List<Cell> familyCells = delete.getFamilyCellMap().get(family);
- if (isFamilyDelete(familyCells)) {
- deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- } else {
- int cellSize = familyCells.size();
- for (int i = 0; i < cellSize; i++) {
- Cell cell = familyCells.get(i);
- deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
- }
- e.getEnvironment().getRegion().put(deleteMarkers);
- // skip normal delete handling
- e.bypass();
- }
-
- private boolean isFamilyDelete(List<Cell> familyCells) {
- return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
- }
-
- @Override
- public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
- throws IOException {
- Transaction tx = getFromOperation(scan);
- if (tx != null) {
- projectFamilyDeletes(scan);
- scan.setMaxVersions();
- scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
- scan.setFilter(newFilter);
- }
- return s;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any scan operation.
- * @param scan The original scan request
- * @return The modified scan request with the family delete qualifiers represented
- */
- private Scan projectFamilyDeletes(Scan scan) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return scan;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any get operation.
- * @param get The original get request
- * @return The modified get request with the family delete qualifiers represented
- */
- private Get projectFamilyDeletes(Get get) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return get;
- }
-
- @Override
- public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- KeyValueScanner memstoreScanner, InternalScanner scanner)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
- Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
- HConstants.OLDEST_TIMESTAMP);
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
- CompactionRequest request)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners,
- scanType, earliestPutTs);
- }
-
- protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
- TransactionVisibilityState snapshot, Store store,
- List<? extends KeyValueScanner> scanners, ScanType type,
- long earliestPutTs) throws IOException {
- if (snapshot == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
- ", no current transaction state found, defaulting to normal " + action + " scanner");
- }
- return null;
- }
-
- // construct a dummy transaction from the latest snapshot
- Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
- Scan scan = new Scan();
- // need to see all versions, since we filter out excludes and applications may rely on multiple versions
- scan.setMaxVersions();
- scan.setFilter(
- new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
- snapshot.getInvalid(),
- getTransactionFilter(dummyTx, type, null)));
-
- return new StoreScanner(store, store.getScanInfo(), scan, scanners,
- type, store.getSmallestReadPoint(), earliestPutTs);
- }
-
- private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
- byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
- if (encoded != null) {
- return txCodec.decode(encoded);
- }
- return null;
- }
-
- /**
- * Derived classes can override this method to customize the filter used to return data visible for the current
- * transaction.
- *
- * @param tx the current transaction to apply
- * @param type the type of scan being performed
- */
- protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
- return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
- }
-
- /**
- * Filter used to include cells visible to in-progress transactions on flush and commit.
- */
- static class IncludeInProgressFilter extends FilterBase {
- private final long visibilityUpperBound;
- private final Set<Long> invalidIds;
- private final Filter txFilter;
-
- public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
- this.visibilityUpperBound = upperBound;
- this.invalidIds = Sets.newHashSet(invalids);
- this.txFilter = transactionFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- // include all cells visible to in-progress transactions, except for those already marked as invalid
- long ts = cell.getTimestamp();
- if (ts > visibilityUpperBound) {
- // include everything that could still be in-progress except invalids
- if (invalidIds.contains(ts)) {
- return ReturnCode.SKIP;
- }
- return ReturnCode.INCLUDE;
- }
- return txFilter.filterKeyValue(cell);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java
deleted file mode 100644
index 427e1bd..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Applies filtering of data based on transactional visibility (HBase 1.0+ specific version).
- * Note: this is intended for server-side use only, as additional properties need to be set on
- * any {@code Scan} or {@code Get} operation performed.
- */
-public class TransactionVisibilityFilter extends FilterBase {
- private final Transaction tx;
- // oldest visible timestamp by column family, used to apply TTL when reading
- private final Map<ImmutableBytesWritable, Long> oldestTsByFamily;
- // if false, empty values will be interpreted as deletes
- private final boolean allowEmptyValues;
- // whether or not we can remove delete markers
- // these can only be safely removed when we are traversing all storefiles
- private final boolean clearDeletes;
- // optional sub-filter to apply to visible cells
- private final Filter cellFilter;
- // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
- private final ImmutableBytesWritable currentFamily = new ImmutableBytesWritable(HConstants.EMPTY_BYTE_ARRAY);
-
- private long currentOldestTs;
-
- private DeleteTracker deleteTracker = new DeleteTracker();
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- this(tx, ttlByFamily, allowEmptyValues, scanType, null);
- }
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- this.tx = tx;
- this.oldestTsByFamily = Maps.newTreeMap();
- for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
- long familyTTL = ttlEntry.getValue();
- oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
- familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
- }
- this.allowEmptyValues = allowEmptyValues;
- this.clearDeletes =
- scanType == ScanType.COMPACT_DROP_DELETES ||
- (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
- this.cellFilter = cellFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (!CellUtil.matchingFamily(cell, currentFamily.get(), currentFamily.getOffset(), currentFamily.getLength())) {
- // column family changed
- currentFamily.set(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
- Long familyOldestTs = oldestTsByFamily.get(currentFamily);
- currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
- deleteTracker.reset();
- }
- // need to apply TTL for the column family here
- long kvTimestamp = cell.getTimestamp();
- if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
- // passed TTL for this column, seek to next
- return ReturnCode.NEXT_COL;
- } else if (tx.isVisible(kvTimestamp)) {
- // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
- // cell is visible
- // visibility SNAPSHOT_ALL needs all matches
- return runSubFilter(ReturnCode.INCLUDE, cell);
- }
- if (DeleteTracker.isFamilyDelete(cell)) {
- deleteTracker.addFamilyDelete(cell);
- if (clearDeletes) {
- return ReturnCode.NEXT_COL;
- } else {
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // check if masked by family delete
- if (deleteTracker.isDeleted(cell)) {
- return ReturnCode.NEXT_COL;
- }
- // check for column delete
- if (isColumnDelete(cell)) {
- if (clearDeletes) {
- // skip "deleted" cell
- return ReturnCode.NEXT_COL;
- } else {
- // keep the marker but skip any remaining versions
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- } else {
- return ReturnCode.SKIP;
- }
- }
-
- private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
- if (cellFilter != null) {
- ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
- return determineReturnCode(txFilterCode, subFilterCode);
- }
- return txFilterCode;
- }
-
- /**
- * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
- * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
- * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
- * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
- *
- * @param txFilterCode return code from TransactionVisibilityFilter
- * @param subFilterCode return code from sub-filter
- * @return final return code
- */
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- // Return the more restrictive of the two filter responses
- switch (subFilterCode) {
- case INCLUDE:
- return txFilterCode;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
-
- @Override
- public boolean filterRow() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRow();
- }
- return super.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- // Convert Tephra deletes back into HBase deletes
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
- if (DeleteTracker.isFamilyDelete(cell)) {
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
- KeyValue.Type.DeleteFamily);
- } else if (isColumnDelete(cell)) {
- // Note: in some cases KeyValue.Type.Delete is used in Delete object,
- // and in some other cases KeyValue.Type.DeleteColumn is used.
- // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
- // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
- // work in both cases.
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
- cell.getTimestamp(), KeyValue.Type.DeleteColumn);
- }
- }
- return cell;
- }
-
- @Override
- public void reset() throws IOException {
- deleteTracker.reset();
- if (cellFilter != null) {
- cellFilter.reset();
- }
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRowKey(buffer, offset, length);
- }
- return super.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterAllRemaining();
- }
- return super.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- if (cellFilter != null) {
- cellFilter.filterRowCells(kvs);
- } else {
- super.filterRowCells(kvs);
- }
- }
-
- @Override
- public boolean hasFilterRow() {
- if (cellFilter != null) {
- return cellFilter.hasFilterRow();
- }
- return super.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextKeyHint(currentKV);
- }
- return super.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextCellHint(currentKV);
- }
- return super.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- if (cellFilter != null) {
- return cellFilter.isFamilyEssential(name);
- }
- return super.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return super.toByteArray();
- }
-
- private boolean isColumnDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
- }
-
- private static final class DeleteTracker {
- private long familyDeleteTs;
-
- public static boolean isFamilyDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
- CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
- CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
- }
-
- public void addFamilyDelete(Cell delete) {
- this.familyDeleteTs = delete.getTimestamp();
- }
-
- public boolean isDeleted(Cell cell) {
- return cell.getTimestamp() <= familyDeleteTs;
- }
-
- public void reset() {
- this.familyDeleteTs = 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProvider.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProvider.java
new file mode 100644
index 0000000..97f60f9
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/HBase10ConfigurationProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.tephra.util.ConfigurationProvider;
+
+/**
+ * HBase 1.0 version of {@link ConfigurationProvider}.
+ */
+public class HBase10ConfigurationProvider extends ConfigurationProvider {
+ @Override
+ public Configuration get() {
+ return HBaseConfiguration.create();
+ }
+
+ @Override
+ public Configuration get(Configuration baseConf) {
+ return HBaseConfiguration.create(baseConf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/SecondaryIndexTable.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/SecondaryIndexTable.java
new file mode 100644
index 0000000..a05cd15
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/SecondaryIndexTable.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase10cdh;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.apache.tephra.distributed.TransactionServiceClient;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A Transactional SecondaryIndexTable.
+ */
+public class SecondaryIndexTable {
+ private byte[] secondaryIndex;
+ private TransactionAwareHTable transactionAwareHTable;
+ private TransactionAwareHTable secondaryIndexTable;
+ private TransactionContext transactionContext;
+ private final TableName secondaryIndexTableName;
+ private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
+ private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
+ private static final byte[] DELIMITER = new byte[] {0};
+
+ public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
+ byte[] secondaryIndex) {
+ secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
+ HTable secondaryIndexHTable = null;
+ HBaseAdmin hBaseAdmin = null;
+ try {
+ hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
+ if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
+ hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
+ }
+ secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ } finally {
+ try {
+ hBaseAdmin.close();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ }
+
+ this.secondaryIndex = secondaryIndex;
+ this.transactionAwareHTable = new TransactionAwareHTable(hTable);
+ this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
+ this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
+ secondaryIndexTable);
+ }
+
+ public Result get(Get get) throws IOException {
+ return get(Collections.singletonList(get))[0];
+ }
+
+ public Result[] get(List<Get> gets) throws IOException {
+ try {
+ transactionContext.start();
+ Result[] result = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return result;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public Result[] getByIndex(byte[] value) throws IOException {
+ try {
+ transactionContext.start();
+ Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
+ scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
+ ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
+
+ ArrayList<Get> gets = new ArrayList<Get>();
+ for (Result result : indexScanner) {
+ for (Cell cell : result.listCells()) {
+ gets.add(new Get(cell.getValue()));
+ }
+ }
+ Result[] results = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return results;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public void put(Put put) throws IOException {
+ put(Collections.singletonList(put));
+ }
+
+
+ public void put(List<Put> puts) throws IOException {
+ try {
+ transactionContext.start();
+ ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
+ for (Put put : puts) {
+ List<Put> indexPuts = new ArrayList<Put>();
+ Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
+ for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
+ for (KeyValue value : family.getValue()) {
+ if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
+ secondaryIndex, 0, secondaryIndex.length)) {
+ byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
+ Bytes.add(value.getValue(), DELIMITER,
+ value.getRow()));
+ Put indexPut = new Put(secondaryRow);
+ indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
+ indexPuts.add(indexPut);
+ }
+ }
+ }
+ secondaryIndexPuts.addAll(indexPuts);
+ }
+ transactionAwareHTable.put(puts);
+ secondaryIndexTable.put(secondaryIndexPuts);
+ transactionContext.finish();
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java
new file mode 100644
index 0000000..ea9fdad
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java
@@ -0,0 +1,674 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase10cdh;
+
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.tephra.AbstractTransactionAwareTable;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionAware;
+import org.apache.tephra.TxConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+/**
+ * A Transaction Aware HTable implementation for HBase 1.0. Operations are committed as usual,
+ * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
+ * was started.
+ */
+public class TransactionAwareHTable extends AbstractTransactionAwareTable
+ implements HTableInterface, TransactionAware {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
+ private final HTableInterface hTable;
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ */
+ public TransactionAwareHTable(HTableInterface hTable) {
+ this(hTable, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
+ this(hTable, conflictLevel, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
+ this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
+ boolean allowNonTransactional) {
+ super(conflictLevel, allowNonTransactional);
+ this.hTable = hTable;
+ }
+
+ /* AbstractTransactionAwareTable implementation */
+
+ @Override
+ protected byte[] getTableKey() {
+ return getTableName();
+ }
+
+ @Override
+ protected boolean doCommit() throws IOException {
+ hTable.flushCommits();
+ return true;
+ }
+
+ @Override
+ protected boolean doRollback() throws Exception {
+ try {
+ // pre-size arraylist of deletes
+ int size = 0;
+ for (Set<ActionChange> cs : changeSets.values()) {
+ size += cs.size();
+ }
+ List<Delete> rollbackDeletes = new ArrayList<>(size);
+ for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
+ long transactionTimestamp = entry.getKey();
+ for (ActionChange change : entry.getValue()) {
+ byte[] row = change.getRow();
+ byte[] family = change.getFamily();
+ byte[] qualifier = change.getQualifier();
+ Delete rollbackDelete = new Delete(row);
+ rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ switch (conflictLevel) {
+ case ROW:
+ case NONE:
+ // issue family delete for the tx write pointer
+ rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
+ break;
+ case COLUMN:
+ if (family != null && qualifier == null) {
+ rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
+ } else if (family != null && qualifier != null) {
+ rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
+ }
+ rollbackDeletes.add(rollbackDelete);
+ }
+ }
+ hTable.delete(rollbackDeletes);
+ return true;
+ } finally {
+ try {
+ hTable.flushCommits();
+ } catch (Exception e) {
+ LOG.error("Could not flush HTable commits", e);
+ }
+ tx = null;
+ changeSets.clear();
+ }
+ }
+
+ /* HTableInterface implementation */
+
+ @Override
+ public byte[] getTableName() {
+ return hTable.getTableName();
+ }
+
+ @Override
+ public TableName getName() {
+ return hTable.getName();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hTable.getConfiguration();
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return hTable.getTableDescriptor();
+ }
+
+ @Override
+ public boolean exists(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.exists(transactionalizeAction(get));
+ }
+
+ @Override
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.exists(transactionalizedGets);
+ }
+
+ @Override
+ public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batch(transactionalizeActions(actions), results);
+ }
+
+ @Override
+ public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batch(transactionalizeActions(actions));
+ }
+
+ @Override
+ public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
+ IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batchCallback(transactionalizeActions(actions), results, callback);
+ }
+
+ @Override
+ public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
+ InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batchCallback(transactionalizeActions(actions), callback);
+ }
+
+ @Override
+ public Result get(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.get(transactionalizeAction(get));
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.get(transactionalizedGets);
+ }
+
+ @Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.getRowOrBefore(row, family);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan scan) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addColumn(family, qualifier);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public void put(Put put) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Put txPut = transactionalizeAction(put);
+ hTable.put(txPut);
+ }
+
+ @Override
+ public void put(List<Put> puts) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
+ for (Put put : puts) {
+ Put txPut = transactionalizeAction(put);
+ transactionalizedPuts.add(txPut);
+ }
+ hTable.put(transactionalizedPuts);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndPut(row, family, qualifier, value, put);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public void delete(Delete delete) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.delete(transactionalizeAction(delete));
+ }
+
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
+ for (Delete delete : deletes) {
+ Delete txDelete = transactionalizeAction(delete);
+ transactionalizedDeletes.add(txDelete);
+ }
+ hTable.delete(transactionalizedDeletes);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndDelete(row, family, qualifier, value, delete);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
+ byte[] bytes3, Delete delete) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndDelete(bytes, bytes1, bytes2, compareOp, bytes3, delete);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
+ byte[] bytes3, Put put) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndPut(bytes, bytes1, bytes2, compareOp, bytes3, put);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean[] existsAll(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.existsAll(transactionalizedGets);
+ }
+
+ @Override
+ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
+ CompareFilter.CompareOp compareOp, byte[] value, RowMutations rowMutations)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndMutate(row, family, qualifier, compareOp, value, rowMutations);
+ }
+
+ throw new UnsupportedOperationException("checkAndMutate operation is not supported transactionally");
+ }
+
+ @Override
+ public void mutateRow(RowMutations rm) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ RowMutations transactionalMutations = new RowMutations();
+ for (Mutation mutation : rm.getMutations()) {
+ if (mutation instanceof Put) {
+ transactionalMutations.add(transactionalizeAction((Put) mutation));
+ } else if (mutation instanceof Delete) {
+ transactionalMutations.add(transactionalizeAction((Delete) mutation));
+ }
+ }
+ hTable.mutateRow(transactionalMutations);
+ }
+
+ @Override
+ public Result append(Append append) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.append(append);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public Result increment(Increment increment) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.increment(increment);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean isAutoFlush() {
+ return hTable.isAutoFlush();
+ }
+
+ @Override
+ public void flushCommits() throws IOException {
+ hTable.flushCommits();
+ }
+
+ @Override
+ public void close() throws IOException {
+ hTable.close();
+ }
+
+ @Override
+ public CoprocessorRpcChannel coprocessorService(byte[] row) {
+ return hTable.coprocessorService(row);
+ }
+
+ @Override
+ public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable)
+ throws ServiceException, Throwable {
+ return hTable.coprocessorService(service, startKey, endKey, callable);
+ }
+
+ @Override
+ public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable, Batch.Callback<R> callback)
+ throws ServiceException, Throwable {
+ hTable.coprocessorService(service, startKey, endKey, callable, callback);
+ }
+
+ @Override
+ public <R extends Message> Map<byte[], R> batchCoprocessorService(
+ MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey,
+ R responsePrototype) throws ServiceException, Throwable {
+ return hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
+ }
+
+ @Override
+ public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
+ Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
+ throws ServiceException, Throwable {
+ hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush) {
+ setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ hTable.setAutoFlush(autoFlush, clearBufferOnFail);
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ hTable.setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public long getWriteBufferSize() {
+ return hTable.getWriteBufferSize();
+ }
+
+ @Override
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ hTable.setWriteBufferSize(writeBufferSize);
+ }
+
+ // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
+
+ private Get transactionalizeAction(Get get) throws IOException {
+ addToOperation(get, tx);
+ return get;
+ }
+
+ private Scan transactionalizeAction(Scan scan) throws IOException {
+ addToOperation(scan, tx);
+ return scan;
+ }
+
+ private Put transactionalizeAction(Put put) throws IOException {
+ Put txPut = new Put(put.getRow(), tx.getWritePointer());
+ Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
+ if (!familyMap.isEmpty()) {
+ for (Map.Entry<byte[], List<Cell>> family : familyMap) {
+ List<Cell> familyValues = family.getValue();
+ if (!familyValues.isEmpty()) {
+ for (Cell value : familyValues) {
+ txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
+ addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
+ txPut.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txPut.setDurability(put.getDurability());
+ addToOperation(txPut, tx);
+ return txPut;
+ }
+
+ private Delete transactionalizeAction(Delete delete) throws IOException {
+ long transactionTimestamp = tx.getWritePointer();
+
+ byte[] deleteRow = delete.getRow();
+ Delete txDelete = new Delete(deleteRow, transactionTimestamp);
+
+ Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
+ if (familyToDelete.isEmpty()) {
+ // perform a row delete if we are using row-level conflict detection
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // Row delete leaves delete markers in all column families of the table
+ // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
+ for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
+ // no need to identify individual columns deleted
+ addToChangeSet(deleteRow, columnDescriptor.getName(), null);
+ }
+ } else {
+ Result result = get(new Get(delete.getRow()));
+ // Delete everything
+ NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
+ for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
+ }
+ }
+ }
+ } else {
+ for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
+ byte[] family = familyEntry.getKey();
+ List<Cell> entries = familyEntry.getValue();
+ boolean isFamilyDelete = false;
+ if (entries.size() == 1) {
+ Cell cell = entries.get(0);
+ isFamilyDelete = CellUtil.isDeleteFamily(cell);
+ }
+ if (isFamilyDelete) {
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // no need to identify individual columns deleted
+ txDelete.deleteFamily(family);
+ addToChangeSet(deleteRow, family, null);
+ } else {
+ Result result = get(new Get(delete.getRow()).addFamily(family));
+ // Delete entire family
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, family, column.getKey());
+ }
+ }
+ } else {
+ for (Cell value : entries) {
+ txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
+ addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ txDelete.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txDelete.setDurability(delete.getDurability());
+ return txDelete;
+ }
+
+ private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
+ List<Row> transactionalizedActions = new ArrayList<>(actions.size());
+ for (Row action : actions) {
+ if (action instanceof Get) {
+ transactionalizedActions.add(transactionalizeAction((Get) action));
+ } else if (action instanceof Put) {
+ transactionalizedActions.add(transactionalizeAction((Put) action));
+ } else if (action instanceof Delete) {
+ transactionalizedActions.add(transactionalizeAction((Delete) action));
+ } else {
+ transactionalizedActions.add(action);
+ }
+ }
+ return transactionalizedActions;
+ }
+
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilter.java
new file mode 100644
index 0000000..49dbf09
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/CellSkipFilter.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
+ * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
+ * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
+ * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
+ * Please see TEPHRA-169 for more details.
+ */
+public class CellSkipFilter extends FilterBase {
+ private final Filter filter;
+ // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
+ private KeyValue skipColumn = null;
+
+ public CellSkipFilter(Filter filter) {
+ this.filter = filter;
+ }
+
+ /**
+ * Determines whether the current cell should be skipped. The cell will be skipped
+ * if the previous keyvalue had the same key as the current cell. This means filter already responded
+ * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
+ * @param cell the {@link Cell} to be tested for skipping
+ * @return true is current cell should be skipped, false otherwise
+ */
+ private boolean skipCellVersion(Cell cell) {
+ return skipColumn != null
+ && CellUtil.matchingRow(cell, skipColumn.getRowArray(), skipColumn.getRowOffset(),
+ skipColumn.getRowLength())
+ && CellUtil.matchingFamily(cell, skipColumn.getFamilyArray(), skipColumn.getFamilyOffset(),
+ skipColumn.getFamilyLength())
+ && CellUtil.matchingQualifier(cell, skipColumn.getQualifierArray(), skipColumn.getQualifierOffset(),
+ skipColumn.getQualifierLength());
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (skipCellVersion(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+
+ ReturnCode code = filter.filterKeyValue(cell);
+ if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
+ // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
+ skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(),
+ cell.getFamilyLength(), cell.getQualifierArray(),
+ cell.getQualifierOffset(), cell.getQualifierLength());
+ } else {
+ skipColumn = null;
+ }
+ return code;
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ return filter.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ return filter.transformCell(cell);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ filter.reset();
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ return filter.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ return filter.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ filter.filterRowCells(kvs);
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ return filter.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ return filter.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ return filter.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ return filter.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return filter.toByteArray();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionFilters.java
new file mode 100644
index 0000000..2cd7f07
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionFilters.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh.coprocessor;
+
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.tephra.Transaction;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Factory class for providing {@link Filter} instances.
+ */
+public class TransactionFilters {
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+ }
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+ }
+}
[56/56] [abbrv] incubator-tephra git commit: Update pom version
Posted by po...@apache.org.
Update pom version
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/82e70f5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/82e70f5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/82e70f5c
Branch: refs/heads/master
Commit: 82e70f5ca118a10d432491096dcebf3739b69587
Parents: e0a6e0a
Author: poorna <po...@cask.co>
Authored: Tue May 10 21:50:46 2016 -0700
Committer: poorna <po...@cask.co>
Committed: Tue May 10 21:50:46 2016 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/82e70f5c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 839d656..f85b59c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.tephra</groupId>
<artifactId>tephra</artifactId>
- <version>0.1.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Tephra</name>
<description>
[14/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java
new file mode 100644
index 0000000..908f9c8
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java
@@ -0,0 +1,1552 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase98;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.LongComparator;
+import org.apache.hadoop.hbase.filter.ValueFilter;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionConflictException;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionSystemClient;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.hbase98.coprocessor.TransactionProcessor;
+import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests for TransactionAwareHTables.
+ */
+public class TransactionAwareHTableTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
+
+ private static HBaseTestingUtility testUtil;
+ private static HBaseAdmin hBaseAdmin;
+ private static TransactionStateStorage txStateStorage;
+ private static TransactionManager txManager;
+ private static Configuration conf;
+ private TransactionContext transactionContext;
+ private TransactionAwareHTable transactionAwareHTable;
+ private HTable hTable;
+
+ private static final class TestBytes {
+ private static final byte[] table = Bytes.toBytes("testtable");
+ private static final byte[] family = Bytes.toBytes("f1");
+ private static final byte[] family2 = Bytes.toBytes("f2");
+ private static final byte[] qualifier = Bytes.toBytes("col1");
+ private static final byte[] qualifier2 = Bytes.toBytes("col2");
+ private static final byte[] row = Bytes.toBytes("row");
+ private static final byte[] row2 = Bytes.toBytes("row2");
+ private static final byte[] row3 = Bytes.toBytes("row3");
+ private static final byte[] row4 = Bytes.toBytes("row4");
+ private static final byte[] value = Bytes.toBytes("value");
+ private static final byte[] value2 = Bytes.toBytes("value2");
+ private static final byte[] value3 = Bytes.toBytes("value3");
+ }
+
+ private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
+
+ public static class TestRegionObserver extends BaseRegionObserver {
+ @Override
+ public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Put put, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (put.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Put should preserve attributes");
+ }
+ if (put.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+
+ @Override
+ public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Delete delete, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Delete should preserve attributes");
+ }
+ if (delete.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ conf = testUtil.getConfiguration();
+
+ // Tune down the connection thread pool size
+ conf.setInt("hbase.hconnection.threads.core", 5);
+ conf.setInt("hbase.hconnection.threads.max", 10);
+ // Tunn down handler threads in regionserver
+ conf.setInt("hbase.regionserver.handler.count", 10);
+
+ // Set to random port
+ conf.setInt("hbase.master.port", 0);
+ conf.setInt("hbase.master.info.port", 0);
+ conf.setInt("hbase.regionserver.port", 0);
+ conf.setInt("hbase.regionserver.info.port", 0);
+
+ testUtil.startMiniCluster();
+ hBaseAdmin = testUtil.getHBaseAdmin();
+ txStateStorage = new InMemoryTransactionStateStorage();
+ txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
+ txManager.startAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ testUtil.shutdownMiniCluster();
+ hBaseAdmin.close();
+ }
+
+ @Before
+ public void setupBeforeTest() throws Exception {
+ hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
+ transactionAwareHTable = new TransactionAwareHTable(hTable);
+ transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
+ }
+
+ @After
+ public void shutdownAfterTest() throws IOException {
+ hBaseAdmin.disableTable(TestBytes.table);
+ hBaseAdmin.deleteTable(TestBytes.table);
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
+ return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
+ List<String> coprocessors) throws Exception {
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+ for (byte[] family : columnFamilies) {
+ HColumnDescriptor columnDesc = new HColumnDescriptor(family);
+ columnDesc.setMaxVersions(Integer.MAX_VALUE);
+ columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
+ desc.addFamily(columnDesc);
+ }
+ if (existingData) {
+ desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
+ }
+ // Divide individually to prevent any overflow
+ int priority = Coprocessor.PRIORITY_USER;
+ desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
+ // order in list is the same order that coprocessors will be invoked
+ for (String coprocessor : coprocessors) {
+ desc.addCoprocessor(coprocessor, null, ++priority, null);
+ }
+ hBaseAdmin.createTable(desc);
+ testUtil.waitTableAvailable(tableName, 5000);
+ return new HTable(testUtil.getConfiguration(), tableName);
+ }
+
+ /**
+ * Test transactional put and get requests.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Test aborted put requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(value, null);
+ }
+
+ /**
+ * Test transactional delete operations.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
+ new byte[][]{TestBytes.family, TestBytes.family2});
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+
+ // test column delete
+ // load 10 rows
+ txContext.start();
+ int rowCount = 10;
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
+ }
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertFalse(r.isEmpty());
+ for (int j = 0; j < 10; j++) {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
+ }
+ }
+ txContext.finish();
+
+ // delete odds columns from odd rows and even columns from even rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ if (i % 2 == j % 2) {
+ LOG.info("Deleting row={}, column={}", i, j);
+ d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
+ }
+ }
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted columns
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertEquals(5, r.size());
+ for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
+ int col = Bytes.toInt(entry.getKey());
+ LOG.info("Got row={}, col={}", i, col);
+ // each row should only have the opposite mod (odd=even, even=odd)
+ assertNotEquals(i % 2, col % 2);
+ assertArrayEquals(TestBytes.value, entry.getValue());
+ }
+ }
+ txContext.finish();
+
+ // test family delete
+ // load 10 rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("famrow" + i));
+ p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify all loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(2, r.size());
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ txContext.finish();
+
+ // delete family1 for even rows, family2 for odd rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("famrow" + i));
+ d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted families
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(1, r.size());
+ if (i % 2 == 0) {
+ assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ } else {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ }
+ txContext.finish();
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test that put and delete attributes are preserved
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAttributesPreserved() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, false,
+ Lists.newArrayList(TestRegionObserver.class.getName()));
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ // set an attribute on the put, TestRegionObserver will verify it still exists
+ put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete, TestRegionObserver will verify it still exists
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test aborted transactional delete requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionalDelete() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ transactionAwareHTable.delete(delete);
+ transactionContext.abort();
+
+ transactionContext.start();
+ result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
+ String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
+ HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ // Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ // Start a tx, delete a column family and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
+ txContext.abort();
+
+ // Above operations should have no effect on the row, since they were aborted
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testDeleteRollback() throws Exception {
+ testDeleteRollback(TxConstants.ConflictDetection.ROW);
+ testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
+ testDeleteRollback(TxConstants.ConflictDetection.NONE);
+ }
+
+ @Test
+ public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ txContext.start();
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ Result result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+
+ //Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ //Start a tx and scan all the col families to make sure none of them have delete markers
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testRowDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][]{TestBytes.family, TestBytes.family2});
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
+ try {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Test 1: full row delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // delete entire row
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.finish();
+
+ // verify row is now empty
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for explicit column retrieval
+ result = txTable.get(new Get(TestBytes.row)
+ .addColumn(TestBytes.family, TestBytes.qualifier)
+ .addFamily(TestBytes.family2));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for scan
+ ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
+ assertNull(scanner.next());
+ scanner.close();
+
+ // verify row is empty for scan with explicit column
+ scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
+ assertNull(scanner.next());
+ scanner.close();
+ txContext.finish();
+
+ // write swapped values to one column per family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
+ txContext.finish();
+
+ // verify new values appear
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertFalse(result.isEmpty());
+ assertEquals(2, result.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+
+ scanner = txTable.getScanner(new Scan(TestBytes.row));
+ Result result1 = scanner.next();
+ assertNotNull(result1);
+ assertFalse(result1.isEmpty());
+ assertEquals(2, result1.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ scanner.close();
+ txContext.finish();
+
+ // Test 2: delete of first column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row2)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
+ txContext.finish();
+
+ txContext.start();
+ Result fam1Result = txTable.get(new Get(TestBytes.row2));
+ assertFalse(fam1Result.isEmpty());
+ assertEquals(2, fam1Result.size());
+ assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 3: delete of second column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
+ txContext.finish();
+
+ txContext.start();
+ Result fam2Result = txTable.get(new Get(TestBytes.row3));
+ assertFalse(fam2Result.isEmpty());
+ assertEquals(2, fam2Result.size());
+ assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 4: delete specific rows in a range
+ txContext.start();
+ for (int i = 0; i < 10; i++) {
+ txTable.put(new Put(Bytes.toBytes("z" + i))
+ .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
+ .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ // delete odd rows
+ for (int i = 1; i < 10; i += 2) {
+ txTable.delete(new Delete(Bytes.toBytes("z" + i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ int cnt = 0;
+ ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
+ Result res;
+ while ((res = zScanner.next()) != null) {
+ assertFalse(res.isEmpty());
+ assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
+ cnt += 2;
+ }
+
+ // Test 5: delete prior writes in the same transaction
+ txContext.start();
+ txTable.put(new Put(TestBytes.row4)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txTable.delete(new Delete(TestBytes.row4));
+ txContext.finish();
+
+ txContext.start();
+ Result row4Result = txTable.get(new Get(TestBytes.row4));
+ assertTrue(row4Result.isEmpty());
+ txContext.finish();
+ } finally {
+ txTable.close();
+ }
+ }
+
+ /**
+ * Expect an exception since a transaction hasn't been started.
+ *
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testTransactionlessFailure() throws Exception {
+ transactionAwareHTable.get(new Get(TestBytes.row));
+ }
+
+ /**
+ * Tests that each transaction can see its own persisted writes, while not seeing writes from other
+ * in-progress transactions.
+ */
+ @Test
+ public void testReadYourWrites() throws Exception {
+ // In-progress tx1: started before our main transaction
+ HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
+ TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
+
+ // In-progress tx2: started while our main transaction is running
+ HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
+ TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
+
+ // create an in-progress write that should be ignored
+ byte[] col2 = Bytes.toBytes("col2");
+ inprogressTxContext1.start();
+ Put putCol2 = new Put(TestBytes.row);
+ byte[] valueCol2 = Bytes.toBytes("writing in progress");
+ putCol2.add(TestBytes.family, col2, valueCol2);
+ txHTable1.put(putCol2);
+
+ // start a tx and write a value to test reading in same tx
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] value = Bytes.toBytes("writing");
+ put.add(TestBytes.family, TestBytes.qualifier, value);
+ transactionAwareHTable.put(put);
+
+ // test that a write from a tx started after the first is not visible
+ inprogressTxContext2.start();
+ Put put2 = new Put(TestBytes.row);
+ byte[] value2 = Bytes.toBytes("writing2");
+ put2.add(TestBytes.family, TestBytes.qualifier, value2);
+ txHTable2.put(put2);
+
+ Get get = new Get(TestBytes.row);
+ Result row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
+ Assert.assertNotNull(col1Value);
+ Assert.assertArrayEquals(value, col1Value);
+ // write from in-progress transaction should not be visible
+ byte[] col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ // commit in-progress transaction, should still not be visible
+ inprogressTxContext1.finish();
+
+ get = new Get(TestBytes.row);
+ row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ transactionContext.finish();
+
+ inprogressTxContext2.abort();
+ }
+
+ @Test
+ public void testRowLevelConflictDetection() throws Exception {
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ byte[] row1 = Bytes.toBytes("row1");
+ byte[] row2 = Bytes.toBytes("row2");
+ byte[] col1 = Bytes.toBytes("c1");
+ byte[] col2 = Bytes.toBytes("c2");
+ byte[] val1 = Bytes.toBytes("val1");
+ byte[] val2 = Bytes.toBytes("val2");
+
+ // test that concurrent writing to different rows succeeds
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
+
+ // should be no conflicts
+ txContext1.finish();
+ txContext2.finish();
+
+ transactionContext.start();
+ Result res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // test that writing to different columns in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
+
+ txContext2.start();
+ txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row-level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ // should now be val2
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ // col2 should not be visible due to conflict
+ assertNull(cell);
+ transactionContext.finish();
+
+ // test that writing to the same column in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row and column level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ assertNotNull(cell);
+ // should now be val1
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // verify change set that is being reported only on rows
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ Collection<byte[]> changeSet = txTable1.getTxChanges();
+ assertNotNull(changeSet);
+ assertEquals(2, changeSet.size());
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
+ txContext1.finish();
+ }
+
+ @Test
+ public void testNoneLevelConflictDetection() throws Exception {
+ InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+
+ // overlapping writes to the same row + column should not conflict
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // changes should not be visible yet
+ txContext2.start();
+ Result row = txTable2.get(new Get(TestBytes.row));
+ assertTrue(row.isEmpty());
+
+ txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // both commits should succeed
+ txContext1.finish();
+ txContext2.finish();
+
+ txContext1.start();
+ row = txTable1.get(new Get(TestBytes.row));
+ assertFalse(row.isEmpty());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext1.finish();
+
+ // transaction abort should still rollback changes
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext1.abort();
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row2 = txTable2.get(new Get(TestBytes.row2));
+ assertTrue(row2.isEmpty());
+ txContext2.finish();
+
+ // transaction invalidate should still make changes invisible
+
+ txContext1.start();
+ Transaction tx1 = txContext1.getCurrentTransaction();
+ txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ assertNotNull(tx1);
+ txClient.invalidate(tx1.getWritePointer());
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row3 = txTable2.get(new Get(TestBytes.row3));
+ assertTrue(row3.isEmpty());
+ txContext2.finish();
+ }
+
+ @Test
+ public void testCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
+ assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
+ long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
+ assertEquals(1, checkpointPtrs.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
+ assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
+ long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
+ assertEquals(2, checkpointPtrs2.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
+ assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // by default, all rows should be visible with Read-Your-Writes
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+
+ // when disabling current write pointer, only the previous checkpoints should be visible
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ Get get = new Get(TestBytes.row);
+ verifyRow(transactionAwareHTable, get, TestBytes.value);
+ get = new Get(TestBytes.row2);
+ verifyRow(transactionAwareHTable, get, TestBytes.value2);
+ get = new Get(TestBytes.row3);
+ verifyRow(transactionAwareHTable, get, null);
+
+ // test scan results excluding current write pointer
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+
+ Result row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row2, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNull(row);
+ scanner.close();
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+
+ // commit transaction, verify writes are visible
+ transactionContext.finish();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testInProgressCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // check that writes are still not visible to other clients
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ txContext2.start();
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ txContext2.finish();
+ txTable2.close();
+
+ transactionContext.finish();
+
+ // verify writes are visible after commit
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointRollback() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, null);
+ verifyRow(transactionAwareHTable, TestBytes.row2, null);
+ verifyRow(transactionAwareHTable, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointInvalidate() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
+
+ // check that writes are not visible
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+ txContext2.start();
+ Transaction newTx = txContext2.getCurrentTransaction();
+
+ // all 3 writes pointers from the previous transaction should now be excluded
+ assertTrue(newTx.isExcluded(origTx.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
+
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ verifyRow(txTable2, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = txTable2.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ txContext2.finish();
+ }
+
+ @Test
+ public void testExistingData() throws Exception {
+ byte[] val11 = Bytes.toBytes("val11");
+ byte[] val12 = Bytes.toBytes("val12");
+ byte[] val21 = Bytes.toBytes("val21");
+ byte[] val22 = Bytes.toBytes("val22");
+ byte[] val31 = Bytes.toBytes("val31");
+ byte[] val111 = Bytes.toBytes("val111");
+
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
+ Collections.<String>emptyList()));
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Add some pre-existing, non-transactional data
+ HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
+ HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.flushCommits();
+
+ // Add transactional data
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
+ txContext.finish();
+
+ txContext.start();
+ // test get
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+
+ // test scan
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+
+ // test update and delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
+ txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+
+ txContext.start();
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+ txContext.finish();
+
+ // test scan
+ txContext.start();
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+ }
+
+ private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
+ verifyRow(table, new Get(rowkey), expectedValue);
+ }
+
+ private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
+ verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
+ }
+
+ private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
+ Result result = table.get(get);
+ if (expectedValues == null) {
+ assertTrue(result.isEmpty());
+ } else {
+ assertFalse(result.isEmpty());
+ byte[] family = TestBytes.family;
+ byte[] col = TestBytes.qualifier;
+ if (get.hasFamilies()) {
+ family = get.getFamilyMap().keySet().iterator().next();
+ col = get.getFamilyMap().get(family).first();
+ }
+ Iterator<Cell> it = result.getColumnCells(family, col).iterator();
+ for (byte[] expectedValue : expectedValues) {
+ Assert.assertTrue(it.hasNext());
+ assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
+ }
+ }
+ }
+
+ private Cell[] getRow(HTableInterface table, Get get) throws Exception {
+ Result result = table.get(get);
+ return result.rawCells();
+ }
+
+ private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
+ List<Cell> actualCells = new ArrayList<>();
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ Result[] results = scanner.next(expectedCells.size() + 1);
+ for (Result result : results) {
+ actualCells.addAll(Lists.newArrayList(result.rawCells()));
+ }
+ Assert.assertEquals(expectedCells, actualCells);
+ }
+ }
+
+ @Test
+ public void testVisibilityAll() throws Exception {
+ HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(nonTxTable,
+ TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // start a transaction and create a delete marker
+ txContext.start();
+ //noinspection ConstantConditions
+ long txWp0 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // start a new transaction and write some values
+ txContext.start();
+ @SuppressWarnings("ConstantConditions")
+ long txWp1 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+
+ // verify written data
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ TestBytes.value2);
+
+ // checkpoint and make changes to written data now
+ txContext.checkpoint();
+ long txWp2 = txContext.getCurrentTransaction().getWritePointer();
+ // delete a column
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
+ // no change to a column
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ // update a column
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
+ // delete column family
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
+
+ // verify changed values
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ null);
+
+ // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
+ // and the raw change by prior transaction
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ List<KeyValue> expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // verify a Get is also able to return all snapshot versions
+ Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
+ Cell[] cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(3, cells.length);
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+ Assert.assertTrue(CellUtil.isDeleteColumns(cells[2]));
+
+ verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ ImmutableList.of(TestBytes.value3, TestBytes.value));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+
+ // Verify VisibilityLevel.SNAPSHOT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+ txContext.finish();
+
+ // finally verify values once more after commit, this time we should get only committed raw values for
+ // all visibility levels
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ txContext.finish();
+
+ // Test with regular HBase deletes in pre-existing data
+ long now = System.currentTimeMillis();
+ Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteColumn);
+ Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteFamily);
+ nonTxTable.flushCommits();
+
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
+ new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
+ );
+ // test scan
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ verifyScan(txTable, scan, expected);
+ txContext.finish();
+ }
+
+ @Test
+ public void testFilters() throws Exception {
+ // Add some values to table
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] val1 = Bytes.toBytes(1L);
+ put.add(TestBytes.family, TestBytes.qualifier, val1);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row2);
+ byte[] val2 = Bytes.toBytes(2L);
+ put.add(TestBytes.family, TestBytes.qualifier, val2);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row3);
+ byte[] val3 = Bytes.toBytes(3L);
+ put.add(TestBytes.family, TestBytes.qualifier, val3);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row4);
+ byte[] val4 = Bytes.toBytes(4L);
+ put.add(TestBytes.family, TestBytes.qualifier, val4);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Delete cell with value 2
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row2);
+ delete.deleteColumn(TestBytes.family, TestBytes.qualifier);
+ transactionAwareHTable.delete(delete);
+ transactionContext.finish();
+
+ // Scan for values less than 4, should get only values 1 and 3
+ transactionContext.start();
+ Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(4)));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run a Get with a filter for less than 10 on row4, should get value 4
+ transactionContext.start();
+ Get get = new Get(TestBytes.row4);
+ get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
+ Result result = transactionAwareHTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
+ transactionContext.finish();
+
+ // Change value of row4 to 40
+ transactionContext.start();
+ put = new Put(TestBytes.row4);
+ byte[] val40 = Bytes.toBytes(40L);
+ put.add(TestBytes.family, TestBytes.qualifier, val40);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Scan for values less than 10, should get only values 1 and 3
+ transactionContext.start();
+ scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run the Get again with a filter for less than 10 on row4, this time should not get any results
+ transactionContext.start();
+ result = transactionAwareHTable.get(get);
+ assertTrue(result.isEmpty());
+ transactionContext.finish();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilterTest.java
new file mode 100644
index 0000000..6223ba3
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilterTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * HBase 0.98 specific test for testing {@link CellSkipFilter}.
+ */
+public class CellSkipFilterTest {
+
+ private static final String ROW1KEY = "row1";
+ private static final String ROW2KEY = "row2";
+ private static final String FAM1KEY = "fam1";
+ private static final String COL1KEY = "col1";
+ private static final String FAM2KEY = "fam2";
+ private static final String COL2KEY = "col2";
+ private static final String VALUE = "value";
+
+ @Test
+ public void testSkipFiltering() throws Exception {
+ long timestamp = System.currentTimeMillis();
+ // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
+ Filter filter = new CellSkipFilter(new MyFilter(0));
+ assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give SKIP from MyFilter
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
+ timestamp - 4)));
+
+ // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
+ filter = new CellSkipFilter(new MyFilter(2));
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give NEXT_ROW from MyFilter
+ assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
+ timestamp - 4)));
+
+ // Next call with the new key should returned the SEEK_NEXT_USING_HINT
+ assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 5)));
+ }
+
+ private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
+ timestamp, Bytes.toBytes(value));
+ }
+
+ /**
+ * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
+ * start index in the list and start serving the return codes corresponding that that index. Every time the
+ * return code is served, index is incremented.
+ */
+ class MyFilter extends FilterBase {
+
+ private final List<ReturnCode> returnCodes;
+ private int counter;
+
+ public MyFilter(int startIndex) {
+ returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
+ ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
+ counter = startIndex;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ ReturnCode code = returnCodes.get(counter % returnCodes.size());
+ counter++;
+ return code;
+ }
+ }
+}
[13/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessorTest.java
new file mode 100644
index 0000000..373eb9d
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessorTest.java
@@ -0,0 +1,814 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98.coprocessor;
+
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.ipc.RpcServerInterface;
+import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
+import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
+import org.apache.hadoop.hbase.regionserver.FlushRequester;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager;
+import org.apache.hadoop.hbase.regionserver.Leases;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.ServerNonceManager;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.HDFSTransactionStateStorage;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.snapshot.DefaultSnapshotCodec;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.util.TxUtils;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
+ */
+public class TransactionProcessorTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
+
+ // 8 versions, 1 hour apart, latest is current ts.
+ private static final long[] V;
+
+ static {
+ long now = System.currentTimeMillis();
+ V = new long[9];
+ for (int i = 0; i < V.length; i++) {
+ V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
+ }
+ }
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+ private static MiniDFSCluster dfsCluster;
+ private static Configuration conf;
+ private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
+ private static TransactionVisibilityState txVisibilityState;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration hConf = new Configuration();
+ String rootDir = tmpFolder.newFolder().getAbsolutePath();
+ hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
+ hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
+
+ dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
+ dfsCluster.waitActive();
+ conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
+
+ conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
+ conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ String localTestDir = tmpFolder.newFolder().getAbsolutePath();
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+
+ // write an initial transaction snapshot
+ TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(
+ System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
+ // this will set visibility upper bound to V[6]
+ Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
+ TransactionType.SHORT))),
+ new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
+ txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
+ txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
+ txSnapshot.getInProgress());
+ HDFSTransactionStateStorage tmpStorage =
+ new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
+ tmpStorage.startAndWait();
+ tmpStorage.writeSnapshot(txSnapshot);
+ tmpStorage.stopAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ @Test
+ public void testDataJanitorRegionScanner() throws Exception {
+ String tableName = "TestRegionScanner";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ for (int i = 1; i <= 8; i++) {
+ for (int k = 1; k <= i; k++) {
+ Put p = new Put(Bytes.toBytes(i));
+ p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
+ region.put(p);
+ }
+ }
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should only return the valid rows
+ // do not use a filter here to test that cleanup works on flush
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+
+ // first returned value should be "4" with version "4"
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 4, new long[] {V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 5, new long[] {V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});
+
+ results.clear();
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteFiltering() throws Exception {
+ String tableName = "TestDeleteFiltering";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ byte[] row = Bytes.toBytes(1);
+ for (int i = 4; i < V.length; i++) {
+ Put p = new Put(row);
+ p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
+ region.put(p);
+ }
+
+ // delete from the third entry back
+ // take that cell's timestamp + 1 to simulate a delete in a new tx
+ long deleteTs = V[5] + 1;
+ Delete d = new Delete(row, deleteTs);
+ LOG.info("Issuing delete at timestamp " + deleteTs);
+ // row deletes are not yet supported (TransactionAwareHTable normally handles this)
+ d.deleteColumns(familyBytes, columnBytes);
+ region.delete(d);
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, we should drop the deleted version, but not the others
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should return row with versions at: V[8], V[6].
+ // V[7] is invalid and V[5] and prior are deleted.
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+ // should be only one row
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 1,
+ new long[]{V[8], V[6], deleteTs},
+ new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteMarkerCleanup() throws Exception {
+ String tableName = "TestDeleteMarkerCleanup";
+ byte[] familyBytes = Bytes.toBytes("f");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+
+ // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
+ long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
+ // deletes are performed after the writes, but still before the visibility upper bound
+ long deleteTs = writeTs + 1;
+ // write separate columns to confirm that delete markers survive across flushes
+ byte[] row = Bytes.toBytes(100);
+ Put p = new Put(row);
+
+ LOG.info("Writing columns at timestamp " + writeTs);
+ for (int i = 0; i < 5; i++) {
+ byte[] iBytes = Bytes.toBytes(i);
+ p.add(familyBytes, iBytes, writeTs, iBytes);
+ }
+ region.put(p);
+ // read all back
+ Scan scan = new Scan(row);
+ RegionScanner regionScanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ for (int i = 0; i < 5; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a flush to clear the memstore
+ LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // delete the odd entries
+ for (int i = 0; i < 5; i++) {
+ if (i % 2 == 1) {
+ // deletes are performed as puts with empty values
+ Put deletePut = new Put(row);
+ deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
+ region.put(deletePut);
+ }
+ }
+
+ // read all back
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ LOG.info("Got cell " + cell);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force another flush on the delete markers
+ // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
+ LOG.info("After delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a major compaction
+ LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
+ region.compactStores(true);
+
+ // perform a raw scan (no filter) to confirm that the delete markers are now gone
+ scan = new Scan(row);
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+ } finally {
+ region.close();
+ }
+ }
+
+ /**
+ * Test that we correctly preserve the timestamp set for column family delete markers. This is not
+ * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
+ * that we make it easy to interoperate with other systems.
+ */
+ @Test
+ public void testFamilyDeleteTimestamp() throws Exception {
+ String tableName = "TestFamilyDeleteTimestamp";
+ byte[] family1Bytes = Bytes.toBytes("f1");
+ byte[] columnBytes = Bytes.toBytes("c");
+ byte[] rowBytes = Bytes.toBytes("row");
+ byte[] valBytes = Bytes.toBytes("val");
+ HRegion region = createRegion(tableName, family1Bytes, 0);
+ try {
+ region.initialize();
+
+ long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+ Put p = new Put(rowBytes);
+ p.add(family1Bytes, columnBytes, now - 10, valBytes);
+ region.put(p);
+
+ // issue a family delete with an explicit timestamp
+ Delete delete = new Delete(rowBytes, now);
+ delete.deleteFamily(family1Bytes, now - 5);
+ region.delete(delete);
+
+ // test that the delete marker preserved the timestamp
+ Scan scan = new Scan();
+ scan.setMaxVersions();
+ RegionScanner scanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(2, results.size());
+ // delete marker should appear first
+ Cell cell = results.get(0);
+ assertArrayEquals(new byte[0], cell.getQualifier());
+ assertArrayEquals(new byte[0], cell.getValue());
+ assertEquals(now - 5, cell.getTimestamp());
+ // since this is an unfiltered scan against the region, the original put should be next
+ cell = results.get(1);
+ assertArrayEquals(valBytes, cell.getValue());
+ assertEquals(now - 10, cell.getTimestamp());
+ scanner.close();
+
+
+ // with a filtered scan the original put should disappear
+ scan = new Scan();
+ scan.setMaxVersions();
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ scanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(0, results.size());
+ scanner.close();
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testPreExistingData() throws Exception {
+ String tableName = "TestPreExistingData";
+ byte[] familyBytes = Bytes.toBytes("f");
+ long ttlMillis = TimeUnit.DAYS.toMillis(14);
+ HRegion region = createRegion(tableName, familyBytes, ttlMillis);
+ try {
+ region.initialize();
+
+ // timestamps for pre-existing, non-transactional data
+ long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
+ long older = now - ttlMillis / 2;
+ long newer = now - ttlMillis / 3;
+ // timestamps for transactional data
+ long nowTx = txVisibilityState.getVisibilityUpperBound();
+ long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
+ long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
+
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ ttls.put(familyBytes, ttlMillis);
+
+ List<Cell> cells = new ArrayList<>();
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
+
+ // Write non-transactional and transactional data
+ for (Cell c : cells) {
+ region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
+ }
+
+ Scan rawScan = new Scan();
+ rawScan.setMaxVersions();
+
+ Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
+ Scan txScan = new Scan();
+ txScan.setMaxVersions();
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // read all back with raw scanner
+ scanAndAssert(region, cells, rawScan);
+
+ // read all back with transaction filter
+ scanAndAssert(region, cells, txScan);
+
+ // force a flush to clear the memstore
+ region.flushcache();
+ scanAndAssert(region, cells, txScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ scanAndAssert(region, cells, txScan);
+
+ // Reduce TTL, this should make cells with timestamps older and olderTx expire
+ long newTtl = ttlMillis / 2 - 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // Raw scan should still give all cells
+ scanAndAssert(region, cells, rawScan);
+ // However, tx scan should not return expired cells
+ scanAndAssert(region, select(cells, 1, 3, 5), txScan);
+
+ region.flushcache();
+ scanAndAssert(region, cells, rawScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ // This time raw scan too should not return expired cells, as they would be dropped during major compaction
+ scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
+
+ // Reduce TTL again to 1 ms, this should expire all cells
+ newTtl = 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // force a major compaction to remove expired cells
+ region.compactStores(true);
+ // This time raw scan should not return any cells, as all cells have expired.
+ scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
+ } finally {
+ region.close();
+ }
+ }
+
+ private List<Cell> select(List<Cell> cells, int... indexes) {
+ List<Cell> newCells = new ArrayList<>();
+ for (int i : indexes) {
+ newCells.add(cells.get(i));
+ }
+ return newCells;
+ }
+
+ @SuppressWarnings("StatementWithEmptyBody")
+ private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
+ try (RegionScanner regionScanner = region.getScanner(scan)) {
+ List<Cell> results = Lists.newArrayList();
+ while (regionScanner.next(results)) { }
+ assertEquals(expected, results);
+ }
+ }
+
+ private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
+ region.close();
+ HTableDescriptor htd = region.getTableDesc();
+ HColumnDescriptor cfd = new HColumnDescriptor(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ htd.addFamily(cfd);
+ return HRegion.openHRegion(region.getRegionInfo(), htd, region.getLog(), conf,
+ new MockRegionServerServices(conf, null), null);
+ }
+
+ private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
+ HColumnDescriptor cfd = new HColumnDescriptor(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ htd.addFamily(cfd);
+ htd.addCoprocessor(TransactionProcessor.class.getName());
+ Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
+ Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
+ FileSystem fs = FileSystem.get(conf);
+ assertTrue(fs.mkdirs(tablePath));
+ HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
+ HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
+ HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
+ return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
+ byte[][] values = new byte[versions.length][];
+ for (int i = 0; i < versions.length; i++) {
+ values[i] = Bytes.toBytes(versions[i]);
+ }
+ assertKeyValueMatches(results, index, versions, values);
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
+ assertEquals(versions.length, results.size());
+ assertEquals(values.length, results.size());
+ for (int i = 0; i < versions.length; i++) {
+ Cell kv = results.get(i);
+ assertArrayEquals(Bytes.toBytes(index), kv.getRow());
+ assertEquals(versions[i], kv.getTimestamp());
+ assertArrayEquals(values[i], kv.getValue());
+ }
+ }
+
+ @Test
+ public void testTransactionStateCache() throws Exception {
+ TransactionStateCache cache = new TransactionStateCache();
+ cache.setConf(conf);
+ cache.startAndWait();
+ // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
+ TransactionVisibilityState cachedSnapshot = cache.getLatestState();
+ assertNotNull(cachedSnapshot);
+ assertEquals(invalidSet, cachedSnapshot.getInvalid());
+ cache.stopAndWait();
+ }
+
+ private static class MockRegionServerServices implements RegionServerServices {
+ private final Configuration hConf;
+ private final ZooKeeperWatcher zookeeper;
+ private final Map<String, HRegion> regions = new HashMap<>();
+ private boolean stopping = false;
+ private final ConcurrentSkipListMap<byte[], Boolean> rit = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
+ private HFileSystem hfs = null;
+ private ServerName serverName = null;
+ private RpcServerInterface rpcServer = null;
+ private volatile boolean abortRequested;
+
+
+ public MockRegionServerServices(Configuration hConf, ZooKeeperWatcher zookeeper) {
+ this.hConf = hConf;
+ this.zookeeper = zookeeper;
+ }
+
+ @Override
+ public boolean isStopping() {
+ return stopping;
+ }
+
+ @Override
+ public HLog getWAL(HRegionInfo regionInfo) throws IOException {
+ return null;
+ }
+
+ @Override
+ public CompactionRequestor getCompactionRequester() {
+ return null;
+ }
+
+ @Override
+ public double getCompactionPressure() {
+ return 0;
+ }
+
+ @Override
+ public FlushRequester getFlushRequester() {
+ return null;
+ }
+
+ @Override
+ public RegionServerAccounting getRegionServerAccounting() {
+ return null;
+ }
+
+ @Override
+ public TableLockManager getTableLockManager() {
+ return new TableLockManager.NullTableLockManager();
+ }
+
+ @Override
+ public void postOpenDeployTasks(HRegion r, CatalogTracker ct) throws KeeperException, IOException {
+ }
+
+ @Override
+ public RpcServerInterface getRpcServer() {
+ return rpcServer;
+ }
+
+ @Override
+ public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
+ return rit;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return hfs;
+ }
+
+ @Override
+ public Leases getLeases() {
+ return null;
+ }
+
+ @Override
+ public ExecutorService getExecutorService() {
+ return null;
+ }
+
+ @Override
+ public CatalogTracker getCatalogTracker() {
+ return null;
+ }
+
+ @Override
+ public Map<String, HRegion> getRecoveringRegions() {
+ return null;
+ }
+
+ @Override
+ public void updateRegionFavoredNodesMapping(String encodedRegionName, List<HBaseProtos.ServerName> favoredNodes) {
+ }
+
+ @Override
+ public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
+ return new InetSocketAddress[0];
+ }
+
+ @Override
+ public void addToOnlineRegions(HRegion r) {
+ regions.put(r.getRegionNameAsString(), r);
+ }
+
+ @Override
+ public boolean removeFromOnlineRegions(HRegion r, ServerName destination) {
+ return regions.remove(r.getRegionInfo().getEncodedName()) != null;
+ }
+
+ @Override
+ public HRegion getFromOnlineRegions(String encodedRegionName) {
+ return regions.get(encodedRegionName);
+ }
+
+ @Override
+ public List<HRegion> getOnlineRegions(TableName tableName) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hConf;
+ }
+
+ @Override
+ public ZooKeeperWatcher getZooKeeper() {
+ return zookeeper;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return serverName;
+ }
+
+ @Override
+ public void abort(String why, Throwable e) {
+ this.abortRequested = true;
+ }
+
+ @Override
+ public boolean isAborted() {
+ return abortRequested;
+ }
+
+ @Override
+ public void stop(String why) {
+ this.stopping = true;
+ }
+
+ @Override
+ public boolean isStopped() {
+ return stopping;
+ }
+
+ @Override
+ public int getPriority(RequestHeader header, Message param) {
+ return 0;
+ }
+
+ @Override
+ public ServerNonceManager getNonceManager() {
+ return null;
+ }
+
+ @Override
+ public boolean reportRegionStateTransition(
+ RegionServerStatusProtos.RegionStateTransition.TransitionCode transitionCode,
+ long l, HRegionInfo... hRegionInfos) {
+ return false;
+ }
+
+ @Override
+ public boolean reportRegionStateTransition(
+ RegionServerStatusProtos.RegionStateTransition.TransitionCode transitionCode,
+ HRegionInfo... hRegionInfos) {
+ return false;
+ }
+
+ @Override
+ public boolean registerService(Service service) {
+ return false;
+ }
+
+ @Override
+ public HeapMemoryManager getHeapMemoryManager() {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java
new file mode 100644
index 0000000..17c542e
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilterTest.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98.coprocessor;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.hbase.AbstractTransactionVisibilityFilterTest;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * HBase 0.98 specific test for filtering logic applied when reading data transactionally.
+ */
+public class TransactionVisibilityFilterTest extends AbstractTransactionVisibilityFilterTest {
+ /**
+ * Test filtering of KeyValues for in-progress and invalid transactions.
+ * @throws Exception
+ */
+ @Test
+ public void testFiltering() throws Exception {
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+ }
+
+ @Test
+ public void testSubFilter() throws Exception {
+ final FilterBase includeFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE;
+ }
+ };
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter skipFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.SKIP;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ final Filter includeNextFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter nextColFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ }
+
+ @Test
+ public void testSubFilterOverride() throws Exception {
+ final FilterBase includeFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE;
+ }
+ };
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.INCLUDE));
+
+ final Filter skipFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.SKIP;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ final Filter includeNextFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter nextColFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ }
+
+ private void runFilteringTest(TxFilterFactory txFilterFactory,
+ List<Filter.ReturnCode> assertCodes) throws Exception {
+ /*
+ * Start and stop some transactions. This will give us a transaction state something like the following
+ * (numbers only reflect ordering, not actual transaction IDs):
+ * 6 - in progress
+ * 5 - committed
+ * 4 - invalid
+ * 3 - in-progress
+ * 2 - committed
+ * 1 - committed
+ *
+ * read ptr = 5
+ * write ptr = 6
+ */
+
+ Transaction tx1 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx1, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx1));
+
+ Transaction tx2 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx2, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx2));
+
+ Transaction tx3 = txManager.startShort();
+ Transaction tx4 = txManager.startShort();
+ txManager.invalidate(tx4.getTransactionId());
+
+ Transaction tx5 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx5, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx5));
+
+ Transaction tx6 = txManager.startShort();
+
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ Filter filter = txFilterFactory.getTxFilter(tx6, ttls);
+
+ assertEquals(assertCodes.get(5),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx6.getTransactionId())));
+ assertEquals(assertCodes.get(4),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx5.getTransactionId())));
+ assertEquals(assertCodes.get(3),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx4.getTransactionId())));
+ assertEquals(assertCodes.get(2),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx3.getTransactionId())));
+ assertEquals(assertCodes.get(1),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx2.getTransactionId())));
+ assertEquals(assertCodes.get(0),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx1.getTransactionId())));
+ }
+
+ /**
+ * Test filtering for TTL settings.
+ * @throws Exception
+ */
+ @Test
+ public void testTTLFiltering() throws Exception {
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ ttls.put(FAM, 10L);
+ ttls.put(FAM2, 30L);
+ ttls.put(FAM3, 0L);
+
+ Transaction tx = txManager.startShort();
+ long now = tx.getVisibilityUpperBound();
+ Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 21 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 1001 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
+
+ // Verify ttl for pre-existing, non-transactional data
+ long preNow = now / TxConstants.MAX_TX_PER_MS;
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 9L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 10L)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 11L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 9L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 10L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 1001L)));
+ }
+
+ protected KeyValue newKeyValue(String rowkey, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), FAM, COL, timestamp, Bytes.toBytes(value));
+ }
+
+ protected KeyValue newKeyValue(String rowkey, byte[] family, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), family, COL, timestamp, Bytes.toBytes(value));
+ }
+
+ private interface TxFilterFactory {
+ Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs);
+ }
+
+ private class CustomTxFilter extends TransactionVisibilityFilter {
+ public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
+ @Nullable Filter cellFilter) {
+ super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+ }
+
+ @Override
+ protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
+ switch (subFilterCode) {
+ case INCLUDE:
+ return ReturnCode.INCLUDE;
+ case INCLUDE_AND_NEXT_COL:
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ case SKIP:
+ return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
+ default:
+ return subFilterCode;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProvider.java b/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProvider.java
deleted file mode 100644
index 120b0f3..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProvider.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh;
-
-import co.cask.tephra.util.ConfigurationProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-/**
- * HBase 1.0 version of {@link co.cask.tephra.util.ConfigurationProvider}.
- */
-public class HBase10ConfigurationProvider extends ConfigurationProvider {
- @Override
- public Configuration get() {
- return HBaseConfiguration.create();
- }
-
- @Override
- public Configuration get(Configuration baseConf) {
- return HBaseConfiguration.create(baseConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/SecondaryIndexTable.java b/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/SecondaryIndexTable.java
deleted file mode 100644
index ada551a..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/SecondaryIndexTable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase10cdh;
-
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * A Transactional SecondaryIndexTable.
- */
-public class SecondaryIndexTable {
- private byte[] secondaryIndex;
- private TransactionAwareHTable transactionAwareHTable;
- private TransactionAwareHTable secondaryIndexTable;
- private TransactionContext transactionContext;
- private final TableName secondaryIndexTableName;
- private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
- private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
- private static final byte[] DELIMITER = new byte[] {0};
-
- public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
- byte[] secondaryIndex) {
- secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
- HTable secondaryIndexHTable = null;
- HBaseAdmin hBaseAdmin = null;
- try {
- hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
- if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
- hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
- }
- secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
- } catch (Exception e) {
- Throwables.propagate(e);
- } finally {
- try {
- hBaseAdmin.close();
- } catch (Exception e) {
- Throwables.propagate(e);
- }
- }
-
- this.secondaryIndex = secondaryIndex;
- this.transactionAwareHTable = new TransactionAwareHTable(hTable);
- this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
- this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
- secondaryIndexTable);
- }
-
- public Result get(Get get) throws IOException {
- return get(Collections.singletonList(get))[0];
- }
-
- public Result[] get(List<Get> gets) throws IOException {
- try {
- transactionContext.start();
- Result[] result = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return result;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public Result[] getByIndex(byte[] value) throws IOException {
- try {
- transactionContext.start();
- Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
- scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
- ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
-
- ArrayList<Get> gets = new ArrayList<Get>();
- for (Result result : indexScanner) {
- for (Cell cell : result.listCells()) {
- gets.add(new Get(cell.getValue()));
- }
- }
- Result[] results = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return results;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public void put(Put put) throws IOException {
- put(Collections.singletonList(put));
- }
-
-
- public void put(List<Put> puts) throws IOException {
- try {
- transactionContext.start();
- ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
- for (Put put : puts) {
- List<Put> indexPuts = new ArrayList<Put>();
- Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
- for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
- for (KeyValue value : family.getValue()) {
- if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
- secondaryIndex, 0, secondaryIndex.length)) {
- byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
- Bytes.add(value.getValue(), DELIMITER,
- value.getRow()));
- Put indexPut = new Put(secondaryRow);
- indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
- indexPuts.add(indexPut);
- }
- }
- }
- secondaryIndexPuts.addAll(indexPuts);
- }
- transactionAwareHTable.put(puts);
- secondaryIndexTable.put(secondaryIndexPuts);
- transactionContext.finish();
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/TransactionAwareHTable.java b/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/TransactionAwareHTable.java
deleted file mode 100644
index fe18f8d..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/co/cask/tephra/hbase10cdh/TransactionAwareHTable.java
+++ /dev/null
@@ -1,674 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase10cdh;
-
-import co.cask.tephra.AbstractTransactionAwareTable;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TxConstants;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.Batch;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-
-/**
- * A Transaction Aware HTable implementation for HBase 1.0. Operations are committed as usual,
- * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
- * was started.
- */
-public class TransactionAwareHTable extends AbstractTransactionAwareTable
- implements HTableInterface, TransactionAware {
-
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
- private final HTableInterface hTable;
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- */
- public TransactionAwareHTable(HTableInterface hTable) {
- this(hTable, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
- this(hTable, conflictLevel, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
- this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
- boolean allowNonTransactional) {
- super(conflictLevel, allowNonTransactional);
- this.hTable = hTable;
- }
-
- /* AbstractTransactionAwareTable implementation */
-
- @Override
- protected byte[] getTableKey() {
- return getTableName();
- }
-
- @Override
- protected boolean doCommit() throws IOException {
- hTable.flushCommits();
- return true;
- }
-
- @Override
- protected boolean doRollback() throws Exception {
- try {
- // pre-size arraylist of deletes
- int size = 0;
- for (Set<ActionChange> cs : changeSets.values()) {
- size += cs.size();
- }
- List<Delete> rollbackDeletes = new ArrayList<>(size);
- for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
- long transactionTimestamp = entry.getKey();
- for (ActionChange change : entry.getValue()) {
- byte[] row = change.getRow();
- byte[] family = change.getFamily();
- byte[] qualifier = change.getQualifier();
- Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- switch (conflictLevel) {
- case ROW:
- case NONE:
- // issue family delete for the tx write pointer
- rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
- break;
- case COLUMN:
- if (family != null && qualifier == null) {
- rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
- } else if (family != null && qualifier != null) {
- rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
- }
- break;
- default:
- throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
- }
- rollbackDeletes.add(rollbackDelete);
- }
- }
- hTable.delete(rollbackDeletes);
- return true;
- } finally {
- try {
- hTable.flushCommits();
- } catch (Exception e) {
- LOG.error("Could not flush HTable commits", e);
- }
- tx = null;
- changeSets.clear();
- }
- }
-
- /* HTableInterface implementation */
-
- @Override
- public byte[] getTableName() {
- return hTable.getTableName();
- }
-
- @Override
- public TableName getName() {
- return hTable.getName();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hTable.getConfiguration();
- }
-
- @Override
- public HTableDescriptor getTableDescriptor() throws IOException {
- return hTable.getTableDescriptor();
- }
-
- @Override
- public boolean exists(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.exists(transactionalizeAction(get));
- }
-
- @Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.exists(transactionalizedGets);
- }
-
- @Override
- public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batch(transactionalizeActions(actions), results);
- }
-
- @Override
- public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batch(transactionalizeActions(actions));
- }
-
- @Override
- public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
- IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batchCallback(transactionalizeActions(actions), results, callback);
- }
-
- @Override
- public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
- InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batchCallback(transactionalizeActions(actions), callback);
- }
-
- @Override
- public Result get(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.get(transactionalizeAction(get));
- }
-
- @Override
- public Result[] get(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.get(transactionalizedGets);
- }
-
- @Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- if (allowNonTransactional) {
- return hTable.getRowOrBefore(row, family);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public ResultScanner getScanner(Scan scan) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addFamily(family);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addColumn(family, qualifier);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public void put(Put put) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Put txPut = transactionalizeAction(put);
- hTable.put(txPut);
- }
-
- @Override
- public void put(List<Put> puts) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
- for (Put put : puts) {
- Put txPut = transactionalizeAction(put);
- transactionalizedPuts.add(txPut);
- }
- hTable.put(transactionalizedPuts);
- }
-
- @Override
- public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndPut(row, family, qualifier, value, put);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public void delete(Delete delete) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.delete(transactionalizeAction(delete));
- }
-
- @Override
- public void delete(List<Delete> deletes) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
- for (Delete delete : deletes) {
- Delete txDelete = transactionalizeAction(delete);
- transactionalizedDeletes.add(txDelete);
- }
- hTable.delete(transactionalizedDeletes);
- }
-
- @Override
- public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndDelete(row, family, qualifier, value, delete);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
- byte[] bytes3, Delete delete) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndDelete(bytes, bytes1, bytes2, compareOp, bytes3, delete);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
- byte[] bytes3, Put put) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndPut(bytes, bytes1, bytes2, compareOp, bytes3, put);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean[] existsAll(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.existsAll(transactionalizedGets);
- }
-
- @Override
- public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
- CompareFilter.CompareOp compareOp, byte[] value, RowMutations rowMutations)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndMutate(row, family, qualifier, compareOp, value, rowMutations);
- }
-
- throw new UnsupportedOperationException("checkAndMutate operation is not supported transactionally");
- }
-
- @Override
- public void mutateRow(RowMutations rm) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- RowMutations transactionalMutations = new RowMutations();
- for (Mutation mutation : rm.getMutations()) {
- if (mutation instanceof Put) {
- transactionalMutations.add(transactionalizeAction((Put) mutation));
- } else if (mutation instanceof Delete) {
- transactionalMutations.add(transactionalizeAction((Delete) mutation));
- }
- }
- hTable.mutateRow(transactionalMutations);
- }
-
- @Override
- public Result append(Append append) throws IOException {
- if (allowNonTransactional) {
- return hTable.append(append);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public Result increment(Increment increment) throws IOException {
- if (allowNonTransactional) {
- return hTable.increment(increment);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean isAutoFlush() {
- return hTable.isAutoFlush();
- }
-
- @Override
- public void flushCommits() throws IOException {
- hTable.flushCommits();
- }
-
- @Override
- public void close() throws IOException {
- hTable.close();
- }
-
- @Override
- public CoprocessorRpcChannel coprocessorService(byte[] row) {
- return hTable.coprocessorService(row);
- }
-
- @Override
- public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable)
- throws ServiceException, Throwable {
- return hTable.coprocessorService(service, startKey, endKey, callable);
- }
-
- @Override
- public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable, Batch.Callback<R> callback)
- throws ServiceException, Throwable {
- hTable.coprocessorService(service, startKey, endKey, callable, callback);
- }
-
- @Override
- public <R extends Message> Map<byte[], R> batchCoprocessorService(
- MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey,
- R responsePrototype) throws ServiceException, Throwable {
- return hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
- }
-
- @Override
- public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
- Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
- throws ServiceException, Throwable {
- hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush) {
- setAutoFlushTo(autoFlush);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- hTable.setAutoFlush(autoFlush, clearBufferOnFail);
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- hTable.setAutoFlushTo(autoFlush);
- }
-
- @Override
- public long getWriteBufferSize() {
- return hTable.getWriteBufferSize();
- }
-
- @Override
- public void setWriteBufferSize(long writeBufferSize) throws IOException {
- hTable.setWriteBufferSize(writeBufferSize);
- }
-
- // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
-
- private Get transactionalizeAction(Get get) throws IOException {
- addToOperation(get, tx);
- return get;
- }
-
- private Scan transactionalizeAction(Scan scan) throws IOException {
- addToOperation(scan, tx);
- return scan;
- }
-
- private Put transactionalizeAction(Put put) throws IOException {
- Put txPut = new Put(put.getRow(), tx.getWritePointer());
- Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
- if (!familyMap.isEmpty()) {
- for (Map.Entry<byte[], List<Cell>> family : familyMap) {
- List<Cell> familyValues = family.getValue();
- if (!familyValues.isEmpty()) {
- for (Cell value : familyValues) {
- txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
- addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
- txPut.setAttribute(entry.getKey(), entry.getValue());
- }
- txPut.setDurability(put.getDurability());
- addToOperation(txPut, tx);
- return txPut;
- }
-
- private Delete transactionalizeAction(Delete delete) throws IOException {
- long transactionTimestamp = tx.getWritePointer();
-
- byte[] deleteRow = delete.getRow();
- Delete txDelete = new Delete(deleteRow, transactionTimestamp);
-
- Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
- if (familyToDelete.isEmpty()) {
- // perform a row delete if we are using row-level conflict detection
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // Row delete leaves delete markers in all column families of the table
- // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
- for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
- // no need to identify individual columns deleted
- addToChangeSet(deleteRow, columnDescriptor.getName(), null);
- }
- } else {
- Result result = get(new Get(delete.getRow()));
- // Delete everything
- NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
- for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
- }
- }
- }
- } else {
- for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
- byte[] family = familyEntry.getKey();
- List<Cell> entries = familyEntry.getValue();
- boolean isFamilyDelete = false;
- if (entries.size() == 1) {
- Cell cell = entries.get(0);
- isFamilyDelete = CellUtil.isDeleteFamily(cell);
- }
- if (isFamilyDelete) {
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // no need to identify individual columns deleted
- txDelete.deleteFamily(family);
- addToChangeSet(deleteRow, family, null);
- } else {
- Result result = get(new Get(delete.getRow()).addFamily(family));
- // Delete entire family
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, family, column.getKey());
- }
- }
- } else {
- for (Cell value : entries) {
- txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
- addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- txDelete.setAttribute(entry.getKey(), entry.getValue());
- }
- txDelete.setDurability(delete.getDurability());
- return txDelete;
- }
-
- private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
- List<Row> transactionalizedActions = new ArrayList<>(actions.size());
- for (Row action : actions) {
- if (action instanceof Get) {
- transactionalizedActions.add(transactionalizeAction((Get) action));
- } else if (action instanceof Put) {
- transactionalizedActions.add(transactionalizeAction((Put) action));
- } else if (action instanceof Delete) {
- transactionalizedActions.add(transactionalizeAction((Delete) action));
- } else {
- transactionalizedActions.add(action);
- }
- }
- return transactionalizedActions;
- }
-
- public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
- op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
- }
-}
[29/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/distributed/ElasticPoolTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/distributed/ElasticPoolTest.java b/tephra-core/src/test/java/co/cask/tephra/distributed/ElasticPoolTest.java
deleted file mode 100644
index 40354cf..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/distributed/ElasticPoolTest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import com.google.common.base.Throwables;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Tests for {@link ElasticPool}.
- */
-public class ElasticPoolTest {
-
- static class Dummy {
- static AtomicInteger count = new AtomicInteger(0);
- boolean valid = true;
- Dummy() {
- count.incrementAndGet();
- }
- void markInvalid() {
- valid = false;
- }
-
- public boolean isValid() {
- return valid;
- }
- }
-
- class DummyPool extends ElasticPool<Dummy, RuntimeException> {
-
- public DummyPool(int sizeLimit) {
- super(sizeLimit);
- }
-
- @Override
- protected Dummy create() {
- return new Dummy();
- }
-
- @Override
- protected boolean recycle(Dummy element) {
- return element.isValid();
- }
- }
-
- @Test(timeout = 5000)
- public void testFewerThreadsThanElements() throws InterruptedException {
- final DummyPool pool = new DummyPool(5);
- Dummy.count.set(0);
- createAndRunThreads(2, pool, false);
- // we only ran 2 threads, so only 2 elements got created, even though pool size is 5
- Assert.assertEquals(2, Dummy.count.get());
- }
-
- @Test(timeout = 5000)
- public void testMoreThreadsThanElements() throws InterruptedException {
- final DummyPool pool = new DummyPool(2);
- Dummy.count.set(0);
- createAndRunThreads(5, pool, false);
- // even though we ran 5 threads, only 2 elements got created because pool size is 2
- Assert.assertEquals(2, Dummy.count.get());
- }
-
- @Test(timeout = 5000)
- public void testMoreThreadsThanElementsWithDiscard() throws InterruptedException {
- final DummyPool pool = new DummyPool(2);
- Dummy.count.set(0);
- int numThreads = 3;
- // pass 'true' as the last parameter, which results in the elements being discarded after each obtain() call.
- createAndRunThreads(numThreads, pool, true);
- // this results in (5 * numThreads) number of elements being created since each thread obtains a client 5 times.
- Assert.assertEquals(5 * numThreads, Dummy.count.get());
- }
-
- // Creates a list of threads which obtain a client from the pool, sleeps for a certain amount of time, and then
- // releases the client back to the pool, optionally marking it invalid before doing so. It repeats this five times.
- // Then, runs these threads to completion.
- private void createAndRunThreads(int numThreads, final DummyPool pool,
- final boolean discardAtEnd) throws InterruptedException {
- Thread[] threads = new Thread[numThreads];
- for (int i = 0; i < numThreads; i++) {
- threads[i] = new Thread() {
- @Override
- public void run() {
- for (int j = 0; j < 5; ++j) {
- Dummy dummy;
- try {
- dummy = pool.obtain();
- } catch (InterruptedException e) {
- throw Throwables.propagate(e);
- }
- try {
- Thread.sleep(10L);
- } catch (InterruptedException e) {
- // ignored
- }
- if (discardAtEnd) {
- dummy.markInvalid();
- }
- pool.release(dummy);
- }
- }
- };
- }
- for (Thread t : threads) {
- t.start();
- }
- for (Thread t : threads) {
- t.join();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/distributed/PooledClientProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/distributed/PooledClientProviderTest.java b/tephra-core/src/test/java/co/cask/tephra/distributed/PooledClientProviderTest.java
deleted file mode 100644
index 354b5b7..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/distributed/PooledClientProviderTest.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.TransactionServiceMain;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import com.google.common.base.Throwables;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.discovery.DiscoveryServiceClient;
-import org.apache.twill.internal.zookeeper.InMemoryZKServer;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.junit.Assert;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
-
-public class PooledClientProviderTest {
-
- public static final int MAX_CLIENT_COUNT = 3;
- public static final long CLIENT_OBTAIN_TIMEOUT = 10;
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @Test
- public void testClientConnectionPoolMaximumNumberOfClients() throws Exception {
- // We need a server for the client to connect to
- InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
- zkServer.startAndWait();
-
- try {
- Configuration conf = new Configuration();
- conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
- conf.set("data.tx.client.count", Integer.toString(MAX_CLIENT_COUNT));
- conf.set("data.tx.client.obtain.timeout", Long.toString(CLIENT_OBTAIN_TIMEOUT));
-
- final TransactionServiceMain main = new TransactionServiceMain(conf);
- final CountDownLatch latch = new CountDownLatch(1);
- Thread t = new Thread() {
- @Override
- public void run() {
- try {
- main.start();
- latch.countDown();
- } catch (Exception e) {
- throw Throwables.propagate(e);
- }
- }
- };
-
- try {
- t.start();
- // Wait for service to startup
- latch.await();
-
- startClientAndTestPool(conf);
- } finally {
- main.stop();
- t.join();
- }
- } finally {
- zkServer.stopAndWait();
- }
- }
-
- private void startClientAndTestPool(Configuration conf) throws Exception {
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- new TransactionModules().getDistributedModules(),
- new TransactionClientModule()
- );
-
- ZKClientService zkClient = injector.getInstance(ZKClientService.class);
- zkClient.startAndWait();
-
- final PooledClientProvider clientProvider = new PooledClientProvider(conf,
- injector.getInstance(DiscoveryServiceClient.class));
-
- // test simple case of get + return. Note: this also initializes the provider's pool, which
- // takes about one second (discovery). Doing it before we test the threads makes it so that one
- // thread doesn't take exceptionally longer than the others.
- try (CloseableThriftClient closeableThriftClient = clientProvider.getCloseableClient()) {
- // do nothing with the client
- }
-
- //Now race to get MAX_CLIENT_COUNT+1 clients, exhausting the pool and requesting 1 more.
- List<Future<Integer>> clientIds = new ArrayList<Future<Integer>>();
- CountDownLatch countDownLatch = new CountDownLatch(1);
- ExecutorService executor = Executors.newFixedThreadPool(MAX_CLIENT_COUNT + 1);
- for (int i = 0; i < MAX_CLIENT_COUNT + 1; i++) {
- clientIds.add(executor.submit(new RetrieveClient(clientProvider, CLIENT_OBTAIN_TIMEOUT / 2, countDownLatch)));
- }
- countDownLatch.countDown();
-
- Set<Integer> ids = new HashSet<Integer>();
- for (Future<Integer> id : clientIds) {
- ids.add(id.get());
- }
- Assert.assertEquals(MAX_CLIENT_COUNT, ids.size());
-
- // now, try it again with, where each thread holds onto the client for twice the client.obtain.timeout value.
- // one of the threads should throw a TimeOutException, because the other threads don't release their clients
- // within the configured timeout.
- countDownLatch = new CountDownLatch(1);
- for (int i = 0; i < MAX_CLIENT_COUNT + 1; i++) {
- clientIds.add(executor.submit(new RetrieveClient(clientProvider, CLIENT_OBTAIN_TIMEOUT * 2, countDownLatch)));
- }
- countDownLatch.countDown();
- int numTimeoutExceptions = 0;
- for (Future<Integer> clientId : clientIds) {
- try {
- clientId.get();
- } catch (ExecutionException expected) {
- Assert.assertEquals(TimeoutException.class, expected.getCause().getClass());
- numTimeoutExceptions++;
- }
- }
- // expect that exactly one of the threads hit the TimeoutException
- Assert.assertEquals(String.format("Expected one thread to not obtain a client within %s milliseconds.",
- CLIENT_OBTAIN_TIMEOUT),
- 1, numTimeoutExceptions);
-
- executor.shutdown();
- }
-
- private static class RetrieveClient implements Callable<Integer> {
- private final PooledClientProvider pool;
- private final long holdClientMs;
- private final CountDownLatch begin;
-
- public RetrieveClient(PooledClientProvider pool, long holdClientMs,
- CountDownLatch begin) {
- this.pool = pool;
- this.holdClientMs = holdClientMs;
- this.begin = begin;
- }
-
- @Override
- public Integer call() throws Exception {
- begin.await();
- try (CloseableThriftClient client = pool.getCloseableClient()) {
- int id = System.identityHashCode(client.getThriftClient());
- // "use" the client for a configured amount of milliseconds
- Thread.sleep(holdClientMs);
- return id;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/distributed/ThriftTransactionServerTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/distributed/ThriftTransactionServerTest.java b/tephra-core/src/test/java/co/cask/tephra/distributed/ThriftTransactionServerTest.java
deleted file mode 100644
index 83aad3c..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/distributed/ThriftTransactionServerTest.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.distributed;
-
-import co.cask.tephra.ThriftTransactionSystemTest;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionEdit;
-import co.cask.tephra.persist.TransactionLog;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import com.google.common.util.concurrent.Service;
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Scopes;
-import com.google.inject.util.Modules;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.internal.zookeeper.InMemoryZKServer;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * This tests whether transaction service hangs on stop when heavily loaded - https://issues.cask.co/browse/TEPHRA-132
- */
-public class ThriftTransactionServerTest {
- private static final Logger LOG = LoggerFactory.getLogger(ThriftTransactionSystemTest.class);
-
- private static InMemoryZKServer zkServer;
- private static ZKClientService zkClientService;
- private static TransactionService txService;
- private static TransactionStateStorage storage;
- static Injector injector;
-
- private static final int NUM_CLIENTS = 17;
- private static final CountDownLatch STORAGE_WAIT_LATCH = new CountDownLatch(1);
- private static final CountDownLatch CLIENTS_DONE_LATCH = new CountDownLatch(NUM_CLIENTS);
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- @BeforeClass
- public static void start() throws Exception {
- zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
- zkServer.startAndWait();
-
- Configuration conf = new Configuration();
- conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
- conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
- conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
- conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
- conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT, NUM_CLIENTS);
- conf.setLong(TxConstants.Service.CFG_DATA_TX_CLIENT_TIMEOUT, TimeUnit.HOURS.toMillis(1));
- conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS, 2);
- conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS, 4);
-
- injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- Modules.override(new TransactionModules().getDistributedModules())
- .with(new AbstractModule() {
- @Override
- protected void configure() {
- bind(TransactionStateStorage.class).to(SlowTransactionStorage.class).in(Scopes.SINGLETON);
- }
- }),
- new TransactionClientModule()
- );
-
- zkClientService = injector.getInstance(ZKClientService.class);
- zkClientService.startAndWait();
-
- // start a tx server
- txService = injector.getInstance(TransactionService.class);
- storage = injector.getInstance(TransactionStateStorage.class);
- try {
- LOG.info("Starting transaction service");
- txService.startAndWait();
- } catch (Exception e) {
- LOG.error("Failed to start service: ", e);
- }
- }
-
- @Before
- public void reset() throws Exception {
- getClient().resetState();
- }
-
- @AfterClass
- public static void stop() throws Exception {
- txService.stopAndWait();
- storage.stopAndWait();
- zkClientService.stopAndWait();
- zkServer.stopAndWait();
- }
-
- public TransactionSystemClient getClient() throws Exception {
- return injector.getInstance(TransactionSystemClient.class);
- }
-
- @Test
- public void testThriftServerStop() throws Exception {
- int nThreads = NUM_CLIENTS;
- ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
- for (int i = 0; i < nThreads; ++i) {
- executorService.submit(new Runnable() {
- @Override
- public void run() {
- try {
- TransactionSystemClient txClient = getClient();
- CLIENTS_DONE_LATCH.countDown();
- txClient.startShort();
- } catch (Exception e) {
- // Exception expected
- }
- }
- });
- }
-
- // Wait till all clients finish sending reqeust to transaction manager
- CLIENTS_DONE_LATCH.await();
- TimeUnit.SECONDS.sleep(1);
-
- // Expire zookeeper session, which causes Thrift server to stop.
- expireZkSession(zkClientService);
- waitForThriftTermination();
-
- // Stop Zookeeper client so that it does not re-connect to Zookeeper and start Thrift sever again.
- zkClientService.stopAndWait();
- STORAGE_WAIT_LATCH.countDown();
- TimeUnit.SECONDS.sleep(1);
-
- // Make sure Thrift server stopped.
- Assert.assertEquals(Service.State.TERMINATED, txService.thriftRPCServerState());
- }
-
- private void expireZkSession(ZKClientService zkClientService) throws Exception {
- ZooKeeper zooKeeper = zkClientService.getZooKeeperSupplier().get();
- final SettableFuture<?> connectFuture = SettableFuture.create();
- Watcher watcher = new Watcher() {
- @Override
- public void process(WatchedEvent event) {
- if (event.getState() == Event.KeeperState.SyncConnected) {
- connectFuture.set(null);
- }
- }
- };
-
- // Create another Zookeeper session with the same sessionId so that the original one expires.
- final ZooKeeper dupZookeeper =
- new ZooKeeper(zkClientService.getConnectString(), zooKeeper.getSessionTimeout(), watcher,
- zooKeeper.getSessionId(), zooKeeper.getSessionPasswd());
- connectFuture.get(30, TimeUnit.SECONDS);
- Assert.assertEquals("Failed to re-create current session", dupZookeeper.getState(), ZooKeeper.States.CONNECTED);
- dupZookeeper.close();
- }
-
- private void waitForThriftTermination() throws InterruptedException {
- int count = 0;
- while (txService.thriftRPCServerState() != Service.State.TERMINATED && count++ < 200) {
- TimeUnit.MILLISECONDS.sleep(50);
- }
- }
-
- private static class SlowTransactionStorage extends InMemoryTransactionStateStorage {
- @Override
- public TransactionLog createLog(long timestamp) throws IOException {
- return new SlowTransactionLog(timestamp);
- }
- }
-
- private static class SlowTransactionLog extends InMemoryTransactionStateStorage.InMemoryTransactionLog {
- public SlowTransactionLog(long timestamp) {
- super(timestamp);
- }
-
- @Override
- public void append(TransactionEdit edit) throws IOException {
- try {
- STORAGE_WAIT_LATCH.await();
- } catch (InterruptedException e) {
- LOG.error("Got exception: ", e);
- }
- super.append(edit);
- }
-
- @Override
- public void append(List<TransactionEdit> edits) throws IOException {
- try {
- STORAGE_WAIT_LATCH.await();
- } catch (InterruptedException e) {
- LOG.error("Got exception: ", e);
- }
- super.append(edits);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/hbase/AbstractTransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/hbase/AbstractTransactionVisibilityFilterTest.java b/tephra-core/src/test/java/co/cask/tephra/hbase/AbstractTransactionVisibilityFilterTest.java
deleted file mode 100644
index 6427b07..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/hbase/AbstractTransactionVisibilityFilterTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase;
-
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.util.ConfigurationFactory;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.After;
-import org.junit.Before;
-
-import java.util.List;
-
-/**
- * Common test class for TransactionVisibilityFilter implementations.
- */
-public abstract class AbstractTransactionVisibilityFilterTest {
-
- protected static final byte[] FAM = new byte[] {'f'};
- protected static final byte[] FAM2 = new byte[] {'f', '2'};
- protected static final byte[] FAM3 = new byte[] {'f', '3'};
- protected static final byte[] COL = new byte[] {'c'};
- protected static final List<byte[]> EMPTY_CHANGESET = Lists.newArrayListWithCapacity(0);
-
- protected TransactionManager txManager;
-
- @Before
- public void setup() throws Exception {
- Configuration conf = new ConfigurationFactory().get();
- conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- txManager = new TransactionManager(conf);
- txManager.startAndWait();
- }
-
- @After
- public void tearDown() throws Exception {
- txManager.stopAndWait();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/persist/AbstractTransactionStateStorageTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/persist/AbstractTransactionStateStorageTest.java b/tephra-core/src/test/java/co/cask/tephra/persist/AbstractTransactionStateStorageTest.java
deleted file mode 100644
index b2bf69c..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/persist/AbstractTransactionStateStorageTest.java
+++ /dev/null
@@ -1,555 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.util.TransactionEditUtil;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Commons tests to run against the {@link TransactionStateStorage} implementations.
- */
-public abstract class AbstractTransactionStateStorageTest {
- private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionStateStorageTest.class);
- private static Random random = new Random();
-
- protected abstract Configuration getConfiguration(String testName) throws IOException;
-
- protected abstract AbstractTransactionStateStorage getStorage(Configuration conf);
-
- @Test
- public void testSnapshotPersistence() throws Exception {
- Configuration conf = getConfiguration("testSnapshotPersistence");
-
- TransactionSnapshot snapshot = createRandomSnapshot();
- TransactionStateStorage storage = getStorage(conf);
- try {
- storage.startAndWait();
- storage.writeSnapshot(snapshot);
-
- TransactionSnapshot readSnapshot = storage.getLatestSnapshot();
- assertNotNull(readSnapshot);
- assertEquals(snapshot, readSnapshot);
- } finally {
- storage.stopAndWait();
- }
- }
-
- @Test
- public void testLogWriteAndRead() throws Exception {
- Configuration conf = getConfiguration("testLogWriteAndRead");
-
- // create some random entries
- List<TransactionEdit> edits = TransactionEditUtil.createRandomEdits(100);
- TransactionStateStorage storage = getStorage(conf);
- try {
- long now = System.currentTimeMillis();
- storage.startAndWait();
- TransactionLog log = storage.createLog(now);
- for (TransactionEdit edit : edits) {
- log.append(edit);
- }
- log.close();
-
- Collection<TransactionLog> logsToRead = storage.getLogsSince(now);
- // should only be our one log
- assertNotNull(logsToRead);
- assertEquals(1, logsToRead.size());
- TransactionLogReader logReader = logsToRead.iterator().next().getReader();
- assertNotNull(logReader);
-
- List<TransactionEdit> readEdits = Lists.newArrayListWithExpectedSize(edits.size());
- TransactionEdit nextEdit;
- while ((nextEdit = logReader.next()) != null) {
- readEdits.add(nextEdit);
- }
- logReader.close();
- assertEquals(edits.size(), readEdits.size());
- for (int i = 0; i < edits.size(); i++) {
- LOG.info("Checking edit " + i);
- assertEquals(edits.get(i), readEdits.get(i));
- }
- } finally {
- storage.stopAndWait();
- }
- }
-
- @Test
- public void testTransactionManagerPersistence() throws Exception {
- Configuration conf = getConfiguration("testTransactionManagerPersistence");
- conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
- // start snapshot thread, but with long enough interval so we only get snapshots on shutdown
- conf.setInt(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 600);
-
- TransactionStateStorage storage = null;
- TransactionStateStorage storage2 = null;
- TransactionStateStorage storage3 = null;
- try {
- storage = getStorage(conf);
- TransactionManager txManager = new TransactionManager
- (conf, storage, new TxMetricsCollector());
- txManager.startAndWait();
-
- // TODO: replace with new persistence tests
- final byte[] a = { 'a' };
- final byte[] b = { 'b' };
- // start a tx1, add a change A and commit
- Transaction tx1 = txManager.startShort();
- Assert.assertTrue(txManager.canCommit(tx1, Collections.singleton(a)));
- Assert.assertTrue(txManager.commit(tx1));
- // start a tx2 and add a change B
- Transaction tx2 = txManager.startShort();
- Assert.assertTrue(txManager.canCommit(tx2, Collections.singleton(b)));
- // start a tx3
- Transaction tx3 = txManager.startShort();
- // restart
- txManager.stopAndWait();
- TransactionSnapshot origState = txManager.getCurrentState();
- LOG.info("Orig state: " + origState);
-
- Thread.sleep(100);
- // starts a new tx manager
- storage2 = getStorage(conf);
- txManager = new TransactionManager(conf, storage2, new TxMetricsCollector());
- txManager.startAndWait();
-
- // check that the reloaded state matches the old
- TransactionSnapshot newState = txManager.getCurrentState();
- LOG.info("New state: " + newState);
- assertEquals(origState, newState);
-
- // commit tx2
- Assert.assertTrue(txManager.commit(tx2));
- // start another transaction, must be greater than tx3
- Transaction tx4 = txManager.startShort();
- Assert.assertTrue(tx4.getTransactionId() > tx3.getTransactionId());
- // tx1 must be visble from tx2, but tx3 and tx4 must not
- Assert.assertTrue(tx2.isVisible(tx1.getTransactionId()));
- Assert.assertFalse(tx2.isVisible(tx3.getTransactionId()));
- Assert.assertFalse(tx2.isVisible(tx4.getTransactionId()));
- // add same change for tx3
- Assert.assertFalse(txManager.canCommit(tx3, Collections.singleton(b)));
- // check visibility with new xaction
- Transaction tx5 = txManager.startShort();
- Assert.assertTrue(tx5.isVisible(tx1.getTransactionId()));
- Assert.assertTrue(tx5.isVisible(tx2.getTransactionId()));
- Assert.assertFalse(tx5.isVisible(tx3.getTransactionId()));
- Assert.assertFalse(tx5.isVisible(tx4.getTransactionId()));
- // can commit tx3?
- txManager.abort(tx3);
- txManager.abort(tx4);
- txManager.abort(tx5);
- // start new tx and verify its exclude list is empty
- Transaction tx6 = txManager.startShort();
- Assert.assertFalse(tx6.hasExcludes());
- txManager.abort(tx6);
-
- // now start 5 x claim size transactions
- Transaction tx = txManager.startShort();
- for (int i = 1; i < 50; i++) {
- tx = txManager.startShort();
- }
- origState = txManager.getCurrentState();
-
- Thread.sleep(100);
- // simulate crash by starting a new tx manager without a stopAndWait
- storage3 = getStorage(conf);
- txManager = new TransactionManager(conf, storage3, new TxMetricsCollector());
- txManager.startAndWait();
-
- // verify state again matches (this time should include WAL replay)
- newState = txManager.getCurrentState();
- assertEquals(origState, newState);
-
- // get a new transaction and verify it is greater
- Transaction txAfter = txManager.startShort();
- Assert.assertTrue(txAfter.getTransactionId() > tx.getTransactionId());
- } finally {
- if (storage != null) {
- storage.stopAndWait();
- }
- if (storage2 != null) {
- storage2.stopAndWait();
- }
- if (storage3 != null) {
- storage3.stopAndWait();
- }
- }
- }
-
- /**
- * Tests whether the committed set is advanced properly on WAL replay.
- */
- @Test
- public void testCommittedSetClearing() throws Exception {
- Configuration conf = getConfiguration("testCommittedSetClearing");
- conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
- conf.setInt(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 0); // no periodic snapshots
-
- TransactionStateStorage storage1 = null;
- TransactionStateStorage storage2 = null;
- try {
- storage1 = getStorage(conf);
- TransactionManager txManager = new TransactionManager
- (conf, storage1, new TxMetricsCollector());
- txManager.startAndWait();
-
- // TODO: replace with new persistence tests
- final byte[] a = { 'a' };
- final byte[] b = { 'b' };
- // start a tx1, add a change A and commit
- Transaction tx1 = txManager.startShort();
- Assert.assertTrue(txManager.canCommit(tx1, Collections.singleton(a)));
- Assert.assertTrue(txManager.commit(tx1));
- // start a tx2 and add a change B
- Transaction tx2 = txManager.startShort();
- Assert.assertTrue(txManager.canCommit(tx2, Collections.singleton(b)));
- // start a tx3
- Transaction tx3 = txManager.startShort();
- TransactionSnapshot origState = txManager.getCurrentState();
- LOG.info("Orig state: " + origState);
-
- // simulate a failure by starting a new tx manager without stopping first
- storage2 = getStorage(conf);
- txManager = new TransactionManager(conf, storage2, new TxMetricsCollector());
- txManager.startAndWait();
-
- // check that the reloaded state matches the old
- TransactionSnapshot newState = txManager.getCurrentState();
- LOG.info("New state: " + newState);
- assertEquals(origState, newState);
-
- } finally {
- if (storage1 != null) {
- storage1.stopAndWait();
- }
- if (storage2 != null) {
- storage2.stopAndWait();
- }
- }
- }
-
- /**
- * Tests removal of old snapshots and old transaction logs.
- */
- @Test
- public void testOldFileRemoval() throws Exception {
- Configuration conf = getConfiguration("testOldFileRemoval");
- TransactionStateStorage storage = null;
- try {
- storage = getStorage(conf);
- storage.startAndWait();
- long now = System.currentTimeMillis();
- long writePointer = 1;
- Collection<Long> invalid = Lists.newArrayList();
- NavigableMap<Long, TransactionManager.InProgressTx> inprogress = Maps.newTreeMap();
- Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
- Map<Long, Set<ChangeId>> committed = Maps.newHashMap();
- TransactionSnapshot snapshot = new TransactionSnapshot(now, 0, writePointer++, invalid,
- inprogress, committing, committed);
- TransactionEdit dummyEdit = TransactionEdit.createStarted(1, 0, Long.MAX_VALUE, TransactionType.SHORT);
-
- // write snapshot 1
- storage.writeSnapshot(snapshot);
- TransactionLog log = storage.createLog(now);
- log.append(dummyEdit);
- log.close();
-
- snapshot = new TransactionSnapshot(now + 1, 0, writePointer++, invalid, inprogress, committing, committed);
- // write snapshot 2
- storage.writeSnapshot(snapshot);
- log = storage.createLog(now + 1);
- log.append(dummyEdit);
- log.close();
-
- snapshot = new TransactionSnapshot(now + 2, 0, writePointer++, invalid, inprogress, committing, committed);
- // write snapshot 3
- storage.writeSnapshot(snapshot);
- log = storage.createLog(now + 2);
- log.append(dummyEdit);
- log.close();
-
- snapshot = new TransactionSnapshot(now + 3, 0, writePointer++, invalid, inprogress, committing, committed);
- // write snapshot 4
- storage.writeSnapshot(snapshot);
- log = storage.createLog(now + 3);
- log.append(dummyEdit);
- log.close();
-
- snapshot = new TransactionSnapshot(now + 4, 0, writePointer++, invalid, inprogress, committing, committed);
- // write snapshot 5
- storage.writeSnapshot(snapshot);
- log = storage.createLog(now + 4);
- log.append(dummyEdit);
- log.close();
-
- snapshot = new TransactionSnapshot(now + 5, 0, writePointer++, invalid, inprogress, committing, committed);
- // write snapshot 6
- storage.writeSnapshot(snapshot);
- log = storage.createLog(now + 5);
- log.append(dummyEdit);
- log.close();
-
- List<String> allSnapshots = storage.listSnapshots();
- LOG.info("All snapshots: " + allSnapshots);
- assertEquals(6, allSnapshots.size());
- List<String> allLogs = storage.listLogs();
- LOG.info("All logs: " + allLogs);
- assertEquals(6, allLogs.size());
-
- long oldestKept = storage.deleteOldSnapshots(3);
- assertEquals(now + 3, oldestKept);
- allSnapshots = storage.listSnapshots();
- LOG.info("All snapshots: " + allSnapshots);
- assertEquals(3, allSnapshots.size());
-
- storage.deleteLogsOlderThan(oldestKept);
- allLogs = storage.listLogs();
- LOG.info("All logs: " + allLogs);
- assertEquals(3, allLogs.size());
- } finally {
- if (storage != null) {
- storage.stopAndWait();
- }
- }
- }
-
- @Test
- public void testLongTxnEditReplay() throws Exception {
- Configuration conf = getConfiguration("testLongTxnEditReplay");
- TransactionStateStorage storage = null;
- try {
- storage = getStorage(conf);
- storage.startAndWait();
-
- // Create long running txns. Abort one of them, invalidate another, invalidate and abort the last.
- long time1 = System.currentTimeMillis();
- long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit1 = TransactionEdit.createStarted(wp1, wp1 - 10, time1 + 100000, TransactionType.LONG);
- TransactionEdit edit2 = TransactionEdit.createAborted(wp1, TransactionType.LONG, null);
-
- long time2 = time1 + 100;
- long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit3 = TransactionEdit.createStarted(wp2, wp2 - 10, time2 + 100000, TransactionType.LONG);
- TransactionEdit edit4 = TransactionEdit.createInvalid(wp2);
-
- long time3 = time1 + 200;
- long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit5 = TransactionEdit.createStarted(wp3, wp3 - 10, time3 + 100000, TransactionType.LONG);
- TransactionEdit edit6 = TransactionEdit.createInvalid(wp3);
- TransactionEdit edit7 = TransactionEdit.createAborted(wp3, TransactionType.LONG, null);
-
- // write transaction edits
- TransactionLog log = storage.createLog(time1);
- log.append(edit1);
- log.append(edit2);
- log.append(edit3);
- log.append(edit4);
- log.append(edit5);
- log.append(edit6);
- log.append(edit7);
- log.close();
-
- // Start transaction manager
- TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
- txm.startAndWait();
- try {
- // Verify that all txns are in invalid list.
- TransactionSnapshot snapshot1 = txm.getCurrentState();
- Assert.assertEquals(ImmutableList.of(wp1, wp2, wp3), snapshot1.getInvalid());
- Assert.assertEquals(0, snapshot1.getInProgress().size());
- Assert.assertEquals(0, snapshot1.getCommittedChangeSets().size());
- Assert.assertEquals(0, snapshot1.getCommittedChangeSets().size());
- } finally {
- txm.stopAndWait();
- }
- } finally {
- if (storage != null) {
- storage.stopAndWait();
- }
- }
- }
-
- @Test
- public void testTruncateInvalidTxEditReplay() throws Exception {
- Configuration conf = getConfiguration("testTruncateInvalidTxEditReplay");
- TransactionStateStorage storage = null;
- try {
- storage = getStorage(conf);
- storage.startAndWait();
-
- // Create some txns, and invalidate all of them.
- long time1 = System.currentTimeMillis();
- long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit1 = TransactionEdit.createStarted(wp1, wp1 - 10, time1 + 100000, TransactionType.LONG);
- TransactionEdit edit2 = TransactionEdit.createInvalid(wp1);
-
- long time2 = time1 + 100;
- long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit3 = TransactionEdit.createStarted(wp2, wp2 - 10, time2 + 10000, TransactionType.SHORT);
- TransactionEdit edit4 = TransactionEdit.createInvalid(wp2);
-
- long time3 = time1 + 2000;
- long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit5 = TransactionEdit.createStarted(wp3, wp3 - 10, time3 + 100000, TransactionType.LONG);
- TransactionEdit edit6 = TransactionEdit.createInvalid(wp3);
-
- long time4 = time1 + 2100;
- long wp4 = time4 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit7 = TransactionEdit.createStarted(wp4, wp4 - 10, time4 + 10000, TransactionType.SHORT);
- TransactionEdit edit8 = TransactionEdit.createInvalid(wp4);
-
- // remove wp1 and wp3 from invalid list
- TransactionEdit edit9 = TransactionEdit.createTruncateInvalidTx(ImmutableSet.of(wp1, wp3));
- // truncate invalid transactions before time3
- TransactionEdit edit10 = TransactionEdit.createTruncateInvalidTxBefore(time3);
-
- // write transaction edits
- TransactionLog log = storage.createLog(time1);
- log.append(edit1);
- log.append(edit2);
- log.append(edit3);
- log.append(edit4);
- log.append(edit5);
- log.append(edit6);
- log.append(edit7);
- log.append(edit8);
- log.append(edit9);
- log.append(edit10);
- log.close();
-
- // Start transaction manager
- TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
- txm.startAndWait();
- try {
- // Only wp4 should be in invalid list.
- TransactionSnapshot snapshot = txm.getCurrentState();
- Assert.assertEquals(ImmutableList.of(wp4), snapshot.getInvalid());
- Assert.assertEquals(0, snapshot.getInProgress().size());
- Assert.assertEquals(0, snapshot.getCommittedChangeSets().size());
- Assert.assertEquals(0, snapshot.getCommittedChangeSets().size());
- } finally {
- txm.stopAndWait();
- }
- } finally {
- if (storage != null) {
- storage.stopAndWait();
- }
- }
- }
-
- /**
- * Generates a new snapshot object with semi-randomly populated values. This does not necessarily accurately
- * represent a typical snapshot's distribution of values, as we only set an upper bound on pointer values.
- *
- * We generate a new snapshot with the contents:
- * <ul>
- * <li>readPointer = 1M + (random % 1M)</li>
- * <li>writePointer = readPointer + 1000</li>
- * <li>waterMark = writePointer + 1000</li>
- * <li>inProgress = one each for (writePointer - 500)..writePointer, ~ 5% "long" transaction</li>
- * <li>invalid = 100 randomly distributed, 0..1M</li>
- * <li>committing = one each, (readPointer + 1)..(readPointer + 100)</li>
- * <li>committed = one each, (readPointer - 1000)..readPointer</li>
- * </ul>
- * @return a new snapshot of transaction state.
- */
- private TransactionSnapshot createRandomSnapshot() {
- // limit readPointer to a reasonable range, but make it > 1M so we can assign enough keys below
- long readPointer = (Math.abs(random.nextLong()) % 1000000L) + 1000000L;
- long writePointer = readPointer + 1000L;
-
- // generate in progress -- assume last 500 write pointer values
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
- long startPointer = writePointer - 500L;
- for (int i = 0; i < 500; i++) {
- long currentTime = System.currentTimeMillis();
- // make some "long" transactions
- if (i % 20 == 0) {
- inProgress.put(startPointer + i,
- new TransactionManager.InProgressTx(startPointer - 1, currentTime + TimeUnit.DAYS.toSeconds(1),
- TransactionType.LONG));
- } else {
- inProgress.put(startPointer + i,
- new TransactionManager.InProgressTx(startPointer - 1, currentTime + 300000L,
- TransactionType.SHORT));
- }
- }
-
- // make 100 random invalid IDs
- LongArrayList invalid = new LongArrayList();
- for (int i = 0; i < 100; i++) {
- invalid.add(Math.abs(random.nextLong()) % 1000000L);
- }
-
- // make 100 committing entries, 10 keys each
- Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
- for (int i = 0; i < 100; i++) {
- committing.put(readPointer + i, generateChangeSet(10));
- }
-
- // make 1000 committed entries, 10 keys each
- long startCommitted = readPointer - 1000L;
- NavigableMap<Long, Set<ChangeId>> committed = Maps.newTreeMap();
- for (int i = 0; i < 1000; i++) {
- committed.put(startCommitted + i, generateChangeSet(10));
- }
-
- return new TransactionSnapshot(System.currentTimeMillis(), readPointer, writePointer,
- invalid, inProgress, committing, committed);
- }
-
- private Set<ChangeId> generateChangeSet(int numEntries) {
- Set<ChangeId> changes = Sets.newHashSet();
- for (int i = 0; i < numEntries; i++) {
- byte[] bytes = new byte[8];
- random.nextBytes(bytes);
- changes.add(new ChangeId(bytes));
- }
- return changes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/persist/CommitMarkerCodecTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/persist/CommitMarkerCodecTest.java b/tephra-core/src/test/java/co/cask/tephra/persist/CommitMarkerCodecTest.java
deleted file mode 100644
index da876fa..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/persist/CommitMarkerCodecTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import com.google.common.primitives.Ints;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-/**
- * Unit Test for {@link CommitMarkerCodec}.
- */
-public class CommitMarkerCodecTest {
-
- @ClassRule
- public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
- private static final String LOG_FILE = "txlog";
- private static final Random RANDOM = new Random();
-
- private static MiniDFSCluster dfsCluster;
- private static Configuration conf;
- private static FileSystem fs;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- Configuration hConf = new Configuration();
- hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
-
- dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
- conf = new Configuration(dfsCluster.getFileSystem().getConf());
- fs = FileSystem.newInstance(FileSystem.getDefaultUri(conf), conf);
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- dfsCluster.shutdown();
- }
-
- @Test
- public void testRandomCommitMarkers() throws Exception {
- List<Integer> randomInts = new ArrayList<>();
- Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE);
-
- // Write a bunch of random commit markers
- try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class,
- LongWritable.class,
- SequenceFile.CompressionType.NONE)) {
- for (int i = 0; i < 1000; i++) {
- int randomNum = RANDOM.nextInt(Integer.MAX_VALUE);
- CommitMarkerCodec.writeMarker(writer, randomNum);
- randomInts.add(randomNum);
- }
- writer.hflush();
- writer.hsync();
- }
-
- // Read the commit markers back to verify the marker
- try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf);
- CommitMarkerCodec markerCodec = new CommitMarkerCodec()) {
- for (int num : randomInts) {
- Assert.assertEquals(num, markerCodec.readMarker(reader));
- }
- }
- }
-
- private static class IncompleteValueBytes implements SequenceFile.ValueBytes {
-
- @Override
- public void writeUncompressedBytes(DataOutputStream outStream) throws IOException {
- // don't write anything to simulate incomplete record
- }
-
- @Override
- public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException {
- throw new IllegalArgumentException("Not possible");
- }
-
- @Override
- public int getSize() {
- return Ints.BYTES;
- }
- }
-
- @Test
- public void testIncompleteCommitMarker() throws Exception {
- Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE);
- try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class,
- LongWritable.class,
- SequenceFile.CompressionType.NONE)) {
- String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED;
- SequenceFile.ValueBytes valueBytes = new IncompleteValueBytes();
- writer.appendRaw(key.getBytes(), 0, key.length(), valueBytes);
- writer.hflush();
- writer.hsync();
- }
-
- // Read the incomplete commit marker
- try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf);
- CommitMarkerCodec markerCodec = new CommitMarkerCodec()) {
- try {
- markerCodec.readMarker(reader);
- Assert.fail("Expected EOF Exception to be thrown");
- } catch (EOFException e) {
- // expected since we didn't write the value bytes
- }
- }
- }
-
- @Test
- public void testIncorrectCommitMarker() throws Exception {
- Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE);
-
- // Write an incorrect marker
- try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class,
- LongWritable.class,
- SequenceFile.CompressionType.NONE)) {
- String invalidKey = "IncorrectKey";
- SequenceFile.ValueBytes valueBytes = new CommitMarkerCodec.CommitEntriesCount(100);
- writer.appendRaw(invalidKey.getBytes(), 0, invalidKey.length(), valueBytes);
- writer.hflush();
- writer.hsync();
- }
-
- // Read the commit markers back to verify the marker
- try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf);
- CommitMarkerCodec markerCodec = new CommitMarkerCodec()) {
- try {
- markerCodec.readMarker(reader);
- Assert.fail("Expected an IOException to be thrown");
- } catch (IOException e) {
- // expected
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionLogTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionLogTest.java b/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionLogTest.java
deleted file mode 100644
index 96015d1..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionLogTest.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.MetricsCollector;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.util.TransactionEditUtil;
-import com.google.common.io.Closeables;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-
-/**
- * Testing for complete and partial sycs of {@link TransactionEdit} to {@link HDFSTransactionLog}
- */
-public class HDFSTransactionLogTest {
- @ClassRule
- public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
- private static final String LOG_FILE_PREFIX = "txlog.";
-
- private static MiniDFSCluster dfsCluster;
- private static Configuration conf;
- private static MetricsCollector metricsCollector;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- Configuration hConf = new Configuration();
- hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
-
- dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
- conf = new Configuration(dfsCluster.getFileSystem().getConf());
- metricsCollector = new TxMetricsCollector();
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- dfsCluster.shutdown();
- }
-
- private Configuration getConfiguration() throws IOException {
- // tests should use the current user for HDFS
- conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
- return conf;
- }
-
- private HDFSTransactionLog getHDFSTransactionLog(Configuration conf,
- FileSystem fs, long timeInMillis) throws Exception {
- String snapshotDir = conf.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR);
- Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timeInMillis);
- return new HDFSTransactionLog(fs, conf, newLog, timeInMillis, metricsCollector);
- }
-
- private SequenceFile.Writer getSequenceFileWriter(Configuration configuration, FileSystem fs,
- long timeInMillis, boolean withMarker) throws IOException {
- String snapshotDir = configuration.get(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR);
- Path newLog = new Path(snapshotDir, LOG_FILE_PREFIX + timeInMillis);
- SequenceFile.Metadata metadata = new SequenceFile.Metadata();
- if (withMarker) {
- metadata.set(new Text(TxConstants.TransactionLog.VERSION_KEY),
- new Text(Byte.toString(TxConstants.TransactionLog.CURRENT_VERSION)));
- }
- return SequenceFile.createWriter(fs, configuration, newLog, LongWritable.class,
- TransactionEdit.class, SequenceFile.CompressionType.NONE, null, null, metadata);
- }
-
- private void writeNumWrites(SequenceFile.Writer writer, final int size) throws Exception {
- String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED;
- CommitMarkerCodec.writeMarker(writer, size);
- }
-
- private void testTransactionLogSync(int totalCount, int batchSize, boolean withMarker, boolean isComplete)
- throws Exception {
- List<TransactionEdit> edits = TransactionEditUtil.createRandomEdits(totalCount);
- long timestamp = System.currentTimeMillis();
- Configuration configuration = getConfiguration();
- FileSystem fs = FileSystem.newInstance(FileSystem.getDefaultUri(configuration), configuration);
- SequenceFile.Writer writer = getSequenceFileWriter(configuration, fs, timestamp, withMarker);
- AtomicLong logSequence = new AtomicLong();
- HDFSTransactionLog transactionLog = getHDFSTransactionLog(configuration, fs, timestamp);
- AbstractTransactionLog.Entry entry;
-
- for (int i = 0; i < totalCount - batchSize; i += batchSize) {
- if (withMarker) {
- writeNumWrites(writer, batchSize);
- }
- for (int j = 0; j < batchSize; j++) {
- entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(j));
- writer.append(entry.getKey(), entry.getEdit());
- }
- writer.syncFs();
- }
-
- if (withMarker) {
- writeNumWrites(writer, batchSize);
- }
-
- for (int i = totalCount - batchSize; i < totalCount - 1; i++) {
- entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(i));
- writer.append(entry.getKey(), entry.getEdit());
- }
-
- entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()),
- edits.get(totalCount - 1));
- if (isComplete) {
- writer.append(entry.getKey(), entry.getEdit());
- } else {
- byte[] bytes = Longs.toByteArray(entry.getKey().get());
- writer.appendRaw(bytes, 0, bytes.length, new SequenceFile.ValueBytes() {
- @Override
- public void writeUncompressedBytes(DataOutputStream outStream) throws IOException {
- byte[] test = new byte[]{0x2};
- outStream.write(test, 0, 1);
- }
-
- @Override
- public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException {
- // no-op
- }
-
- @Override
- public int getSize() {
- // mimic size longer than the actual byte array size written, so we would reach EOF
- return 12;
- }
- });
- }
- writer.syncFs();
- Closeables.closeQuietly(writer);
-
- // now let's try to read this log
- TransactionLogReader reader = transactionLog.getReader();
- int syncedEdits = 0;
- while (reader.next() != null) {
- // testing reading the transaction edits
- syncedEdits++;
- }
- if (isComplete) {
- Assert.assertEquals(totalCount, syncedEdits);
- } else {
- Assert.assertEquals(totalCount - batchSize, syncedEdits);
- }
- }
-
- @Test
- public void testTransactionLogNewVersion() throws Exception {
- // in-complete sync
- testTransactionLogSync(1000, 1, true, false);
- testTransactionLogSync(2000, 5, true, false);
-
- // complete sync
- testTransactionLogSync(1000, 1, true, true);
- testTransactionLogSync(2000, 5, true, true);
- }
-
- @Test
- public void testTransactionLogOldVersion() throws Exception {
- // in-complete sync
- testTransactionLogSync(1000, 1, false, false);
-
- // complete sync
- testTransactionLogSync(2000, 5, false, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionStateStorageTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionStateStorageTest.java b/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionStateStorageTest.java
deleted file mode 100644
index 30ce455..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/persist/HDFSTransactionStateStorageTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.IOException;
-
-
-/**
- * Tests persistence of transaction snapshots and write-ahead logs to HDFS storage, using the
- * {@link HDFSTransactionStateStorage} and {@link HDFSTransactionLog} implementations.
- */
-public class HDFSTransactionStateStorageTest extends AbstractTransactionStateStorageTest {
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- private static MiniDFSCluster dfsCluster;
- private static Configuration conf;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- Configuration hConf = new Configuration();
- hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
-
- dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
- conf = new Configuration(dfsCluster.getFileSystem().getConf());
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- dfsCluster.shutdown();
- }
-
- @Override
- protected Configuration getConfiguration(String testName) throws IOException {
- // tests should use the current user for HDFS
- conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());
- return conf;
- }
-
- @Override
- protected AbstractTransactionStateStorage getStorage(Configuration conf) {
- return new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/persist/InMemoryTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/persist/InMemoryTransactionStateStorage.java b/tephra-core/src/test/java/co/cask/tephra/persist/InMemoryTransactionStateStorage.java
deleted file mode 100644
index 1b69c7a..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/persist/InMemoryTransactionStateStorage.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.AbstractIdleService;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-import javax.annotation.Nullable;
-
-/**
- * Stores the latest transaction snapshot and logs in memory.
- */
-public class InMemoryTransactionStateStorage extends AbstractIdleService implements TransactionStateStorage {
- // only keeps the most recent snapshot in memory
- private TransactionSnapshot lastSnapshot;
-
- private NavigableMap<Long, TransactionLog> logs = new TreeMap<>();
-
- @Override
- protected void startUp() throws Exception {
- }
-
- @Override
- protected void shutDown() throws Exception {
- lastSnapshot = null;
- logs = new TreeMap<>();
- }
-
- @Override
- public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException {
- // no codecs in in-memory mode
- }
-
- @Override
- public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
- lastSnapshot = snapshot;
- }
-
- @Override
- public TransactionSnapshot getLatestSnapshot() throws IOException {
- return lastSnapshot;
- }
-
- @Override
- public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
- return lastSnapshot;
- }
-
- @Override
- public long deleteOldSnapshots(int numberToKeep) throws IOException {
- // always only keep the last snapshot
- return lastSnapshot.getTimestamp();
- }
-
- @Override
- public List<String> listSnapshots() throws IOException {
- List<String> snapshots = Lists.newArrayListWithCapacity(1);
- if (lastSnapshot != null) {
- snapshots.add(Long.toString(lastSnapshot.getTimestamp()));
- }
- return snapshots;
- }
-
- @Override
- public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
- return Lists.newArrayList(logs.tailMap(timestamp).values());
- }
-
- @Override
- public TransactionLog createLog(long timestamp) throws IOException {
- TransactionLog log = new InMemoryTransactionLog(timestamp);
- logs.put(timestamp, log);
- return log;
- }
-
- @Override
- public void deleteLogsOlderThan(long timestamp) throws IOException {
- Iterator<Map.Entry<Long, TransactionLog>> logIter = logs.entrySet().iterator();
- while (logIter.hasNext()) {
- Map.Entry<Long, TransactionLog> logEntry = logIter.next();
- if (logEntry.getKey() < timestamp) {
- logIter.remove();
- }
- }
- }
-
- @Override
- public void setupStorage() throws IOException {
- }
-
- @Override
- public List<String> listLogs() throws IOException {
- return Lists.transform(Lists.newArrayList(logs.keySet()), new Function<Long, String>() {
- @Nullable
- @Override
- public String apply(@Nullable Long input) {
- return input.toString();
- }
- });
- }
-
- @Override
- public String getLocation() {
- return "in-memory";
- }
-
- public static class InMemoryTransactionLog implements TransactionLog {
- private long timestamp;
- private List<TransactionEdit> edits = Lists.newArrayList();
- boolean isClosed = false;
- public InMemoryTransactionLog(long timestamp) {
- this.timestamp = timestamp;
- }
-
- @Override
- public String getName() {
- return "in-memory@" + timestamp;
- }
-
- @Override
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public void append(TransactionEdit edit) throws IOException {
- if (isClosed) {
- throw new IOException("Log is closed");
- }
- edits.add(edit);
- }
-
- @Override
- public void append(List<TransactionEdit> edits) throws IOException {
- if (isClosed) {
- throw new IOException("Log is closed");
- }
- edits.addAll(edits);
- }
-
- @Override
- public void close() {
- isClosed = true;
- }
-
- @Override
- public TransactionLogReader getReader() throws IOException {
- return new InMemoryLogReader(edits.iterator());
- }
- }
-
- public static class InMemoryLogReader implements TransactionLogReader {
- private final Iterator<TransactionEdit> editIterator;
-
- public InMemoryLogReader(Iterator<TransactionEdit> editIterator) {
- this.editIterator = editIterator;
- }
-
- @Override
- public TransactionEdit next() throws IOException {
- if (editIterator.hasNext()) {
- return editIterator.next();
- }
- return null;
- }
-
- @Override
- public TransactionEdit next(TransactionEdit reuse) throws IOException {
- return next();
- }
-
- @Override
- public void close() throws IOException {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/test/java/co/cask/tephra/persist/LocalTransactionStateStorageTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/co/cask/tephra/persist/LocalTransactionStateStorageTest.java b/tephra-core/src/test/java/co/cask/tephra/persist/LocalTransactionStateStorageTest.java
deleted file mode 100644
index e2886ae..0000000
--- a/tephra-core/src/test/java/co/cask/tephra/persist/LocalTransactionStateStorageTest.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.snapshot.DefaultSnapshotCodec;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.snapshot.SnapshotCodecV4;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Assert;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Runs transaction persistence tests against the {@link LocalFileTransactionStateStorage} and
- * {@link LocalFileTransactionLog} implementations.
- */
-public class LocalTransactionStateStorageTest extends AbstractTransactionStateStorageTest {
- @ClassRule
- public static TemporaryFolder tmpDir = new TemporaryFolder();
-
- @Override
- protected Configuration getConfiguration(String testName) throws IOException {
- File testDir = tmpDir.newFolder(testName);
- Configuration conf = new Configuration();
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath());
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName());
- return conf;
- }
-
- @Override
- protected AbstractTransactionStateStorage getStorage(Configuration conf) {
- return new LocalFileTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
- }
-
- // v2 TransactionEdit
- @SuppressWarnings("deprecation")
- private class TransactionEditV2 extends TransactionEdit {
- public TransactionEditV2(long writePointer, long visibilityUpperBound, State state, long expirationDate,
- Set<ChangeId> changes, long commitPointer, boolean canCommit, TransactionType type) {
- super(writePointer, visibilityUpperBound, state, expirationDate, changes, commitPointer, canCommit, type,
- null, 0L, 0L, null);
- }
- @Override
- public void write(DataOutput out) throws IOException {
- TransactionEditCodecs.encode(this, out, new TransactionEditCodecs.TransactionEditCodecV2());
- }
- }
-
- // Note: this test cannot run in AbstractTransactionStateStorageTest, since SequenceFile throws exception saying
- // TransactionEditV2 is not TransactionEdit. Since the code path this test is verifying is the same path between
- // HDFS and Local Storage, having this only over here is fine.
- @SuppressWarnings("deprecation")
- @Test
- public void testLongTxnBackwardsCompatibility() throws Exception {
- Configuration conf = getConfiguration("testLongTxnBackwardsCompatibility");
-
- // Use SnapshotCodec version 1
- String latestSnapshotCodec = conf.get(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
-
- TransactionStateStorage storage = null;
- try {
- storage = getStorage(conf);
- storage.startAndWait();
-
- // Create transaction snapshot and transaction edits with version when long running txns had -1 expiration.
- Collection<Long> invalid = Lists.newArrayList();
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
- long time1 = System.currentTimeMillis();
- long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
- inProgress.put(wp1, new TransactionManager.InProgressTx(wp1 - 5, -1L));
- long time2 = time1 + 100;
- long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
- inProgress.put(wp2, new TransactionManager.InProgressTx(wp2 - 50, time2 + 1000));
- Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
- Map<Long, Set<ChangeId>> committed = Maps.newHashMap();
- TransactionSnapshot snapshot = new TransactionSnapshot(time2, 0, wp2, invalid,
- inProgress, committing, committed);
- long time3 = time1 + 200;
- long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit1 = new TransactionEditV2(wp3, wp3 - 10, TransactionEdit.State.INPROGRESS, -1L,
- null, 0L, false, null);
- long time4 = time1 + 300;
- long wp4 = time4 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit2 = new TransactionEditV2(wp4, wp4 - 10, TransactionEdit.State.INPROGRESS, time4 + 1000,
- null, 0L, false, null);
-
- // write snapshot and transaction edit
- storage.writeSnapshot(snapshot);
- TransactionLog log = storage.createLog(time2);
- log.append(edit1);
- log.append(edit2);
- log.close();
-
- // Start transaction manager
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, latestSnapshotCodec);
- long longTimeout = TimeUnit.SECONDS.toMillis(conf.getLong(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
- TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT));
- TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
- txm.startAndWait();
- try {
- // Verify that the txns in old format were read correctly.
- // There should be four in-progress transactions, and no invalid transactions
- TransactionSnapshot snapshot1 = txm.getCurrentState();
- Assert.assertEquals(ImmutableSortedSet.of(wp1, wp2, wp3, wp4), snapshot1.getInProgress().keySet());
- verifyInProgress(snapshot1.getInProgress().get(wp1), TransactionType.LONG, time1 + longTimeout);
- verifyInProgress(snapshot1.getInProgress().get(wp2), TransactionType.SHORT, time2 + 1000);
- verifyInProgress(snapshot1.getInProgress().get(wp3), TransactionType.LONG, time3 + longTimeout);
- verifyInProgress(snapshot1.getInProgress().get(wp4), TransactionType.SHORT, time4 + 1000);
- Assert.assertEquals(0, snapshot1.getInvalid().size());
- } finally {
- txm.stopAndWait();
- }
- } finally {
- if (storage != null) {
- storage.stopAndWait();
- }
- }
- }
-
- // Note: this test cannot run in AbstractTransactionStateStorageTest, since SequenceFile throws exception saying
- // TransactionEditV2 is not TransactionEdit. Since the code path this test is verifying is the same path between
- // HDFS and Local Storage, having this only over here is fine.
- @SuppressWarnings("deprecation")
- @Test
- public void testAbortEditBackwardsCompatibility() throws Exception {
- Configuration conf = getConfiguration("testAbortEditBackwardsCompatibility");
-
- TransactionStateStorage storage = null;
- try {
- storage = getStorage(conf);
- storage.startAndWait();
-
- // Create edits for transaction type addition to abort
- long time1 = System.currentTimeMillis();
- long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit1 = new TransactionEditV2(wp1, wp1 - 10, TransactionEdit.State.INPROGRESS, -1L,
- null, 0L, false, null);
- TransactionEdit edit2 = new TransactionEditV2(wp1, 0L, TransactionEdit.State.ABORTED, 0L,
- null, 0L, false, null);
-
- long time2 = time1 + 400;
- long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
- TransactionEdit edit3 = new TransactionEditV2(wp2, wp2 - 10, TransactionEdit.State.INPROGRESS, time2 + 10000,
- null, 0L, false, null);
- TransactionEdit edit4 = new TransactionEditV2(wp2, 0L, TransactionEdit.State.INVALID, 0L, null, 0L, false, null);
- // Simulate case where we cannot determine txn state during abort
- TransactionEdit edit5 = new TransactionEditV2(wp2, 0L, TransactionEdit.State.ABORTED, 0L, null, 0L, false, null);
-
- // write snapshot and transaction edit
- TransactionLog log = storage.createLog(time1);
- log.append(edit1);
- log.append(edit2);
- log.append(edit3);
- log.append(edit4);
- log.append(edit5);
- log.close();
-
- // Start transaction manager
- TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
- txm.startAndWait();
- try {
- // Verify that the txns in old format were read correctly.
- // Both transactions should be in invalid state
- TransactionSnapshot snapshot1 = txm.getCurrentState();
- Assert.assertEquals(ImmutableList.of(wp1, wp2), snapshot1.getInvalid());
- Assert.assertEquals(0, snapshot1.getInProgress().size());
- Assert.assertEquals(0, snapshot1.getCommittedChangeSets().size());
- Assert.assertEquals(0, snapshot1.getCommittingChangeSets().size());
- } finally {
- txm.stopAndWait();
- }
- } finally {
- if (storage != null) {
- storage.stopAndWait();
- }
- }
- }
-
- private void verifyInProgress(TransactionManager.InProgressTx inProgressTx, TransactionType type,
- long expiration) throws Exception {
- Assert.assertEquals(type, inProgressTx.getType());
- Assert.assertTrue(inProgressTx.getExpiration() == expiration);
- }
-}
[11/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
new file mode 100644
index 0000000..45ac114
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh.coprocessor;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCodec;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+
+/**
+ * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
+ * for transactions:
+ * <ul>
+ * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
+ * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
+ * or expired due to TTL.</li>
+ * </ul>
+ *
+ * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
+ * or on all user tables by adding the following to hbase-site.xml:
+ * {@code
+ * <property>
+ * <name>hbase.coprocessor.region.classes</name>
+ * <value>TransactionProcessor</value>
+ * </property>
+ * }
+ * </p>
+ *
+ * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
+ * as an attribute:
+ * {@code
+ * Transaction t = ...;
+ * Get get = new Get(...);
+ * TransactionCodec codec = new TransactionCodec();
+ * codec.addToOperation(get, t);
+ * }
+ * </p>
+ */
+public class TransactionProcessor extends BaseRegionObserver {
+ private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
+
+ private TransactionStateCache cache;
+ private final TransactionCodec txCodec;
+ protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
+ protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
+
+ public TransactionProcessor() {
+ this.txCodec = new TransactionCodec();
+ }
+
+ /* RegionObserver implementation */
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ if (e instanceof RegionCoprocessorEnvironment) {
+ RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
+ Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
+ this.cache = cacheSupplier.get();
+
+ HTableDescriptor tableDesc = env.getRegion().getTableDesc();
+ for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
+ String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
+ long ttl = 0;
+ if (columnTTL != null) {
+ try {
+ ttl = Long.parseLong(columnTTL);
+ LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
+ ", value = " + columnTTL);
+ }
+ }
+ ttlByFamily.put(columnDesc.getName(), ttl);
+ }
+
+ this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
+ TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
+ this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
+ if (readNonTxnData) {
+ LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
+ }
+ }
+ }
+
+ protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
+ return new TransactionStateCacheSupplier(env.getConfiguration());
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ // nothing to do
+ }
+
+ @Override
+ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
+ throws IOException {
+ Transaction tx = getFromOperation(get);
+ if (tx != null) {
+ projectFamilyDeletes(get);
+ get.setMaxVersions();
+ get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
+ get.setFilter(newFilter);
+ }
+ }
+
+ @Override
+ public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
+ Durability durability) throws IOException {
+ // Translate deletes into our own delete tombstones
+ // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
+ // us to rollback the changes (by a real delete) if the transaction fails
+
+ // Deletes that are part of a transaction rollback do not need special handling.
+ // They will never be rolled back, so are performed as normal HBase deletes.
+ if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ return;
+ }
+
+ // Other deletes are client-initiated and need to be translated into our own tombstones
+ // TODO: this should delegate to the DeleteStrategy implementation.
+ Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
+ for (byte[] family : delete.getFamilyCellMap().keySet()) {
+ List<Cell> familyCells = delete.getFamilyCellMap().get(family);
+ if (isFamilyDelete(familyCells)) {
+ deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ } else {
+ int cellSize = familyCells.size();
+ for (int i = 0; i < cellSize; i++) {
+ Cell cell = familyCells.get(i);
+ deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
+ }
+ e.getEnvironment().getRegion().put(deleteMarkers);
+ // skip normal delete handling
+ e.bypass();
+ }
+
+ private boolean isFamilyDelete(List<Cell> familyCells) {
+ return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
+ }
+
+ @Override
+ public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
+ throws IOException {
+ Transaction tx = getFromOperation(scan);
+ if (tx != null) {
+ projectFamilyDeletes(scan);
+ scan.setMaxVersions();
+ scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
+ scan.setFilter(newFilter);
+ }
+ return s;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any scan operation.
+ * @param scan The original scan request
+ * @return The modified scan request with the family delete qualifiers represented
+ */
+ private Scan projectFamilyDeletes(Scan scan) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return scan;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any get operation.
+ * @param get The original get request
+ * @return The modified get request with the family delete qualifiers represented
+ */
+ private Get projectFamilyDeletes(Get get) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return get;
+ }
+
+ @Override
+ public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ KeyValueScanner memstoreScanner, InternalScanner scanner)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
+ Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
+ HConstants.OLDEST_TIMESTAMP);
+ }
+
+ @Override
+ public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
+ CompactionRequest request)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners,
+ scanType, earliestPutTs);
+ }
+
+ protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
+ TransactionVisibilityState snapshot, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType type,
+ long earliestPutTs) throws IOException {
+ if (snapshot == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
+ ", no current transaction state found, defaulting to normal " + action + " scanner");
+ }
+ return null;
+ }
+
+ // construct a dummy transaction from the latest snapshot
+ Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
+ Scan scan = new Scan();
+ // need to see all versions, since we filter out excludes and applications may rely on multiple versions
+ scan.setMaxVersions();
+ scan.setFilter(
+ new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
+ snapshot.getInvalid(),
+ getTransactionFilter(dummyTx, type, null)));
+
+ return new StoreScanner(store, store.getScanInfo(), scan, scanners,
+ type, store.getSmallestReadPoint(), earliestPutTs);
+ }
+
+ private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
+ byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded != null) {
+ return txCodec.decode(encoded);
+ }
+ return null;
+ }
+
+ /**
+ * Derived classes can override this method to customize the filter used to return data visible for the current
+ * transaction.
+ *
+ * @param tx the current transaction to apply
+ * @param type the type of scan being performed
+ */
+ protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
+ return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+ }
+
+ /**
+ * Filter used to include cells visible to in-progress transactions on flush and commit.
+ */
+ static class IncludeInProgressFilter extends FilterBase {
+ private final long visibilityUpperBound;
+ private final Set<Long> invalidIds;
+ private final Filter txFilter;
+
+ public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
+ this.visibilityUpperBound = upperBound;
+ this.invalidIds = Sets.newHashSet(invalids);
+ this.txFilter = transactionFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ // include all cells visible to in-progress transactions, except for those already marked as invalid
+ long ts = cell.getTimestamp();
+ if (ts > visibilityUpperBound) {
+ // include everything that could still be in-progress except invalids
+ if (invalidIds.contains(ts)) {
+ return ReturnCode.SKIP;
+ }
+ return ReturnCode.INCLUDE;
+ }
+ return txFilter.filterKeyValue(cell);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java
new file mode 100644
index 0000000..5e727c6
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilter.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh.coprocessor;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Applies filtering of data based on transactional visibility (HBase 1.0+ specific version).
+ * Note: this is intended for server-side use only, as additional properties need to be set on
+ * any {@code Scan} or {@code Get} operation performed.
+ */
+public class TransactionVisibilityFilter extends FilterBase {
+ private final Transaction tx;
+ // oldest visible timestamp by column family, used to apply TTL when reading
+ private final Map<ImmutableBytesWritable, Long> oldestTsByFamily;
+ // if false, empty values will be interpreted as deletes
+ private final boolean allowEmptyValues;
+ // whether or not we can remove delete markers
+ // these can only be safely removed when we are traversing all storefiles
+ private final boolean clearDeletes;
+ // optional sub-filter to apply to visible cells
+ private final Filter cellFilter;
+ // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
+ private final ImmutableBytesWritable currentFamily = new ImmutableBytesWritable(HConstants.EMPTY_BYTE_ARRAY);
+
+ private long currentOldestTs;
+
+ private DeleteTracker deleteTracker = new DeleteTracker();
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+ }
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ this.tx = tx;
+ this.oldestTsByFamily = Maps.newTreeMap();
+ for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
+ long familyTTL = ttlEntry.getValue();
+ oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
+ familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+ }
+ this.allowEmptyValues = allowEmptyValues;
+ this.clearDeletes =
+ scanType == ScanType.COMPACT_DROP_DELETES ||
+ (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ this.cellFilter = cellFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (!CellUtil.matchingFamily(cell, currentFamily.get(), currentFamily.getOffset(), currentFamily.getLength())) {
+ // column family changed
+ currentFamily.set(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
+ Long familyOldestTs = oldestTsByFamily.get(currentFamily);
+ currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
+ deleteTracker.reset();
+ }
+ // need to apply TTL for the column family here
+ long kvTimestamp = cell.getTimestamp();
+ if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
+ // passed TTL for this column, seek to next
+ return ReturnCode.NEXT_COL;
+ } else if (tx.isVisible(kvTimestamp)) {
+ // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
+ // cell is visible
+ // visibility SNAPSHOT_ALL needs all matches
+ return runSubFilter(ReturnCode.INCLUDE, cell);
+ }
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ deleteTracker.addFamilyDelete(cell);
+ if (clearDeletes) {
+ return ReturnCode.NEXT_COL;
+ } else {
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // check if masked by family delete
+ if (deleteTracker.isDeleted(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+ // check for column delete
+ if (isColumnDelete(cell)) {
+ if (clearDeletes) {
+ // skip "deleted" cell
+ return ReturnCode.NEXT_COL;
+ } else {
+ // keep the marker but skip any remaining versions
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ } else {
+ return ReturnCode.SKIP;
+ }
+ }
+
+ private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
+ if (cellFilter != null) {
+ ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
+ return determineReturnCode(txFilterCode, subFilterCode);
+ }
+ return txFilterCode;
+ }
+
+ /**
+ * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
+ * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
+ * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
+ * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
+ *
+ * @param txFilterCode return code from TransactionVisibilityFilter
+ * @param subFilterCode return code from sub-filter
+ * @return final return code
+ */
+ protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
+ // Return the more restrictive of the two filter responses
+ switch (subFilterCode) {
+ case INCLUDE:
+ return txFilterCode;
+ case INCLUDE_AND_NEXT_COL:
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ case SKIP:
+ return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
+ default:
+ return subFilterCode;
+ }
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRow();
+ }
+ return super.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ // Convert Tephra deletes back into HBase deletes
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
+ KeyValue.Type.DeleteFamily);
+ } else if (isColumnDelete(cell)) {
+ // Note: in some cases KeyValue.Type.Delete is used in Delete object,
+ // and in some other cases KeyValue.Type.DeleteColumn is used.
+ // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
+ // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
+ // work in both cases.
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
+ cell.getTimestamp(), KeyValue.Type.DeleteColumn);
+ }
+ }
+ return cell;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ deleteTracker.reset();
+ if (cellFilter != null) {
+ cellFilter.reset();
+ }
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRowKey(buffer, offset, length);
+ }
+ return super.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterAllRemaining();
+ }
+ return super.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ if (cellFilter != null) {
+ cellFilter.filterRowCells(kvs);
+ } else {
+ super.filterRowCells(kvs);
+ }
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ if (cellFilter != null) {
+ return cellFilter.hasFilterRow();
+ }
+ return super.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextKeyHint(currentKV);
+ }
+ return super.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextCellHint(currentKV);
+ }
+ return super.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.isFamilyEssential(name);
+ }
+ return super.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return super.toByteArray();
+ }
+
+ private boolean isColumnDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
+ }
+
+ private static final class DeleteTracker {
+ private long familyDeleteTs;
+
+ public static boolean isFamilyDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
+ CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
+ CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
+ }
+
+ public void addFamilyDelete(Cell delete) {
+ this.familyDeleteTs = delete.getTimestamp();
+ }
+
+ public boolean isDeleted(Cell cell) {
+ return cell.getTimestamp() <= familyDeleteTs;
+ }
+
+ public void reset() {
+ this.familyDeleteTs = 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java
deleted file mode 100644
index 716ea38..0000000
--- a/tephra-hbase-compat-1.0-cdh/src/test/java/co/cask/tephra/hbase10cdh/HBase10ConfigurationProviderTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10cdh;
-
-import co.cask.tephra.util.AbstractConfigurationProviderTest;
-import co.cask.tephra.util.HBaseVersion;
-
-/**
- * Test for HBase 1.0 (CDH) version specific behavior.
- */
-public class HBase10ConfigurationProviderTest extends AbstractConfigurationProviderTest {
- @Override
- protected HBaseVersion.Version getExpectedVersion() {
- return HBaseVersion.Version.HBASE_10_CDH;
- }
-}
[17/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java
new file mode 100644
index 0000000..e8e045a
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98.coprocessor;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCodec;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+
+/**
+ * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
+ * for transactions:
+ * <ul>
+ * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
+ * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
+ * or expired due to TTL.</li>
+ * </ul>
+ *
+ * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
+ * or on all user tables by adding the following to hbase-site.xml:
+ * {@code
+ * <property>
+ * <name>hbase.coprocessor.region.classes</name>
+ * <value>TransactionProcessor</value>
+ * </property>
+ * }
+ * </p>
+ *
+ * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
+ * as an attribute:
+ * {@code
+ * Transaction t = ...;
+ * Get get = new Get(...);
+ * TransactionCodec codec = new TransactionCodec();
+ * codec.addToOperation(get, t);
+ * }
+ * </p>
+ */
+public class TransactionProcessor extends BaseRegionObserver {
+ private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
+
+ private TransactionStateCache cache;
+ private final TransactionCodec txCodec;
+ protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
+ protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
+
+ public TransactionProcessor() {
+ this.txCodec = new TransactionCodec();
+ }
+
+ /* RegionObserver implementation */
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ if (e instanceof RegionCoprocessorEnvironment) {
+ RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
+ Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
+ this.cache = cacheSupplier.get();
+
+ HTableDescriptor tableDesc = env.getRegion().getTableDesc();
+ for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
+ String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
+ long ttl = 0;
+ if (columnTTL != null) {
+ try {
+ ttl = Long.parseLong(columnTTL);
+ LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
+ ", value = " + columnTTL);
+ }
+ }
+ ttlByFamily.put(columnDesc.getName(), ttl);
+ }
+
+ this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
+ TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
+ this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
+ if (readNonTxnData) {
+ LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
+ }
+ }
+ }
+
+ protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
+ return new TransactionStateCacheSupplier(env.getConfiguration());
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ // nothing to do
+ }
+
+ @Override
+ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
+ throws IOException {
+ Transaction tx = getFromOperation(get);
+ if (tx != null) {
+ projectFamilyDeletes(get);
+ get.setMaxVersions();
+ get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
+ get.setFilter(newFilter);
+ }
+ }
+
+ @Override
+ public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
+ Durability durability) throws IOException {
+ // Translate deletes into our own delete tombstones
+ // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
+ // us to rollback the changes (by a real delete) if the transaction fails
+
+ // Deletes that are part of a transaction rollback do not need special handling.
+ // They will never be rolled back, so are performed as normal HBase deletes.
+ if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ return;
+ }
+
+ // Other deletes are client-initiated and need to be translated into our own tombstones
+ // TODO: this should delegate to the DeleteStrategy implementation.
+ Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
+ for (byte[] family : delete.getFamilyCellMap().keySet()) {
+ List<Cell> familyCells = delete.getFamilyCellMap().get(family);
+ if (isFamilyDelete(familyCells)) {
+ deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ } else {
+ int cellSize = familyCells.size();
+ for (int i = 0; i < cellSize; i++) {
+ Cell cell = familyCells.get(i);
+ deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
+ }
+ e.getEnvironment().getRegion().put(deleteMarkers);
+ // skip normal delete handling
+ e.bypass();
+ }
+
+ private boolean isFamilyDelete(List<Cell> familyCells) {
+ return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
+ }
+
+ @Override
+ public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
+ throws IOException {
+ Transaction tx = getFromOperation(scan);
+ if (tx != null) {
+ projectFamilyDeletes(scan);
+ scan.setMaxVersions();
+ scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
+ scan.setFilter(newFilter);
+ }
+ return s;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any scan operation.
+ * @param scan The original scan request
+ * @return The modified scan request with the family delete qualifiers represented
+ */
+ private Scan projectFamilyDeletes(Scan scan) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return scan;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any get operation.
+ * @param get The original get request
+ * @return The modified get request with the family delete qualifiers represented
+ */
+ private Get projectFamilyDeletes(Get get) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return get;
+ }
+
+ @Override
+ public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ KeyValueScanner memstoreScanner, InternalScanner scanner)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
+ Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
+ HConstants.OLDEST_TIMESTAMP);
+ }
+
+ @Override
+ public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
+ CompactionRequest request)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners,
+ scanType, earliestPutTs);
+ }
+
+ protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
+ TransactionVisibilityState snapshot, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType type,
+ long earliestPutTs) throws IOException {
+ if (snapshot == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
+ ", no current transaction state found, defaulting to normal " + action + " scanner");
+ }
+ return null;
+ }
+
+ // construct a dummy transaction from the latest snapshot
+ Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
+ Scan scan = new Scan();
+ // need to see all versions, since we filter out excludes and applications may rely on multiple versions
+ scan.setMaxVersions();
+ scan.setFilter(
+ new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
+ snapshot.getInvalid(),
+ getTransactionFilter(dummyTx, type, null)));
+
+ return new StoreScanner(store, store.getScanInfo(), scan, scanners,
+ type, store.getSmallestReadPoint(), earliestPutTs);
+ }
+
+ private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
+ byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded != null) {
+ return txCodec.decode(encoded);
+ }
+ return null;
+ }
+
+ /**
+ * Derived classes can override this method to customize the filter used to return data visible for the current
+ * transaction.
+ *
+ * @param tx the current transaction to apply
+ * @param type the type of scan being performed
+ */
+ protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
+ return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+ }
+
+ /**
+ * Filter used to include cells visible to in-progress transactions on flush and commit.
+ */
+ static class IncludeInProgressFilter extends FilterBase {
+ private final long visibilityUpperBound;
+ private final Set<Long> invalidIds;
+ private final Filter txFilter;
+
+ public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
+ this.visibilityUpperBound = upperBound;
+ this.invalidIds = Sets.newHashSet(invalids);
+ this.txFilter = transactionFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ // include all cells visible to in-progress transactions, except for those already marked as invalid
+ long ts = cell.getTimestamp();
+ if (ts > visibilityUpperBound) {
+ // include everything that could still be in-progress except invalids
+ if (invalidIds.contains(ts)) {
+ return ReturnCode.SKIP;
+ }
+ return ReturnCode.INCLUDE;
+ }
+ return txFilter.filterKeyValue(cell);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java
new file mode 100644
index 0000000..d01f83c
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98.coprocessor;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Applies filtering of data based on transactional visibility (HBase 0.98+ specific version).
+ * Note: this is intended for server-side use only, as additional properties need to be set on
+ * any {@code Scan} or {@code Get} operation performed.
+ */
+public class TransactionVisibilityFilter extends FilterBase {
+ private final Transaction tx;
+ // oldest visible timestamp by column family, used to apply TTL when reading
+ private final Map<byte[], Long> oldestTsByFamily;
+ // if false, empty values will be interpreted as deletes
+ private final boolean allowEmptyValues;
+ // whether or not we can remove delete markers
+ // these can only be safely removed when we are traversing all storefiles
+ private final boolean clearDeletes;
+ // optional sub-filter to apply to visible cells
+ private final Filter cellFilter;
+
+ // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
+ private byte[] currentFamily = new byte[0];
+ private long currentOldestTs;
+
+ private DeleteTracker deleteTracker = new DeleteTracker();
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+ }
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ this.tx = tx;
+ this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
+ long familyTTL = ttlEntry.getValue();
+ oldestTsByFamily.put(ttlEntry.getKey(),
+ familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+ }
+ this.allowEmptyValues = allowEmptyValues;
+ this.clearDeletes =
+ scanType == ScanType.COMPACT_DROP_DELETES ||
+ (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ this.cellFilter = cellFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (!CellUtil.matchingFamily(cell, currentFamily)) {
+ // column family changed
+ currentFamily = CellUtil.cloneFamily(cell);
+ Long familyOldestTs = oldestTsByFamily.get(currentFamily);
+ currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
+ deleteTracker.reset();
+ }
+ // need to apply TTL for the column family here
+ long kvTimestamp = cell.getTimestamp();
+ if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
+ // passed TTL for this column, seek to next
+ return ReturnCode.NEXT_COL;
+ } else if (tx.isVisible(kvTimestamp)) {
+ // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
+ // cell is visible
+ // visibility SNAPSHOT_ALL needs all matches
+ return runSubFilter(ReturnCode.INCLUDE, cell);
+ }
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ deleteTracker.addFamilyDelete(cell);
+ if (clearDeletes) {
+ return ReturnCode.NEXT_COL;
+ } else {
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // check if masked by family delete
+ if (deleteTracker.isDeleted(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+ // check for column delete
+ if (isColumnDelete(cell)) {
+ if (clearDeletes) {
+ // skip "deleted" cell
+ return ReturnCode.NEXT_COL;
+ } else {
+ // keep the marker but skip any remaining versions
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ } else {
+ return ReturnCode.SKIP;
+ }
+ }
+
+ private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
+ if (cellFilter != null) {
+ ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
+ return determineReturnCode(txFilterCode, subFilterCode);
+ }
+ return txFilterCode;
+ }
+
+ /**
+ * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
+ * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
+ * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
+ * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
+ *
+ * @param txFilterCode return code from TransactionVisibilityFilter
+ * @param subFilterCode return code from sub-filter
+ * @return final return code
+ */
+ protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
+ // Return the more restrictive of the two filter responses
+ switch (subFilterCode) {
+ case INCLUDE:
+ return txFilterCode;
+ case INCLUDE_AND_NEXT_COL:
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ case SKIP:
+ return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
+ default:
+ return subFilterCode;
+ }
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRow();
+ }
+ return super.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ // Convert Tephra deletes back into HBase deletes
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
+ KeyValue.Type.DeleteFamily);
+ } else if (isColumnDelete(cell)) {
+ // Note: in some cases KeyValue.Type.Delete is used in Delete object,
+ // and in some other cases KeyValue.Type.DeleteColumn is used.
+ // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
+ // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
+ // work in both cases.
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
+ cell.getTimestamp(), KeyValue.Type.DeleteColumn);
+ }
+ }
+ return cell;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ deleteTracker.reset();
+ if (cellFilter != null) {
+ cellFilter.reset();
+ }
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRowKey(buffer, offset, length);
+ }
+ return super.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterAllRemaining();
+ }
+ return super.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ if (cellFilter != null) {
+ cellFilter.filterRowCells(kvs);
+ } else {
+ super.filterRowCells(kvs);
+ }
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ if (cellFilter != null) {
+ return cellFilter.hasFilterRow();
+ }
+ return super.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextKeyHint(currentKV);
+ }
+ return super.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextCellHint(currentKV);
+ }
+ return super.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.isFamilyEssential(name);
+ }
+ return super.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return super.toByteArray();
+ }
+
+ private boolean isColumnDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
+ }
+
+ private static final class DeleteTracker {
+ private long familyDeleteTs;
+
+ public static boolean isFamilyDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
+ CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
+ CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
+ }
+
+ public void addFamilyDelete(Cell delete) {
+ this.familyDeleteTs = delete.getTimestamp();
+ }
+
+ public boolean isDeleted(Cell cell) {
+ return cell.getTimestamp() <= familyDeleteTs;
+ }
+
+ public void reset() {
+ this.familyDeleteTs = 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/HBase98ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/HBase98ConfigurationProviderTest.java b/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/HBase98ConfigurationProviderTest.java
deleted file mode 100644
index d297c14..0000000
--- a/tephra-hbase-compat-0.98/src/test/java/co/cask/tephra/hbase98/HBase98ConfigurationProviderTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98;
-
-import co.cask.tephra.util.AbstractConfigurationProviderTest;
-import co.cask.tephra.util.HBaseVersion;
-
-/**
- * Test for HBase 0.98 version specific behavior.
- */
-public class HBase98ConfigurationProviderTest extends AbstractConfigurationProviderTest {
- @Override
- protected HBaseVersion.Version getExpectedVersion() {
- return HBaseVersion.Version.HBASE_98;
- }
-}
[21/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessorTest.java
deleted file mode 100644
index 3769fa5..0000000
--- a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionProcessorTest.java
+++ /dev/null
@@ -1,771 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96.coprocessor;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.HDFSTransactionStateStorage;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.snapshot.DefaultSnapshotCodec;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.executor.ExecutorService;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.ipc.RpcServerInterface;
-import org.apache.hadoop.hbase.master.TableLockManager;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
-import org.apache.hadoop.hbase.regionserver.FlushRequester;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.Leases;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.zookeeper.KeeperException;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
- */
-public class TransactionProcessorTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
-
- // 8 versions, 1 hour apart, latest is current ts.
- private static final long[] V;
-
- static {
- long now = System.currentTimeMillis();
- V = new long[9];
- for (int i = 0; i < V.length; i++) {
- V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
- }
- }
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
- private static MiniDFSCluster dfsCluster;
- private static Configuration conf;
- private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
- private static TransactionVisibilityState txVisibilityState;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- Configuration hConf = new Configuration();
- String rootDir = tmpFolder.newFolder().getAbsolutePath();
- hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
- hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
-
- dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
- dfsCluster.waitActive();
- conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
-
- conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
- conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- String localTestDir = tmpFolder.newFolder().getAbsolutePath();
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
-
- // write an initial transaction snapshot
- TransactionSnapshot txSnapshot =
- TransactionSnapshot.copyFrom(
- System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
- // this will set visibility upper bound to V[6]
- Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
- TransactionType.SHORT))),
- new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
- txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
- txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
- txSnapshot.getInProgress());
- HDFSTransactionStateStorage tmpStorage =
- new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
- tmpStorage.startAndWait();
- tmpStorage.writeSnapshot(txSnapshot);
- tmpStorage.stopAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- dfsCluster.shutdown();
- }
-
- @Test
- public void testDataJanitorRegionScanner() throws Exception {
- String tableName = "TestRegionScanner";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- for (int i = 1; i <= 8; i++) {
- for (int k = 1; k <= i; k++) {
- Put p = new Put(Bytes.toBytes(i));
- p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
- region.put(p);
- }
- }
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should only return the valid rows - testing that cleanup works on flush
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
-
- // first returned value should be "4" with version "4"
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 4, new long[] {V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 5, new long[] {V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});
-
- results.clear();
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteFiltering() throws Exception {
- String tableName = "TestDeleteFiltering";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- byte[] row = Bytes.toBytes(1);
- for (int i = 4; i < V.length; i++) {
- Put p = new Put(row);
- p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
- region.put(p);
- }
-
- // delete from the third entry back
- // take that cell's timestamp + 1 to simulate a delete in a new tx
- long deleteTs = V[5] + 1;
- Delete d = new Delete(row, deleteTs);
- LOG.info("Issuing delete at timestamp " + deleteTs);
- // row deletes are not yet supported (TransactionAwareHTable normally handles this)
- d.deleteColumns(familyBytes, columnBytes, deleteTs);
- region.delete(d);
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, we should drop the deleted version, but not the others
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should return row with versions at: V[8], V[6].
- // V[7] is invalid and V[5] and prior are deleted.
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
- // should be only one row
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 1,
- new long[]{V[8], V[6], deleteTs},
- new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteMarkerCleanup() throws Exception {
- String tableName = "TestDeleteMarkerCleanup";
- byte[] familyBytes = Bytes.toBytes("f");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
-
- // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
- long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
- // deletes are performed after the writes, but still before the visibility upper bound
- long deleteTs = writeTs + 1;
- // write separate columns to confirm that delete markers survive across flushes
- byte[] row = Bytes.toBytes(100);
- Put p = new Put(row);
-
- LOG.info("Writing columns at timestamp " + writeTs);
- for (int i = 0; i < 5; i++) {
- byte[] iBytes = Bytes.toBytes(i);
- p.add(familyBytes, iBytes, writeTs, iBytes);
- }
- region.put(p);
- // read all back
- Scan scan = new Scan(row);
- RegionScanner regionScanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- for (int i = 0; i < 5; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a flush to clear the memstore
- LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // delete the odd entries
- for (int i = 0; i < 5; i++) {
- if (i % 2 == 1) {
- // deletes are performed as puts with empty values
- Put deletePut = new Put(row);
- deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
- region.put(deletePut);
- }
- }
-
- // read all back
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- LOG.info("Got cell " + cell);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force another flush on the delete markers
- // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
- LOG.info("After delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
-
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a major compaction
- LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
- region.compactStores(true);
-
- // perform a raw scan (no filter) to confirm that the delete markers are now gone
- scan = new Scan(row);
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
- } finally {
- region.close();
- }
- }
-
- /**
- * Test that we correctly preserve the timestamp set for column family delete markers. This is not
- * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
- * that we make it easy to interoperate with other systems.
- */
- @Test
- public void testFamilyDeleteTimestamp() throws Exception {
- String tableName = "TestFamilyDeleteTimestamp";
- byte[] family1Bytes = Bytes.toBytes("f1");
- byte[] columnBytes = Bytes.toBytes("c");
- byte[] rowBytes = Bytes.toBytes("row");
- byte[] valBytes = Bytes.toBytes("val");
- HRegion region = createRegion(tableName, family1Bytes, 0);
- try {
- region.initialize();
-
- long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
- Put p = new Put(rowBytes);
- p.add(family1Bytes, columnBytes, now - 10, valBytes);
- region.put(p);
-
- // issue a family delete with an explicit timestamp
- Delete delete = new Delete(rowBytes, now);
- delete.deleteFamily(family1Bytes, now - 5);
- region.delete(delete);
-
- // test that the delete marker preserved the timestamp
- Scan scan = new Scan();
- scan.setMaxVersions();
- RegionScanner scanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(2, results.size());
- // delete marker should appear first
- Cell cell = results.get(0);
- assertArrayEquals(new byte[0], cell.getQualifier());
- assertArrayEquals(new byte[0], cell.getValue());
- assertEquals(now - 5, cell.getTimestamp());
- // since this is an unfiltered scan against the region, the original put should be next
- cell = results.get(1);
- assertArrayEquals(valBytes, cell.getValue());
- assertEquals(now - 10, cell.getTimestamp());
- scanner.close();
-
-
- // with a filtered scan the original put should disappear
- scan = new Scan();
- scan.setMaxVersions();
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
-
- scanner = region.getScanner(scan);
- results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(0, results.size());
- scanner.close();
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testPreExistingData() throws Exception {
- String tableName = "TestPreExistingData";
- byte[] familyBytes = Bytes.toBytes("f");
- long ttlMillis = TimeUnit.DAYS.toMillis(14);
- HRegion region = createRegion(tableName, familyBytes, ttlMillis);
- try {
- region.initialize();
-
- // timestamps for pre-existing, non-transactional data
- long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
- long older = now - ttlMillis / 2;
- long newer = now - ttlMillis / 3;
- // timestamps for transactional data
- long nowTx = txVisibilityState.getVisibilityUpperBound();
- long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
- long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(familyBytes, ttlMillis);
-
- List<Cell> cells = new ArrayList<>();
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
-
- // Write non-transactional and transactional data
- for (Cell c : cells) {
- region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
- }
-
- Scan rawScan = new Scan();
- rawScan.setMaxVersions();
-
- Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
- Scan txScan = new Scan();
- txScan.setMaxVersions();
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // read all back with raw scanner
- scanAndAssert(region, cells, rawScan);
-
- // read all back with transaction filter
- scanAndAssert(region, cells, txScan);
-
- // force a flush to clear the memstore
- region.flushcache();
- scanAndAssert(region, cells, txScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- scanAndAssert(region, cells, txScan);
-
- // Reduce TTL, this should make cells with timestamps older and olderTx expire
- long newTtl = ttlMillis / 2 - 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // Raw scan should still give all cells
- scanAndAssert(region, cells, rawScan);
- // However, tx scan should not return expired cells
- scanAndAssert(region, select(cells, 1, 3, 5), txScan);
-
- region.flushcache();
- scanAndAssert(region, cells, rawScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- // This time raw scan too should not return expired cells, as they would be dropped during major compaction
- scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
-
- // Reduce TTL again to 1 ms, this should expire all cells
- newTtl = 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // force a major compaction to remove expired cells
- region.compactStores(true);
- // This time raw scan should not return any cells, as all cells have expired.
- scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
- } finally {
- region.close();
- }
- }
-
- private List<Cell> select(List<Cell> cells, int... indexes) {
- List<Cell> newCells = new ArrayList<>();
- for (int i : indexes) {
- newCells.add(cells.get(i));
- }
- return newCells;
- }
-
- @SuppressWarnings("StatementWithEmptyBody")
- private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
- try (RegionScanner regionScanner = region.getScanner(scan)) {
- List<Cell> results = Lists.newArrayList();
- while (regionScanner.next(results)) { }
- assertEquals(expected, results);
- }
- }
-
- private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
- region.close();
- HTableDescriptor htd = region.getTableDesc();
- HColumnDescriptor cfd = new HColumnDescriptor(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- htd.addFamily(cfd);
- return HRegion.openHRegion(region.getRegionInfo(), htd, region.getLog(), conf,
- new MockRegionServerServices(conf, null), null);
- }
-
- private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
- HColumnDescriptor cfd = new HColumnDescriptor(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- htd.addFamily(cfd);
- htd.addCoprocessor(TransactionProcessor.class.getName());
- Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
- Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
- FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.mkdirs(tablePath));
- HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
- HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
- HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
- return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
- byte[][] values = new byte[versions.length][];
- for (int i = 0; i < versions.length; i++) {
- values[i] = Bytes.toBytes(versions[i]);
- }
- assertKeyValueMatches(results, index, versions, values);
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
- assertEquals(versions.length, results.size());
- assertEquals(values.length, results.size());
- for (int i = 0; i < versions.length; i++) {
- Cell kv = results.get(i);
- assertArrayEquals(Bytes.toBytes(index), kv.getRow());
- assertEquals(versions[i], kv.getTimestamp());
- assertArrayEquals(values[i], kv.getValue());
- }
- }
-
- @Test
- public void testTransactionStateCache() throws Exception {
- TransactionStateCache cache = new TransactionStateCache();
- cache.setConf(conf);
- cache.startAndWait();
- // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
- TransactionVisibilityState cachedSnapshot = cache.getLatestState();
- assertNotNull(cachedSnapshot);
- assertEquals(invalidSet, cachedSnapshot.getInvalid());
- cache.stopAndWait();
- }
-
- private static class MockRegionServerServices implements RegionServerServices {
- private final Configuration hConf;
- private final ZooKeeperWatcher zookeeper;
- private final Map<String, HRegion> regions = new HashMap<String, HRegion>();
- private boolean stopping = false;
- private final ConcurrentSkipListMap<byte[], Boolean> rit = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
- private HFileSystem hfs = null;
- private ServerName serverName = null;
- private RpcServerInterface rpcServer = null;
- private volatile boolean abortRequested;
-
-
- public MockRegionServerServices(Configuration hConf, ZooKeeperWatcher zookeeper) {
- this.hConf = hConf;
- this.zookeeper = zookeeper;
- }
-
- @Override
- public boolean isStopping() {
- return stopping;
- }
-
- @Override
- public HLog getWAL(HRegionInfo regionInfo) throws IOException {
- return null;
- }
-
- @Override
- public CompactionRequestor getCompactionRequester() {
- return null;
- }
-
- @Override
- public FlushRequester getFlushRequester() {
- return null;
- }
-
- @Override
- public RegionServerAccounting getRegionServerAccounting() {
- return null;
- }
-
- @Override
- public TableLockManager getTableLockManager() {
- return new TableLockManager.NullTableLockManager();
- }
-
- @Override
- public void postOpenDeployTasks(HRegion r, CatalogTracker ct) throws KeeperException, IOException {
- }
-
- @Override
- public RpcServerInterface getRpcServer() {
- return rpcServer;
- }
-
- @Override
- public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
- return rit;
- }
-
- @Override
- public FileSystem getFileSystem() {
- return hfs;
- }
-
- @Override
- public Leases getLeases() {
- return null;
- }
-
- @Override
- public ExecutorService getExecutorService() {
- return null;
- }
-
- @Override
- public CatalogTracker getCatalogTracker() {
- return null;
- }
-
- @Override
- public Map<String, HRegion> getRecoveringRegions() {
- return null;
- }
-
- @Override
- public void updateRegionFavoredNodesMapping(String encodedRegionName, List<HBaseProtos.ServerName> favoredNodes) {
- }
-
- @Override
- public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
- return new InetSocketAddress[0];
- }
-
- @Override
- public void addToOnlineRegions(HRegion r) {
- regions.put(r.getRegionNameAsString(), r);
- }
-
- @Override
- public boolean removeFromOnlineRegions(HRegion r, ServerName destination) {
- return regions.remove(r.getRegionInfo().getEncodedName()) != null;
- }
-
- @Override
- public HRegion getFromOnlineRegions(String encodedRegionName) {
- return regions.get(encodedRegionName);
- }
-
- @Override
- public List<HRegion> getOnlineRegions(TableName tableName) throws IOException {
- return null;
- }
-
- @Override
- public Configuration getConfiguration() {
- return hConf;
- }
-
- @Override
- public ZooKeeperWatcher getZooKeeper() {
- return zookeeper;
- }
-
- @Override
- public ServerName getServerName() {
- return serverName;
- }
-
- @Override
- public void abort(String why, Throwable e) {
- this.abortRequested = true;
- }
-
- @Override
- public boolean isAborted() {
- return abortRequested;
- }
-
- @Override
- public void stop(String why) {
- this.stopping = true;
- }
-
- @Override
- public boolean isStopped() {
- return stopping;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java
deleted file mode 100644
index 120be8b..0000000
--- a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/coprocessor/TransactionVisibilityFilterTest.java
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase.AbstractTransactionVisibilityFilterTest;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * HBase 0.96 specific test for filtering logic applied when reading data transactionally.
- */
-public class TransactionVisibilityFilterTest extends AbstractTransactionVisibilityFilterTest {
- /**
- * Test filtering of KeyValues for in-progress and invalid transactions.
- * @throws Exception
- */
- @Test
- public void testFiltering() throws Exception {
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
- }
-
- @Test
- public void testSubFilter() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- @Test
- public void testSubFilterOverride() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- private void runFilteringTest(TxFilterFactory txFilterFactory,
- List<Filter.ReturnCode> assertCodes) throws Exception {
-
- /*
- * Start and stop some transactions. This will give us a transaction state something like the following
- * (numbers only reflect ordering, not actual transaction IDs):
- * 6 - in progress
- * 5 - committed
- * 4 - invalid
- * 3 - in-progress
- * 2 - committed
- * 1 - committed
- *
- * read ptr = 5
- * write ptr = 6
- */
-
- Transaction tx1 = txManager.startShort();
- assertTrue(txManager.canCommit(tx1, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx1));
-
- Transaction tx2 = txManager.startShort();
- assertTrue(txManager.canCommit(tx2, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx2));
-
- Transaction tx3 = txManager.startShort();
- Transaction tx4 = txManager.startShort();
- txManager.invalidate(tx4.getTransactionId());
-
- Transaction tx5 = txManager.startShort();
- assertTrue(txManager.canCommit(tx5, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx5));
-
- Transaction tx6 = txManager.startShort();
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- Filter filter = txFilterFactory.getTxFilter(tx6, ttls);
-
- assertEquals(assertCodes.get(5),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx6.getTransactionId())));
- assertEquals(assertCodes.get(4),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx5.getTransactionId())));
- assertEquals(assertCodes.get(3),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx4.getTransactionId())));
- assertEquals(assertCodes.get(2),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx3.getTransactionId())));
- assertEquals(assertCodes.get(1),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx2.getTransactionId())));
- assertEquals(assertCodes.get(0),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx1.getTransactionId())));
- }
-
- /**
- * Test filtering for TTL settings.
- * @throws Exception
- */
- @Test
- public void testTTLFiltering() throws Exception {
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(FAM, 10L);
- ttls.put(FAM2, 30L);
- ttls.put(FAM3, 0L);
-
- Transaction tx = txManager.startShort();
- long now = tx.getVisibilityUpperBound();
- Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 21 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 1001 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
-
- // Verify ttl for pre-existing, non-transactional data
- long preNow = now / TxConstants.MAX_TX_PER_MS;
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 11L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 1001L)));
- }
-
- protected KeyValue newKeyValue(String rowkey, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), FAM, COL, timestamp, Bytes.toBytes(value));
- }
-
- protected KeyValue newKeyValue(String rowkey, byte[] family, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), family, COL, timestamp, Bytes.toBytes(value));
- }
-
- private interface TxFilterFactory {
- Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs);
- }
-
- private class CustomTxFilter extends TransactionVisibilityFilter {
- public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
- @Nullable Filter cellFilter) {
- super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
- }
-
- @Override
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- switch (subFilterCode) {
- case INCLUDE:
- return ReturnCode.INCLUDE;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/HBase96ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/HBase96ConfigurationProviderTest.java b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/HBase96ConfigurationProviderTest.java
new file mode 100644
index 0000000..1a9d68b
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/HBase96ConfigurationProviderTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96;
+
+import org.apache.tephra.util.AbstractConfigurationProviderTest;
+import org.apache.tephra.util.HBaseVersion;
+
+/**
+ * Test for HBase 0.96 version specific behavior.
+ */
+public class HBase96ConfigurationProviderTest extends AbstractConfigurationProviderTest {
+ @Override
+ protected HBaseVersion.Version getExpectedVersion() {
+ return HBaseVersion.Version.HBASE_96;
+ }
+}
[45/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionServer.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionServer.java b/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionServer.java
deleted file mode 100644
index 862b7d4..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/distributed/thrift/TTransactionServer.java
+++ /dev/null
@@ -1,11072 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package co.cask.tephra.distributed.thrift;
-
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-import org.apache.thrift.scheme.TupleScheme;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.ByteBuffer;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public class TTransactionServer {
-
- public interface Iface {
-
- public TTransaction startLong() throws org.apache.thrift.TException;
-
- public TTransaction startShort() throws org.apache.thrift.TException;
-
- public TTransaction startShortTimeout(int timeout) throws org.apache.thrift.TException;
-
- public TBoolean canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws TTransactionNotInProgressException, org.apache.thrift.TException;
-
- public TBoolean commitTx(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException;
-
- public void abortTx(TTransaction tx) throws org.apache.thrift.TException;
-
- public boolean invalidateTx(long tx) throws org.apache.thrift.TException;
-
- public ByteBuffer getSnapshot() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException;
-
- public void resetState() throws org.apache.thrift.TException;
-
- public String status() throws org.apache.thrift.TException;
-
- public TBoolean truncateInvalidTx(Set<Long> txns) throws org.apache.thrift.TException;
-
- public TBoolean truncateInvalidTxBefore(long time) throws TInvalidTruncateTimeException, org.apache.thrift.TException;
-
- public int invalidTxSize() throws org.apache.thrift.TException;
-
- public TTransaction checkpoint(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException;
-
- }
-
- public interface AsyncIface {
-
- public void startLong(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.startLong_call> resultHandler) throws org.apache.thrift.TException;
-
- public void startShort(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.startShort_call> resultHandler) throws org.apache.thrift.TException;
-
- public void startShortTimeout(int timeout, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.startShortTimeout_call> resultHandler) throws org.apache.thrift.TException;
-
- public void canCommitTx(TTransaction tx, Set<ByteBuffer> changes, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.canCommitTx_call> resultHandler) throws org.apache.thrift.TException;
-
- public void commitTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.commitTx_call> resultHandler) throws org.apache.thrift.TException;
-
- public void abortTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.abortTx_call> resultHandler) throws org.apache.thrift.TException;
-
- public void invalidateTx(long tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.invalidateTx_call> resultHandler) throws org.apache.thrift.TException;
-
- public void getSnapshot(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.getSnapshot_call> resultHandler) throws org.apache.thrift.TException;
-
- public void resetState(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.resetState_call> resultHandler) throws org.apache.thrift.TException;
-
- public void status(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.status_call> resultHandler) throws org.apache.thrift.TException;
-
- public void truncateInvalidTx(Set<Long> txns, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.truncateInvalidTx_call> resultHandler) throws org.apache.thrift.TException;
-
- public void truncateInvalidTxBefore(long time, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.truncateInvalidTxBefore_call> resultHandler) throws org.apache.thrift.TException;
-
- public void invalidTxSize(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.invalidTxSize_call> resultHandler) throws org.apache.thrift.TException;
-
- public void checkpoint(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.checkpoint_call> resultHandler) throws org.apache.thrift.TException;
-
- }
-
- public static class Client extends org.apache.thrift.TServiceClient implements Iface {
- public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
- public Factory() {}
- public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
- return new Client(prot);
- }
- public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- return new Client(iprot, oprot);
- }
- }
-
- public Client(org.apache.thrift.protocol.TProtocol prot)
- {
- super(prot, prot);
- }
-
- public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- super(iprot, oprot);
- }
-
- public TTransaction startLong() throws org.apache.thrift.TException
- {
- send_startLong();
- return recv_startLong();
- }
-
- public void send_startLong() throws org.apache.thrift.TException
- {
- startLong_args args = new startLong_args();
- sendBase("startLong", args);
- }
-
- public TTransaction recv_startLong() throws org.apache.thrift.TException
- {
- startLong_result result = new startLong_result();
- receiveBase(result, "startLong");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "startLong failed: unknown result");
- }
-
- public TTransaction startShort() throws org.apache.thrift.TException
- {
- send_startShort();
- return recv_startShort();
- }
-
- public void send_startShort() throws org.apache.thrift.TException
- {
- startShort_args args = new startShort_args();
- sendBase("startShort", args);
- }
-
- public TTransaction recv_startShort() throws org.apache.thrift.TException
- {
- startShort_result result = new startShort_result();
- receiveBase(result, "startShort");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "startShort failed: unknown result");
- }
-
- public TTransaction startShortTimeout(int timeout) throws org.apache.thrift.TException
- {
- send_startShortTimeout(timeout);
- return recv_startShortTimeout();
- }
-
- public void send_startShortTimeout(int timeout) throws org.apache.thrift.TException
- {
- startShortTimeout_args args = new startShortTimeout_args();
- args.setTimeout(timeout);
- sendBase("startShortTimeout", args);
- }
-
- public TTransaction recv_startShortTimeout() throws org.apache.thrift.TException
- {
- startShortTimeout_result result = new startShortTimeout_result();
- receiveBase(result, "startShortTimeout");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "startShortTimeout failed: unknown result");
- }
-
- public TBoolean canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws TTransactionNotInProgressException, org.apache.thrift.TException
- {
- send_canCommitTx(tx, changes);
- return recv_canCommitTx();
- }
-
- public void send_canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws org.apache.thrift.TException
- {
- canCommitTx_args args = new canCommitTx_args();
- args.setTx(tx);
- args.setChanges(changes);
- sendBase("canCommitTx", args);
- }
-
- public TBoolean recv_canCommitTx() throws TTransactionNotInProgressException, org.apache.thrift.TException
- {
- canCommitTx_result result = new canCommitTx_result();
- receiveBase(result, "canCommitTx");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.e != null) {
- throw result.e;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "canCommitTx failed: unknown result");
- }
-
- public TBoolean commitTx(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException
- {
- send_commitTx(tx);
- return recv_commitTx();
- }
-
- public void send_commitTx(TTransaction tx) throws org.apache.thrift.TException
- {
- commitTx_args args = new commitTx_args();
- args.setTx(tx);
- sendBase("commitTx", args);
- }
-
- public TBoolean recv_commitTx() throws TTransactionNotInProgressException, org.apache.thrift.TException
- {
- commitTx_result result = new commitTx_result();
- receiveBase(result, "commitTx");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.e != null) {
- throw result.e;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "commitTx failed: unknown result");
- }
-
- public void abortTx(TTransaction tx) throws org.apache.thrift.TException
- {
- send_abortTx(tx);
- recv_abortTx();
- }
-
- public void send_abortTx(TTransaction tx) throws org.apache.thrift.TException
- {
- abortTx_args args = new abortTx_args();
- args.setTx(tx);
- sendBase("abortTx", args);
- }
-
- public void recv_abortTx() throws org.apache.thrift.TException
- {
- abortTx_result result = new abortTx_result();
- receiveBase(result, "abortTx");
- return;
- }
-
- public boolean invalidateTx(long tx) throws org.apache.thrift.TException
- {
- send_invalidateTx(tx);
- return recv_invalidateTx();
- }
-
- public void send_invalidateTx(long tx) throws org.apache.thrift.TException
- {
- invalidateTx_args args = new invalidateTx_args();
- args.setTx(tx);
- sendBase("invalidateTx", args);
- }
-
- public boolean recv_invalidateTx() throws org.apache.thrift.TException
- {
- invalidateTx_result result = new invalidateTx_result();
- receiveBase(result, "invalidateTx");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "invalidateTx failed: unknown result");
- }
-
- public ByteBuffer getSnapshot() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException
- {
- send_getSnapshot();
- return recv_getSnapshot();
- }
-
- public void send_getSnapshot() throws org.apache.thrift.TException
- {
- getSnapshot_args args = new getSnapshot_args();
- sendBase("getSnapshot", args);
- }
-
- public ByteBuffer recv_getSnapshot() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException
- {
- getSnapshot_result result = new getSnapshot_result();
- receiveBase(result, "getSnapshot");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.e != null) {
- throw result.e;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getSnapshot failed: unknown result");
- }
-
- public void resetState() throws org.apache.thrift.TException
- {
- send_resetState();
- recv_resetState();
- }
-
- public void send_resetState() throws org.apache.thrift.TException
- {
- resetState_args args = new resetState_args();
- sendBase("resetState", args);
- }
-
- public void recv_resetState() throws org.apache.thrift.TException
- {
- resetState_result result = new resetState_result();
- receiveBase(result, "resetState");
- return;
- }
-
- public String status() throws org.apache.thrift.TException
- {
- send_status();
- return recv_status();
- }
-
- public void send_status() throws org.apache.thrift.TException
- {
- status_args args = new status_args();
- sendBase("status", args);
- }
-
- public String recv_status() throws org.apache.thrift.TException
- {
- status_result result = new status_result();
- receiveBase(result, "status");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "status failed: unknown result");
- }
-
- public TBoolean truncateInvalidTx(Set<Long> txns) throws org.apache.thrift.TException
- {
- send_truncateInvalidTx(txns);
- return recv_truncateInvalidTx();
- }
-
- public void send_truncateInvalidTx(Set<Long> txns) throws org.apache.thrift.TException
- {
- truncateInvalidTx_args args = new truncateInvalidTx_args();
- args.setTxns(txns);
- sendBase("truncateInvalidTx", args);
- }
-
- public TBoolean recv_truncateInvalidTx() throws org.apache.thrift.TException
- {
- truncateInvalidTx_result result = new truncateInvalidTx_result();
- receiveBase(result, "truncateInvalidTx");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "truncateInvalidTx failed: unknown result");
- }
-
- public TBoolean truncateInvalidTxBefore(long time) throws TInvalidTruncateTimeException, org.apache.thrift.TException
- {
- send_truncateInvalidTxBefore(time);
- return recv_truncateInvalidTxBefore();
- }
-
- public void send_truncateInvalidTxBefore(long time) throws org.apache.thrift.TException
- {
- truncateInvalidTxBefore_args args = new truncateInvalidTxBefore_args();
- args.setTime(time);
- sendBase("truncateInvalidTxBefore", args);
- }
-
- public TBoolean recv_truncateInvalidTxBefore() throws TInvalidTruncateTimeException, org.apache.thrift.TException
- {
- truncateInvalidTxBefore_result result = new truncateInvalidTxBefore_result();
- receiveBase(result, "truncateInvalidTxBefore");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.e != null) {
- throw result.e;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "truncateInvalidTxBefore failed: unknown result");
- }
-
- public int invalidTxSize() throws org.apache.thrift.TException
- {
- send_invalidTxSize();
- return recv_invalidTxSize();
- }
-
- public void send_invalidTxSize() throws org.apache.thrift.TException
- {
- invalidTxSize_args args = new invalidTxSize_args();
- sendBase("invalidTxSize", args);
- }
-
- public int recv_invalidTxSize() throws org.apache.thrift.TException
- {
- invalidTxSize_result result = new invalidTxSize_result();
- receiveBase(result, "invalidTxSize");
- if (result.isSetSuccess()) {
- return result.success;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "invalidTxSize failed: unknown result");
- }
-
- public TTransaction checkpoint(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException
- {
- send_checkpoint(tx);
- return recv_checkpoint();
- }
-
- public void send_checkpoint(TTransaction tx) throws org.apache.thrift.TException
- {
- checkpoint_args args = new checkpoint_args();
- args.setTx(tx);
- sendBase("checkpoint", args);
- }
-
- public TTransaction recv_checkpoint() throws TTransactionNotInProgressException, org.apache.thrift.TException
- {
- checkpoint_result result = new checkpoint_result();
- receiveBase(result, "checkpoint");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.e != null) {
- throw result.e;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "checkpoint failed: unknown result");
- }
-
- }
- public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
- public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
- private org.apache.thrift.async.TAsyncClientManager clientManager;
- private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
- public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
- this.clientManager = clientManager;
- this.protocolFactory = protocolFactory;
- }
- public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
- return new AsyncClient(protocolFactory, clientManager, transport);
- }
- }
-
- public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
- super(protocolFactory, clientManager, transport);
- }
-
- public void startLong(org.apache.thrift.async.AsyncMethodCallback<startLong_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- startLong_call method_call = new startLong_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class startLong_call extends org.apache.thrift.async.TAsyncMethodCall {
- public startLong_call(org.apache.thrift.async.AsyncMethodCallback<startLong_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("startLong", org.apache.thrift.protocol.TMessageType.CALL, 0));
- startLong_args args = new startLong_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TTransaction getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_startLong();
- }
- }
-
- public void startShort(org.apache.thrift.async.AsyncMethodCallback<startShort_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- startShort_call method_call = new startShort_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class startShort_call extends org.apache.thrift.async.TAsyncMethodCall {
- public startShort_call(org.apache.thrift.async.AsyncMethodCallback<startShort_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("startShort", org.apache.thrift.protocol.TMessageType.CALL, 0));
- startShort_args args = new startShort_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TTransaction getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_startShort();
- }
- }
-
- public void startShortTimeout(int timeout, org.apache.thrift.async.AsyncMethodCallback<startShortTimeout_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- startShortTimeout_call method_call = new startShortTimeout_call(timeout, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class startShortTimeout_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int timeout;
- public startShortTimeout_call(int timeout, org.apache.thrift.async.AsyncMethodCallback<startShortTimeout_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.timeout = timeout;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("startShortTimeout", org.apache.thrift.protocol.TMessageType.CALL, 0));
- startShortTimeout_args args = new startShortTimeout_args();
- args.setTimeout(timeout);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TTransaction getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_startShortTimeout();
- }
- }
-
- public void canCommitTx(TTransaction tx, Set<ByteBuffer> changes, org.apache.thrift.async.AsyncMethodCallback<canCommitTx_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- canCommitTx_call method_call = new canCommitTx_call(tx, changes, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class canCommitTx_call extends org.apache.thrift.async.TAsyncMethodCall {
- private TTransaction tx;
- private Set<ByteBuffer> changes;
- public canCommitTx_call(TTransaction tx, Set<ByteBuffer> changes, org.apache.thrift.async.AsyncMethodCallback<canCommitTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.tx = tx;
- this.changes = changes;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("canCommitTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
- canCommitTx_args args = new canCommitTx_args();
- args.setTx(tx);
- args.setChanges(changes);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TBoolean getResult() throws TTransactionNotInProgressException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_canCommitTx();
- }
- }
-
- public void commitTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<commitTx_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- commitTx_call method_call = new commitTx_call(tx, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class commitTx_call extends org.apache.thrift.async.TAsyncMethodCall {
- private TTransaction tx;
- public commitTx_call(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<commitTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.tx = tx;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("commitTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
- commitTx_args args = new commitTx_args();
- args.setTx(tx);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TBoolean getResult() throws TTransactionNotInProgressException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_commitTx();
- }
- }
-
- public void abortTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<abortTx_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- abortTx_call method_call = new abortTx_call(tx, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class abortTx_call extends org.apache.thrift.async.TAsyncMethodCall {
- private TTransaction tx;
- public abortTx_call(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<abortTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.tx = tx;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("abortTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
- abortTx_args args = new abortTx_args();
- args.setTx(tx);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_abortTx();
- }
- }
-
- public void invalidateTx(long tx, org.apache.thrift.async.AsyncMethodCallback<invalidateTx_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- invalidateTx_call method_call = new invalidateTx_call(tx, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class invalidateTx_call extends org.apache.thrift.async.TAsyncMethodCall {
- private long tx;
- public invalidateTx_call(long tx, org.apache.thrift.async.AsyncMethodCallback<invalidateTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.tx = tx;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("invalidateTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
- invalidateTx_args args = new invalidateTx_args();
- args.setTx(tx);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public boolean getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_invalidateTx();
- }
- }
-
- public void getSnapshot(org.apache.thrift.async.AsyncMethodCallback<getSnapshot_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- getSnapshot_call method_call = new getSnapshot_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class getSnapshot_call extends org.apache.thrift.async.TAsyncMethodCall {
- public getSnapshot_call(org.apache.thrift.async.AsyncMethodCallback<getSnapshot_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getSnapshot", org.apache.thrift.protocol.TMessageType.CALL, 0));
- getSnapshot_args args = new getSnapshot_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public ByteBuffer getResult() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_getSnapshot();
- }
- }
-
- public void resetState(org.apache.thrift.async.AsyncMethodCallback<resetState_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- resetState_call method_call = new resetState_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class resetState_call extends org.apache.thrift.async.TAsyncMethodCall {
- public resetState_call(org.apache.thrift.async.AsyncMethodCallback<resetState_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("resetState", org.apache.thrift.protocol.TMessageType.CALL, 0));
- resetState_args args = new resetState_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_resetState();
- }
- }
-
- public void status(org.apache.thrift.async.AsyncMethodCallback<status_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- status_call method_call = new status_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class status_call extends org.apache.thrift.async.TAsyncMethodCall {
- public status_call(org.apache.thrift.async.AsyncMethodCallback<status_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("status", org.apache.thrift.protocol.TMessageType.CALL, 0));
- status_args args = new status_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_status();
- }
- }
-
- public void truncateInvalidTx(Set<Long> txns, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTx_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- truncateInvalidTx_call method_call = new truncateInvalidTx_call(txns, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class truncateInvalidTx_call extends org.apache.thrift.async.TAsyncMethodCall {
- private Set<Long> txns;
- public truncateInvalidTx_call(Set<Long> txns, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.txns = txns;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("truncateInvalidTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
- truncateInvalidTx_args args = new truncateInvalidTx_args();
- args.setTxns(txns);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TBoolean getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_truncateInvalidTx();
- }
- }
-
- public void truncateInvalidTxBefore(long time, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTxBefore_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- truncateInvalidTxBefore_call method_call = new truncateInvalidTxBefore_call(time, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class truncateInvalidTxBefore_call extends org.apache.thrift.async.TAsyncMethodCall {
- private long time;
- public truncateInvalidTxBefore_call(long time, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTxBefore_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.time = time;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("truncateInvalidTxBefore", org.apache.thrift.protocol.TMessageType.CALL, 0));
- truncateInvalidTxBefore_args args = new truncateInvalidTxBefore_args();
- args.setTime(time);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TBoolean getResult() throws TInvalidTruncateTimeException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_truncateInvalidTxBefore();
- }
- }
-
- public void invalidTxSize(org.apache.thrift.async.AsyncMethodCallback<invalidTxSize_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- invalidTxSize_call method_call = new invalidTxSize_call(resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class invalidTxSize_call extends org.apache.thrift.async.TAsyncMethodCall {
- public invalidTxSize_call(org.apache.thrift.async.AsyncMethodCallback<invalidTxSize_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("invalidTxSize", org.apache.thrift.protocol.TMessageType.CALL, 0));
- invalidTxSize_args args = new invalidTxSize_args();
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public int getResult() throws org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_invalidTxSize();
- }
- }
-
- public void checkpoint(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<checkpoint_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- checkpoint_call method_call = new checkpoint_call(tx, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class checkpoint_call extends org.apache.thrift.async.TAsyncMethodCall {
- private TTransaction tx;
- public checkpoint_call(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<checkpoint_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.tx = tx;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("checkpoint", org.apache.thrift.protocol.TMessageType.CALL, 0));
- checkpoint_args args = new checkpoint_args();
- args.setTx(tx);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public TTransaction getResult() throws TTransactionNotInProgressException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_checkpoint();
- }
- }
-
- }
-
- public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
- private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
- public Processor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
- }
-
- protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- processMap.put("startLong", new startLong());
- processMap.put("startShort", new startShort());
- processMap.put("startShortTimeout", new startShortTimeout());
- processMap.put("canCommitTx", new canCommitTx());
- processMap.put("commitTx", new commitTx());
- processMap.put("abortTx", new abortTx());
- processMap.put("invalidateTx", new invalidateTx());
- processMap.put("getSnapshot", new getSnapshot());
- processMap.put("resetState", new resetState());
- processMap.put("status", new status());
- processMap.put("truncateInvalidTx", new truncateInvalidTx());
- processMap.put("truncateInvalidTxBefore", new truncateInvalidTxBefore());
- processMap.put("invalidTxSize", new invalidTxSize());
- processMap.put("checkpoint", new checkpoint());
- return processMap;
- }
-
- public static class startLong<I extends Iface> extends org.apache.thrift.ProcessFunction<I, startLong_args> {
- public startLong() {
- super("startLong");
- }
-
- public startLong_args getEmptyArgsInstance() {
- return new startLong_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public startLong_result getResult(I iface, startLong_args args) throws org.apache.thrift.TException {
- startLong_result result = new startLong_result();
- result.success = iface.startLong();
- return result;
- }
- }
-
- public static class startShort<I extends Iface> extends org.apache.thrift.ProcessFunction<I, startShort_args> {
- public startShort() {
- super("startShort");
- }
-
- public startShort_args getEmptyArgsInstance() {
- return new startShort_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public startShort_result getResult(I iface, startShort_args args) throws org.apache.thrift.TException {
- startShort_result result = new startShort_result();
- result.success = iface.startShort();
- return result;
- }
- }
-
- public static class startShortTimeout<I extends Iface> extends org.apache.thrift.ProcessFunction<I, startShortTimeout_args> {
- public startShortTimeout() {
- super("startShortTimeout");
- }
-
- public startShortTimeout_args getEmptyArgsInstance() {
- return new startShortTimeout_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public startShortTimeout_result getResult(I iface, startShortTimeout_args args) throws org.apache.thrift.TException {
- startShortTimeout_result result = new startShortTimeout_result();
- result.success = iface.startShortTimeout(args.timeout);
- return result;
- }
- }
-
- public static class canCommitTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, canCommitTx_args> {
- public canCommitTx() {
- super("canCommitTx");
- }
-
- public canCommitTx_args getEmptyArgsInstance() {
- return new canCommitTx_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public canCommitTx_result getResult(I iface, canCommitTx_args args) throws org.apache.thrift.TException {
- canCommitTx_result result = new canCommitTx_result();
- try {
- result.success = iface.canCommitTx(args.tx, args.changes);
- } catch (TTransactionNotInProgressException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- public static class commitTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, commitTx_args> {
- public commitTx() {
- super("commitTx");
- }
-
- public commitTx_args getEmptyArgsInstance() {
- return new commitTx_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public commitTx_result getResult(I iface, commitTx_args args) throws org.apache.thrift.TException {
- commitTx_result result = new commitTx_result();
- try {
- result.success = iface.commitTx(args.tx);
- } catch (TTransactionNotInProgressException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- public static class abortTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, abortTx_args> {
- public abortTx() {
- super("abortTx");
- }
-
- public abortTx_args getEmptyArgsInstance() {
- return new abortTx_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public abortTx_result getResult(I iface, abortTx_args args) throws org.apache.thrift.TException {
- abortTx_result result = new abortTx_result();
- iface.abortTx(args.tx);
- return result;
- }
- }
-
- public static class invalidateTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, invalidateTx_args> {
- public invalidateTx() {
- super("invalidateTx");
- }
-
- public invalidateTx_args getEmptyArgsInstance() {
- return new invalidateTx_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public invalidateTx_result getResult(I iface, invalidateTx_args args) throws org.apache.thrift.TException {
- invalidateTx_result result = new invalidateTx_result();
- result.success = iface.invalidateTx(args.tx);
- result.setSuccessIsSet(true);
- return result;
- }
- }
-
- public static class getSnapshot<I extends Iface> extends org.apache.thrift.ProcessFunction<I, getSnapshot_args> {
- public getSnapshot() {
- super("getSnapshot");
- }
-
- public getSnapshot_args getEmptyArgsInstance() {
- return new getSnapshot_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public getSnapshot_result getResult(I iface, getSnapshot_args args) throws org.apache.thrift.TException {
- getSnapshot_result result = new getSnapshot_result();
- try {
- result.success = iface.getSnapshot();
- } catch (TTransactionCouldNotTakeSnapshotException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- public static class resetState<I extends Iface> extends org.apache.thrift.ProcessFunction<I, resetState_args> {
- public resetState() {
- super("resetState");
- }
-
- public resetState_args getEmptyArgsInstance() {
- return new resetState_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public resetState_result getResult(I iface, resetState_args args) throws org.apache.thrift.TException {
- resetState_result result = new resetState_result();
- iface.resetState();
- return result;
- }
- }
-
- public static class status<I extends Iface> extends org.apache.thrift.ProcessFunction<I, status_args> {
- public status() {
- super("status");
- }
-
- public status_args getEmptyArgsInstance() {
- return new status_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public status_result getResult(I iface, status_args args) throws org.apache.thrift.TException {
- status_result result = new status_result();
- result.success = iface.status();
- return result;
- }
- }
-
- public static class truncateInvalidTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, truncateInvalidTx_args> {
- public truncateInvalidTx() {
- super("truncateInvalidTx");
- }
-
- public truncateInvalidTx_args getEmptyArgsInstance() {
- return new truncateInvalidTx_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public truncateInvalidTx_result getResult(I iface, truncateInvalidTx_args args) throws org.apache.thrift.TException {
- truncateInvalidTx_result result = new truncateInvalidTx_result();
- result.success = iface.truncateInvalidTx(args.txns);
- return result;
- }
- }
-
- public static class truncateInvalidTxBefore<I extends Iface> extends org.apache.thrift.ProcessFunction<I, truncateInvalidTxBefore_args> {
- public truncateInvalidTxBefore() {
- super("truncateInvalidTxBefore");
- }
-
- public truncateInvalidTxBefore_args getEmptyArgsInstance() {
- return new truncateInvalidTxBefore_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public truncateInvalidTxBefore_result getResult(I iface, truncateInvalidTxBefore_args args) throws org.apache.thrift.TException {
- truncateInvalidTxBefore_result result = new truncateInvalidTxBefore_result();
- try {
- result.success = iface.truncateInvalidTxBefore(args.time);
- } catch (TInvalidTruncateTimeException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- public static class invalidTxSize<I extends Iface> extends org.apache.thrift.ProcessFunction<I, invalidTxSize_args> {
- public invalidTxSize() {
- super("invalidTxSize");
- }
-
- public invalidTxSize_args getEmptyArgsInstance() {
- return new invalidTxSize_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public invalidTxSize_result getResult(I iface, invalidTxSize_args args) throws org.apache.thrift.TException {
- invalidTxSize_result result = new invalidTxSize_result();
- result.success = iface.invalidTxSize();
- result.setSuccessIsSet(true);
- return result;
- }
- }
-
- public static class checkpoint<I extends Iface> extends org.apache.thrift.ProcessFunction<I, checkpoint_args> {
- public checkpoint() {
- super("checkpoint");
- }
-
- public checkpoint_args getEmptyArgsInstance() {
- return new checkpoint_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public checkpoint_result getResult(I iface, checkpoint_args args) throws org.apache.thrift.TException {
- checkpoint_result result = new checkpoint_result();
- try {
- result.success = iface.checkpoint(args.tx);
- } catch (TTransactionNotInProgressException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- }
-
- public static class startLong_args implements org.apache.thrift.TBase<startLong_args, startLong_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startLong_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new startLong_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new startLong_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startLong_args.class, metaDataMap);
- }
-
- public startLong_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public startLong_args(startLong_args other) {
- }
-
- public startLong_args deepCopy() {
- return new startLong_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof startLong_args)
- return this.equals((startLong_args)that);
- return false;
- }
-
- public boolean equals(startLong_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(startLong_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- startLong_args typedOther = (startLong_args)other;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("startLong_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class startLong_argsStandardSchemeFactory implements SchemeFactory {
- public startLong_argsStandardScheme getScheme() {
- return new startLong_argsStandardScheme();
- }
- }
-
- private static class startLong_argsStandardScheme extends StandardScheme<startLong_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, startLong_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, startLong_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class startLong_argsTupleSchemeFactory implements SchemeFactory {
- public startLong_argsTupleScheme getScheme() {
- return new startLong_argsTupleScheme();
- }
- }
-
- private static class startLong_argsTupleScheme extends TupleScheme<startLong_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, startLong_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, startLong_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class startLong_result implements org.apache.thrift.TBase<startLong_result, startLong_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startLong_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new startLong_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new startLong_resultTupleSchemeFactory());
- }
-
- public TTransaction success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTransaction.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startLong_result.class, metaDataMap);
- }
-
- public startLong_result() {
- }
-
- public startLong_result(
- TTransaction success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public startLong_result(startLong_result other) {
- if (other.isSetSuccess()) {
- this.success = new TTransaction(other.success);
- }
- }
-
- public startLong_result deepCopy() {
- return new startLong_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- public TTransaction getSuccess() {
- return this.success;
- }
-
- public startLong_result setSuccess(TTransaction success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((TTransaction)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof startLong_result)
- return this.equals((startLong_result)that);
- return false;
- }
-
- public boolean equals(startLong_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(startLong_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- startLong_result typedOther = (startLong_result)other;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("startLong_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (success != null) {
- success.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class startLong_resultStandardSchemeFactory implements SchemeFactory {
- public startLong_resultStandardScheme getScheme() {
- return new startLong_resultStandardScheme();
- }
- }
-
- private static class startLong_resultStandardScheme extends StandardScheme<startLong_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, startLong_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.success = new TTransaction();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, startLong_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- struct.success.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class startLong_resultTupleSchemeFactory implements SchemeFactory {
- public startLong_resultTupleScheme getScheme() {
- return new startLong_resultTupleScheme();
- }
- }
-
- private static class startLong_resultTupleScheme extends TupleScheme<startLong_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, startLong_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetSuccess()) {
- struct.success.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, startLong_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.success = new TTransaction();
- struct.success.read(iprot);
- struct.setSuccessIsSet(true);
- }
- }
- }
-
- }
-
- public static class startShort_args implements org.apache.thrift.TBase<startShort_args, startShort_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startShort_args");
-
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new startShort_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new startShort_argsTupleSchemeFactory());
- }
-
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-;
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startShort_args.class, metaDataMap);
- }
-
- public startShort_args() {
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public startShort_args(startShort_args other) {
- }
-
- public startShort_args deepCopy() {
- return new startShort_args(this);
- }
-
- @Override
- public void clear() {
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof startShort_args)
- return this.equals((startShort_args)that);
- return false;
- }
-
- public boolean equals(startShort_args that) {
- if (that == null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(startShort_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- startShort_args typedOther = (startShort_args)other;
-
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("startShort_args(");
- boolean first = true;
-
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class startShort_argsStandardSchemeFactory implements SchemeFactory {
- public startShort_argsStandardScheme getScheme() {
- return new startShort_argsStandardScheme();
- }
- }
-
- private static class startShort_argsStandardScheme extends StandardScheme<startShort_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, startShort_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, startShort_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class startShort_argsTupleSchemeFactory implements SchemeFactory {
- public startShort_argsTupleScheme getScheme() {
- return new startShort_argsTupleScheme();
- }
- }
-
- private static class startShort_argsTupleScheme extends TupleScheme<startShort_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, startShort_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, startShort_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- }
- }
-
- }
-
- public static class startShort_result implements org.apache.thrift.TBase<startShort_result, startShort_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startShort_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new startShort_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new startShort_resultTupleSchemeFactory());
- }
-
- public TTransaction success; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTransaction.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startShort_result.class, metaDataMap);
- }
-
- public startShort_result() {
- }
-
- public startShort_result(
- TTransaction success)
- {
- this();
- this.success = success;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public startShort_result(startShort_result other) {
- if (other.isSetSuccess()) {
- this.success = new TTransaction(other.success);
- }
- }
-
- public startShort_result deepCopy() {
- return new startShort_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- }
-
- public TTransaction getSuccess() {
- return this.success;
- }
-
- public startShort_result setSuccess(TTransaction success) {
- this.success = success;
- return this;
<TRUNCATED>
[35/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionServer.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionServer.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionServer.java
new file mode 100644
index 0000000..3b44534
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransactionServer.java
@@ -0,0 +1,11072 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+import org.apache.thrift.scheme.TupleScheme;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class TTransactionServer {
+
+ public interface Iface {
+
+ public TTransaction startLong() throws org.apache.thrift.TException;
+
+ public TTransaction startShort() throws org.apache.thrift.TException;
+
+ public TTransaction startShortTimeout(int timeout) throws org.apache.thrift.TException;
+
+ public TBoolean canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws TTransactionNotInProgressException, org.apache.thrift.TException;
+
+ public TBoolean commitTx(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException;
+
+ public void abortTx(TTransaction tx) throws org.apache.thrift.TException;
+
+ public boolean invalidateTx(long tx) throws org.apache.thrift.TException;
+
+ public ByteBuffer getSnapshot() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException;
+
+ public void resetState() throws org.apache.thrift.TException;
+
+ public String status() throws org.apache.thrift.TException;
+
+ public TBoolean truncateInvalidTx(Set<Long> txns) throws org.apache.thrift.TException;
+
+ public TBoolean truncateInvalidTxBefore(long time) throws TInvalidTruncateTimeException, org.apache.thrift.TException;
+
+ public int invalidTxSize() throws org.apache.thrift.TException;
+
+ public TTransaction checkpoint(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException;
+
+ }
+
+ public interface AsyncIface {
+
+ public void startLong(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.startLong_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void startShort(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.startShort_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void startShortTimeout(int timeout, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.startShortTimeout_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void canCommitTx(TTransaction tx, Set<ByteBuffer> changes, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.canCommitTx_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void commitTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.commitTx_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void abortTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.abortTx_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void invalidateTx(long tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.invalidateTx_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void getSnapshot(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.getSnapshot_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void resetState(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.resetState_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void status(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.status_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void truncateInvalidTx(Set<Long> txns, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.truncateInvalidTx_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void truncateInvalidTxBefore(long time, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.truncateInvalidTxBefore_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void invalidTxSize(org.apache.thrift.async.AsyncMethodCallback<AsyncClient.invalidTxSize_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void checkpoint(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.checkpoint_call> resultHandler) throws org.apache.thrift.TException;
+
+ }
+
+ public static class Client extends org.apache.thrift.TServiceClient implements Iface {
+ public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
+ public Factory() {}
+ public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
+ return new Client(prot);
+ }
+ public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ return new Client(iprot, oprot);
+ }
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol prot)
+ {
+ super(prot, prot);
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ super(iprot, oprot);
+ }
+
+ public TTransaction startLong() throws org.apache.thrift.TException
+ {
+ send_startLong();
+ return recv_startLong();
+ }
+
+ public void send_startLong() throws org.apache.thrift.TException
+ {
+ startLong_args args = new startLong_args();
+ sendBase("startLong", args);
+ }
+
+ public TTransaction recv_startLong() throws org.apache.thrift.TException
+ {
+ startLong_result result = new startLong_result();
+ receiveBase(result, "startLong");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "startLong failed: unknown result");
+ }
+
+ public TTransaction startShort() throws org.apache.thrift.TException
+ {
+ send_startShort();
+ return recv_startShort();
+ }
+
+ public void send_startShort() throws org.apache.thrift.TException
+ {
+ startShort_args args = new startShort_args();
+ sendBase("startShort", args);
+ }
+
+ public TTransaction recv_startShort() throws org.apache.thrift.TException
+ {
+ startShort_result result = new startShort_result();
+ receiveBase(result, "startShort");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "startShort failed: unknown result");
+ }
+
+ public TTransaction startShortTimeout(int timeout) throws org.apache.thrift.TException
+ {
+ send_startShortTimeout(timeout);
+ return recv_startShortTimeout();
+ }
+
+ public void send_startShortTimeout(int timeout) throws org.apache.thrift.TException
+ {
+ startShortTimeout_args args = new startShortTimeout_args();
+ args.setTimeout(timeout);
+ sendBase("startShortTimeout", args);
+ }
+
+ public TTransaction recv_startShortTimeout() throws org.apache.thrift.TException
+ {
+ startShortTimeout_result result = new startShortTimeout_result();
+ receiveBase(result, "startShortTimeout");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "startShortTimeout failed: unknown result");
+ }
+
+ public TBoolean canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws TTransactionNotInProgressException, org.apache.thrift.TException
+ {
+ send_canCommitTx(tx, changes);
+ return recv_canCommitTx();
+ }
+
+ public void send_canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws org.apache.thrift.TException
+ {
+ canCommitTx_args args = new canCommitTx_args();
+ args.setTx(tx);
+ args.setChanges(changes);
+ sendBase("canCommitTx", args);
+ }
+
+ public TBoolean recv_canCommitTx() throws TTransactionNotInProgressException, org.apache.thrift.TException
+ {
+ canCommitTx_result result = new canCommitTx_result();
+ receiveBase(result, "canCommitTx");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "canCommitTx failed: unknown result");
+ }
+
+ public TBoolean commitTx(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException
+ {
+ send_commitTx(tx);
+ return recv_commitTx();
+ }
+
+ public void send_commitTx(TTransaction tx) throws org.apache.thrift.TException
+ {
+ commitTx_args args = new commitTx_args();
+ args.setTx(tx);
+ sendBase("commitTx", args);
+ }
+
+ public TBoolean recv_commitTx() throws TTransactionNotInProgressException, org.apache.thrift.TException
+ {
+ commitTx_result result = new commitTx_result();
+ receiveBase(result, "commitTx");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "commitTx failed: unknown result");
+ }
+
+ public void abortTx(TTransaction tx) throws org.apache.thrift.TException
+ {
+ send_abortTx(tx);
+ recv_abortTx();
+ }
+
+ public void send_abortTx(TTransaction tx) throws org.apache.thrift.TException
+ {
+ abortTx_args args = new abortTx_args();
+ args.setTx(tx);
+ sendBase("abortTx", args);
+ }
+
+ public void recv_abortTx() throws org.apache.thrift.TException
+ {
+ abortTx_result result = new abortTx_result();
+ receiveBase(result, "abortTx");
+ return;
+ }
+
+ public boolean invalidateTx(long tx) throws org.apache.thrift.TException
+ {
+ send_invalidateTx(tx);
+ return recv_invalidateTx();
+ }
+
+ public void send_invalidateTx(long tx) throws org.apache.thrift.TException
+ {
+ invalidateTx_args args = new invalidateTx_args();
+ args.setTx(tx);
+ sendBase("invalidateTx", args);
+ }
+
+ public boolean recv_invalidateTx() throws org.apache.thrift.TException
+ {
+ invalidateTx_result result = new invalidateTx_result();
+ receiveBase(result, "invalidateTx");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "invalidateTx failed: unknown result");
+ }
+
+ public ByteBuffer getSnapshot() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException
+ {
+ send_getSnapshot();
+ return recv_getSnapshot();
+ }
+
+ public void send_getSnapshot() throws org.apache.thrift.TException
+ {
+ getSnapshot_args args = new getSnapshot_args();
+ sendBase("getSnapshot", args);
+ }
+
+ public ByteBuffer recv_getSnapshot() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException
+ {
+ getSnapshot_result result = new getSnapshot_result();
+ receiveBase(result, "getSnapshot");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getSnapshot failed: unknown result");
+ }
+
+ public void resetState() throws org.apache.thrift.TException
+ {
+ send_resetState();
+ recv_resetState();
+ }
+
+ public void send_resetState() throws org.apache.thrift.TException
+ {
+ resetState_args args = new resetState_args();
+ sendBase("resetState", args);
+ }
+
+ public void recv_resetState() throws org.apache.thrift.TException
+ {
+ resetState_result result = new resetState_result();
+ receiveBase(result, "resetState");
+ return;
+ }
+
+ public String status() throws org.apache.thrift.TException
+ {
+ send_status();
+ return recv_status();
+ }
+
+ public void send_status() throws org.apache.thrift.TException
+ {
+ status_args args = new status_args();
+ sendBase("status", args);
+ }
+
+ public String recv_status() throws org.apache.thrift.TException
+ {
+ status_result result = new status_result();
+ receiveBase(result, "status");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "status failed: unknown result");
+ }
+
+ public TBoolean truncateInvalidTx(Set<Long> txns) throws org.apache.thrift.TException
+ {
+ send_truncateInvalidTx(txns);
+ return recv_truncateInvalidTx();
+ }
+
+ public void send_truncateInvalidTx(Set<Long> txns) throws org.apache.thrift.TException
+ {
+ truncateInvalidTx_args args = new truncateInvalidTx_args();
+ args.setTxns(txns);
+ sendBase("truncateInvalidTx", args);
+ }
+
+ public TBoolean recv_truncateInvalidTx() throws org.apache.thrift.TException
+ {
+ truncateInvalidTx_result result = new truncateInvalidTx_result();
+ receiveBase(result, "truncateInvalidTx");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "truncateInvalidTx failed: unknown result");
+ }
+
+ public TBoolean truncateInvalidTxBefore(long time) throws TInvalidTruncateTimeException, org.apache.thrift.TException
+ {
+ send_truncateInvalidTxBefore(time);
+ return recv_truncateInvalidTxBefore();
+ }
+
+ public void send_truncateInvalidTxBefore(long time) throws org.apache.thrift.TException
+ {
+ truncateInvalidTxBefore_args args = new truncateInvalidTxBefore_args();
+ args.setTime(time);
+ sendBase("truncateInvalidTxBefore", args);
+ }
+
+ public TBoolean recv_truncateInvalidTxBefore() throws TInvalidTruncateTimeException, org.apache.thrift.TException
+ {
+ truncateInvalidTxBefore_result result = new truncateInvalidTxBefore_result();
+ receiveBase(result, "truncateInvalidTxBefore");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "truncateInvalidTxBefore failed: unknown result");
+ }
+
+ public int invalidTxSize() throws org.apache.thrift.TException
+ {
+ send_invalidTxSize();
+ return recv_invalidTxSize();
+ }
+
+ public void send_invalidTxSize() throws org.apache.thrift.TException
+ {
+ invalidTxSize_args args = new invalidTxSize_args();
+ sendBase("invalidTxSize", args);
+ }
+
+ public int recv_invalidTxSize() throws org.apache.thrift.TException
+ {
+ invalidTxSize_result result = new invalidTxSize_result();
+ receiveBase(result, "invalidTxSize");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "invalidTxSize failed: unknown result");
+ }
+
+ public TTransaction checkpoint(TTransaction tx) throws TTransactionNotInProgressException, org.apache.thrift.TException
+ {
+ send_checkpoint(tx);
+ return recv_checkpoint();
+ }
+
+ public void send_checkpoint(TTransaction tx) throws org.apache.thrift.TException
+ {
+ checkpoint_args args = new checkpoint_args();
+ args.setTx(tx);
+ sendBase("checkpoint", args);
+ }
+
+ public TTransaction recv_checkpoint() throws TTransactionNotInProgressException, org.apache.thrift.TException
+ {
+ checkpoint_result result = new checkpoint_result();
+ receiveBase(result, "checkpoint");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "checkpoint failed: unknown result");
+ }
+
+ }
+ public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
+ public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
+ private org.apache.thrift.async.TAsyncClientManager clientManager;
+ private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
+ public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
+ this.clientManager = clientManager;
+ this.protocolFactory = protocolFactory;
+ }
+ public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
+ return new AsyncClient(protocolFactory, clientManager, transport);
+ }
+ }
+
+ public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
+ super(protocolFactory, clientManager, transport);
+ }
+
+ public void startLong(org.apache.thrift.async.AsyncMethodCallback<startLong_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ startLong_call method_call = new startLong_call(resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class startLong_call extends org.apache.thrift.async.TAsyncMethodCall {
+ public startLong_call(org.apache.thrift.async.AsyncMethodCallback<startLong_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("startLong", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ startLong_args args = new startLong_args();
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TTransaction getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_startLong();
+ }
+ }
+
+ public void startShort(org.apache.thrift.async.AsyncMethodCallback<startShort_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ startShort_call method_call = new startShort_call(resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class startShort_call extends org.apache.thrift.async.TAsyncMethodCall {
+ public startShort_call(org.apache.thrift.async.AsyncMethodCallback<startShort_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("startShort", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ startShort_args args = new startShort_args();
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TTransaction getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_startShort();
+ }
+ }
+
+ public void startShortTimeout(int timeout, org.apache.thrift.async.AsyncMethodCallback<startShortTimeout_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ startShortTimeout_call method_call = new startShortTimeout_call(timeout, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class startShortTimeout_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private int timeout;
+ public startShortTimeout_call(int timeout, org.apache.thrift.async.AsyncMethodCallback<startShortTimeout_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.timeout = timeout;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("startShortTimeout", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ startShortTimeout_args args = new startShortTimeout_args();
+ args.setTimeout(timeout);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TTransaction getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_startShortTimeout();
+ }
+ }
+
+ public void canCommitTx(TTransaction tx, Set<ByteBuffer> changes, org.apache.thrift.async.AsyncMethodCallback<canCommitTx_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ canCommitTx_call method_call = new canCommitTx_call(tx, changes, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class canCommitTx_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private TTransaction tx;
+ private Set<ByteBuffer> changes;
+ public canCommitTx_call(TTransaction tx, Set<ByteBuffer> changes, org.apache.thrift.async.AsyncMethodCallback<canCommitTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.tx = tx;
+ this.changes = changes;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("canCommitTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ canCommitTx_args args = new canCommitTx_args();
+ args.setTx(tx);
+ args.setChanges(changes);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TBoolean getResult() throws TTransactionNotInProgressException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_canCommitTx();
+ }
+ }
+
+ public void commitTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<commitTx_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ commitTx_call method_call = new commitTx_call(tx, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class commitTx_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private TTransaction tx;
+ public commitTx_call(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<commitTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.tx = tx;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("commitTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ commitTx_args args = new commitTx_args();
+ args.setTx(tx);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TBoolean getResult() throws TTransactionNotInProgressException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_commitTx();
+ }
+ }
+
+ public void abortTx(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<abortTx_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ abortTx_call method_call = new abortTx_call(tx, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class abortTx_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private TTransaction tx;
+ public abortTx_call(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<abortTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.tx = tx;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("abortTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ abortTx_args args = new abortTx_args();
+ args.setTx(tx);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public void getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ (new Client(prot)).recv_abortTx();
+ }
+ }
+
+ public void invalidateTx(long tx, org.apache.thrift.async.AsyncMethodCallback<invalidateTx_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ invalidateTx_call method_call = new invalidateTx_call(tx, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class invalidateTx_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private long tx;
+ public invalidateTx_call(long tx, org.apache.thrift.async.AsyncMethodCallback<invalidateTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.tx = tx;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("invalidateTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ invalidateTx_args args = new invalidateTx_args();
+ args.setTx(tx);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public boolean getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_invalidateTx();
+ }
+ }
+
+ public void getSnapshot(org.apache.thrift.async.AsyncMethodCallback<getSnapshot_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ getSnapshot_call method_call = new getSnapshot_call(resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class getSnapshot_call extends org.apache.thrift.async.TAsyncMethodCall {
+ public getSnapshot_call(org.apache.thrift.async.AsyncMethodCallback<getSnapshot_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getSnapshot", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ getSnapshot_args args = new getSnapshot_args();
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public ByteBuffer getResult() throws TTransactionCouldNotTakeSnapshotException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_getSnapshot();
+ }
+ }
+
+ public void resetState(org.apache.thrift.async.AsyncMethodCallback<resetState_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ resetState_call method_call = new resetState_call(resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class resetState_call extends org.apache.thrift.async.TAsyncMethodCall {
+ public resetState_call(org.apache.thrift.async.AsyncMethodCallback<resetState_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("resetState", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ resetState_args args = new resetState_args();
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public void getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ (new Client(prot)).recv_resetState();
+ }
+ }
+
+ public void status(org.apache.thrift.async.AsyncMethodCallback<status_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ status_call method_call = new status_call(resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class status_call extends org.apache.thrift.async.TAsyncMethodCall {
+ public status_call(org.apache.thrift.async.AsyncMethodCallback<status_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("status", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ status_args args = new status_args();
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public String getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_status();
+ }
+ }
+
+ public void truncateInvalidTx(Set<Long> txns, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTx_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ truncateInvalidTx_call method_call = new truncateInvalidTx_call(txns, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class truncateInvalidTx_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private Set<Long> txns;
+ public truncateInvalidTx_call(Set<Long> txns, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTx_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.txns = txns;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("truncateInvalidTx", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ truncateInvalidTx_args args = new truncateInvalidTx_args();
+ args.setTxns(txns);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TBoolean getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_truncateInvalidTx();
+ }
+ }
+
+ public void truncateInvalidTxBefore(long time, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTxBefore_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ truncateInvalidTxBefore_call method_call = new truncateInvalidTxBefore_call(time, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class truncateInvalidTxBefore_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private long time;
+ public truncateInvalidTxBefore_call(long time, org.apache.thrift.async.AsyncMethodCallback<truncateInvalidTxBefore_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.time = time;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("truncateInvalidTxBefore", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ truncateInvalidTxBefore_args args = new truncateInvalidTxBefore_args();
+ args.setTime(time);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TBoolean getResult() throws TInvalidTruncateTimeException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_truncateInvalidTxBefore();
+ }
+ }
+
+ public void invalidTxSize(org.apache.thrift.async.AsyncMethodCallback<invalidTxSize_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ invalidTxSize_call method_call = new invalidTxSize_call(resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class invalidTxSize_call extends org.apache.thrift.async.TAsyncMethodCall {
+ public invalidTxSize_call(org.apache.thrift.async.AsyncMethodCallback<invalidTxSize_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("invalidTxSize", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ invalidTxSize_args args = new invalidTxSize_args();
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public int getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_invalidTxSize();
+ }
+ }
+
+ public void checkpoint(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<checkpoint_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ checkpoint_call method_call = new checkpoint_call(tx, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class checkpoint_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private TTransaction tx;
+ public checkpoint_call(TTransaction tx, org.apache.thrift.async.AsyncMethodCallback<checkpoint_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.tx = tx;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("checkpoint", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ checkpoint_args args = new checkpoint_args();
+ args.setTx(tx);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public TTransaction getResult() throws TTransactionNotInProgressException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_checkpoint();
+ }
+ }
+
+ }
+
+ public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
+ public Processor(I iface) {
+ super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
+ }
+
+ protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ super(iface, getProcessMap(processMap));
+ }
+
+ private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ processMap.put("startLong", new startLong());
+ processMap.put("startShort", new startShort());
+ processMap.put("startShortTimeout", new startShortTimeout());
+ processMap.put("canCommitTx", new canCommitTx());
+ processMap.put("commitTx", new commitTx());
+ processMap.put("abortTx", new abortTx());
+ processMap.put("invalidateTx", new invalidateTx());
+ processMap.put("getSnapshot", new getSnapshot());
+ processMap.put("resetState", new resetState());
+ processMap.put("status", new status());
+ processMap.put("truncateInvalidTx", new truncateInvalidTx());
+ processMap.put("truncateInvalidTxBefore", new truncateInvalidTxBefore());
+ processMap.put("invalidTxSize", new invalidTxSize());
+ processMap.put("checkpoint", new checkpoint());
+ return processMap;
+ }
+
+ public static class startLong<I extends Iface> extends org.apache.thrift.ProcessFunction<I, startLong_args> {
+ public startLong() {
+ super("startLong");
+ }
+
+ public startLong_args getEmptyArgsInstance() {
+ return new startLong_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public startLong_result getResult(I iface, startLong_args args) throws org.apache.thrift.TException {
+ startLong_result result = new startLong_result();
+ result.success = iface.startLong();
+ return result;
+ }
+ }
+
+ public static class startShort<I extends Iface> extends org.apache.thrift.ProcessFunction<I, startShort_args> {
+ public startShort() {
+ super("startShort");
+ }
+
+ public startShort_args getEmptyArgsInstance() {
+ return new startShort_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public startShort_result getResult(I iface, startShort_args args) throws org.apache.thrift.TException {
+ startShort_result result = new startShort_result();
+ result.success = iface.startShort();
+ return result;
+ }
+ }
+
+ public static class startShortTimeout<I extends Iface> extends org.apache.thrift.ProcessFunction<I, startShortTimeout_args> {
+ public startShortTimeout() {
+ super("startShortTimeout");
+ }
+
+ public startShortTimeout_args getEmptyArgsInstance() {
+ return new startShortTimeout_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public startShortTimeout_result getResult(I iface, startShortTimeout_args args) throws org.apache.thrift.TException {
+ startShortTimeout_result result = new startShortTimeout_result();
+ result.success = iface.startShortTimeout(args.timeout);
+ return result;
+ }
+ }
+
+ public static class canCommitTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, canCommitTx_args> {
+ public canCommitTx() {
+ super("canCommitTx");
+ }
+
+ public canCommitTx_args getEmptyArgsInstance() {
+ return new canCommitTx_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public canCommitTx_result getResult(I iface, canCommitTx_args args) throws org.apache.thrift.TException {
+ canCommitTx_result result = new canCommitTx_result();
+ try {
+ result.success = iface.canCommitTx(args.tx, args.changes);
+ } catch (TTransactionNotInProgressException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ public static class commitTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, commitTx_args> {
+ public commitTx() {
+ super("commitTx");
+ }
+
+ public commitTx_args getEmptyArgsInstance() {
+ return new commitTx_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public commitTx_result getResult(I iface, commitTx_args args) throws org.apache.thrift.TException {
+ commitTx_result result = new commitTx_result();
+ try {
+ result.success = iface.commitTx(args.tx);
+ } catch (TTransactionNotInProgressException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ public static class abortTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, abortTx_args> {
+ public abortTx() {
+ super("abortTx");
+ }
+
+ public abortTx_args getEmptyArgsInstance() {
+ return new abortTx_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public abortTx_result getResult(I iface, abortTx_args args) throws org.apache.thrift.TException {
+ abortTx_result result = new abortTx_result();
+ iface.abortTx(args.tx);
+ return result;
+ }
+ }
+
+ public static class invalidateTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, invalidateTx_args> {
+ public invalidateTx() {
+ super("invalidateTx");
+ }
+
+ public invalidateTx_args getEmptyArgsInstance() {
+ return new invalidateTx_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public invalidateTx_result getResult(I iface, invalidateTx_args args) throws org.apache.thrift.TException {
+ invalidateTx_result result = new invalidateTx_result();
+ result.success = iface.invalidateTx(args.tx);
+ result.setSuccessIsSet(true);
+ return result;
+ }
+ }
+
+ public static class getSnapshot<I extends Iface> extends org.apache.thrift.ProcessFunction<I, getSnapshot_args> {
+ public getSnapshot() {
+ super("getSnapshot");
+ }
+
+ public getSnapshot_args getEmptyArgsInstance() {
+ return new getSnapshot_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public getSnapshot_result getResult(I iface, getSnapshot_args args) throws org.apache.thrift.TException {
+ getSnapshot_result result = new getSnapshot_result();
+ try {
+ result.success = iface.getSnapshot();
+ } catch (TTransactionCouldNotTakeSnapshotException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ public static class resetState<I extends Iface> extends org.apache.thrift.ProcessFunction<I, resetState_args> {
+ public resetState() {
+ super("resetState");
+ }
+
+ public resetState_args getEmptyArgsInstance() {
+ return new resetState_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public resetState_result getResult(I iface, resetState_args args) throws org.apache.thrift.TException {
+ resetState_result result = new resetState_result();
+ iface.resetState();
+ return result;
+ }
+ }
+
+ public static class status<I extends Iface> extends org.apache.thrift.ProcessFunction<I, status_args> {
+ public status() {
+ super("status");
+ }
+
+ public status_args getEmptyArgsInstance() {
+ return new status_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public status_result getResult(I iface, status_args args) throws org.apache.thrift.TException {
+ status_result result = new status_result();
+ result.success = iface.status();
+ return result;
+ }
+ }
+
+ public static class truncateInvalidTx<I extends Iface> extends org.apache.thrift.ProcessFunction<I, truncateInvalidTx_args> {
+ public truncateInvalidTx() {
+ super("truncateInvalidTx");
+ }
+
+ public truncateInvalidTx_args getEmptyArgsInstance() {
+ return new truncateInvalidTx_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public truncateInvalidTx_result getResult(I iface, truncateInvalidTx_args args) throws org.apache.thrift.TException {
+ truncateInvalidTx_result result = new truncateInvalidTx_result();
+ result.success = iface.truncateInvalidTx(args.txns);
+ return result;
+ }
+ }
+
+ public static class truncateInvalidTxBefore<I extends Iface> extends org.apache.thrift.ProcessFunction<I, truncateInvalidTxBefore_args> {
+ public truncateInvalidTxBefore() {
+ super("truncateInvalidTxBefore");
+ }
+
+ public truncateInvalidTxBefore_args getEmptyArgsInstance() {
+ return new truncateInvalidTxBefore_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public truncateInvalidTxBefore_result getResult(I iface, truncateInvalidTxBefore_args args) throws org.apache.thrift.TException {
+ truncateInvalidTxBefore_result result = new truncateInvalidTxBefore_result();
+ try {
+ result.success = iface.truncateInvalidTxBefore(args.time);
+ } catch (TInvalidTruncateTimeException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ public static class invalidTxSize<I extends Iface> extends org.apache.thrift.ProcessFunction<I, invalidTxSize_args> {
+ public invalidTxSize() {
+ super("invalidTxSize");
+ }
+
+ public invalidTxSize_args getEmptyArgsInstance() {
+ return new invalidTxSize_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public invalidTxSize_result getResult(I iface, invalidTxSize_args args) throws org.apache.thrift.TException {
+ invalidTxSize_result result = new invalidTxSize_result();
+ result.success = iface.invalidTxSize();
+ result.setSuccessIsSet(true);
+ return result;
+ }
+ }
+
+ public static class checkpoint<I extends Iface> extends org.apache.thrift.ProcessFunction<I, checkpoint_args> {
+ public checkpoint() {
+ super("checkpoint");
+ }
+
+ public checkpoint_args getEmptyArgsInstance() {
+ return new checkpoint_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public checkpoint_result getResult(I iface, checkpoint_args args) throws org.apache.thrift.TException {
+ checkpoint_result result = new checkpoint_result();
+ try {
+ result.success = iface.checkpoint(args.tx);
+ } catch (TTransactionNotInProgressException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ }
+
+ public static class startLong_args implements org.apache.thrift.TBase<startLong_args, startLong_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startLong_args");
+
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new startLong_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new startLong_argsTupleSchemeFactory());
+ }
+
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+;
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startLong_args.class, metaDataMap);
+ }
+
+ public startLong_args() {
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public startLong_args(startLong_args other) {
+ }
+
+ public startLong_args deepCopy() {
+ return new startLong_args(this);
+ }
+
+ @Override
+ public void clear() {
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof startLong_args)
+ return this.equals((startLong_args)that);
+ return false;
+ }
+
+ public boolean equals(startLong_args that) {
+ if (that == null)
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(startLong_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ startLong_args typedOther = (startLong_args)other;
+
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("startLong_args(");
+ boolean first = true;
+
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class startLong_argsStandardSchemeFactory implements SchemeFactory {
+ public startLong_argsStandardScheme getScheme() {
+ return new startLong_argsStandardScheme();
+ }
+ }
+
+ private static class startLong_argsStandardScheme extends StandardScheme<startLong_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, startLong_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, startLong_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class startLong_argsTupleSchemeFactory implements SchemeFactory {
+ public startLong_argsTupleScheme getScheme() {
+ return new startLong_argsTupleScheme();
+ }
+ }
+
+ private static class startLong_argsTupleScheme extends TupleScheme<startLong_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, startLong_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, startLong_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ }
+ }
+
+ }
+
+ public static class startLong_result implements org.apache.thrift.TBase<startLong_result, startLong_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startLong_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new startLong_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new startLong_resultTupleSchemeFactory());
+ }
+
+ public TTransaction success; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTransaction.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startLong_result.class, metaDataMap);
+ }
+
+ public startLong_result() {
+ }
+
+ public startLong_result(
+ TTransaction success)
+ {
+ this();
+ this.success = success;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public startLong_result(startLong_result other) {
+ if (other.isSetSuccess()) {
+ this.success = new TTransaction(other.success);
+ }
+ }
+
+ public startLong_result deepCopy() {
+ return new startLong_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ }
+
+ public TTransaction getSuccess() {
+ return this.success;
+ }
+
+ public startLong_result setSuccess(TTransaction success) {
+ this.success = success;
+ return this;
+ }
+
+ public void unsetSuccess() {
+ this.success = null;
+ }
+
+ /** Returns true if field success is set (has been assigned a value) and false otherwise */
+ public boolean isSetSuccess() {
+ return this.success != null;
+ }
+
+ public void setSuccessIsSet(boolean value) {
+ if (!value) {
+ this.success = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case SUCCESS:
+ if (value == null) {
+ unsetSuccess();
+ } else {
+ setSuccess((TTransaction)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SUCCESS:
+ return getSuccess();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SUCCESS:
+ return isSetSuccess();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof startLong_result)
+ return this.equals((startLong_result)that);
+ return false;
+ }
+
+ public boolean equals(startLong_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_success = true && this.isSetSuccess();
+ boolean that_present_success = true && that.isSetSuccess();
+ if (this_present_success || that_present_success) {
+ if (!(this_present_success && that_present_success))
+ return false;
+ if (!this.success.equals(that.success))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(startLong_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ startLong_result typedOther = (startLong_result)other;
+
+ lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSuccess()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("startLong_result(");
+ boolean first = true;
+
+ sb.append("success:");
+ if (this.success == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.success);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ if (success != null) {
+ success.validate();
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class startLong_resultStandardSchemeFactory implements SchemeFactory {
+ public startLong_resultStandardScheme getScheme() {
+ return new startLong_resultStandardScheme();
+ }
+ }
+
+ private static class startLong_resultStandardScheme extends StandardScheme<startLong_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, startLong_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 0: // SUCCESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.success = new TTransaction();
+ struct.success.read(iprot);
+ struct.setSuccessIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, startLong_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.success != null) {
+ oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+ struct.success.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class startLong_resultTupleSchemeFactory implements SchemeFactory {
+ public startLong_resultTupleScheme getScheme() {
+ return new startLong_resultTupleScheme();
+ }
+ }
+
+ private static class startLong_resultTupleScheme extends TupleScheme<startLong_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, startLong_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetSuccess()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetSuccess()) {
+ struct.success.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, startLong_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.success = new TTransaction();
+ struct.success.read(iprot);
+ struct.setSuccessIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class startShort_args implements org.apache.thrift.TBase<startShort_args, startShort_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startShort_args");
+
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new startShort_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new startShort_argsTupleSchemeFactory());
+ }
+
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+;
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startShort_args.class, metaDataMap);
+ }
+
+ public startShort_args() {
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public startShort_args(startShort_args other) {
+ }
+
+ public startShort_args deepCopy() {
+ return new startShort_args(this);
+ }
+
+ @Override
+ public void clear() {
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof startShort_args)
+ return this.equals((startShort_args)that);
+ return false;
+ }
+
+ public boolean equals(startShort_args that) {
+ if (that == null)
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(startShort_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ startShort_args typedOther = (startShort_args)other;
+
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("startShort_args(");
+ boolean first = true;
+
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class startShort_argsStandardSchemeFactory implements SchemeFactory {
+ public startShort_argsStandardScheme getScheme() {
+ return new startShort_argsStandardScheme();
+ }
+ }
+
+ private static class startShort_argsStandardScheme extends StandardScheme<startShort_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, startShort_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, startShort_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class startShort_argsTupleSchemeFactory implements SchemeFactory {
+ public startShort_argsTupleScheme getScheme() {
+ return new startShort_argsTupleScheme();
+ }
+ }
+
+ private static class startShort_argsTupleScheme extends TupleScheme<startShort_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, startShort_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, startShort_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ }
+ }
+
+ }
+
+ public static class startShort_result implements org.apache.thrift.TBase<startShort_result, startShort_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("startShort_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new startShort_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new startShort_resultTupleSchemeFactory());
+ }
+
+ public TTransaction success; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTransaction.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(startShort_result.class, metaDataMap);
+ }
+
+ public startShort_result() {
+ }
+
+ public startShort_result(
+ TTransaction success)
+ {
+ this();
+ this.success = success;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public startShort_result(startShort_result other) {
+ if (other.isSetSuccess()) {
+ this.success = new TTransaction(other.success);
+ }
+ }
+
+ public startShort_result deepCopy() {
+ return new startShort_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ }
+
+ public TTransaction getSuccess() {
+ return this.success;
+ }
+
+ public startShort_result setSuccess(TTransaction success) {
+ this.success = success;
+ re
<TRUNCATED>
[39/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java b/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java
new file mode 100644
index 0000000..cfefc83
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionManager.java
@@ -0,0 +1,1398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.AbstractService;
+import com.google.inject.Inject;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.metrics.DefaultMetricsCollector;
+import org.apache.tephra.metrics.MetricsCollector;
+import org.apache.tephra.persist.NoOpTransactionStateStorage;
+import org.apache.tephra.persist.TransactionEdit;
+import org.apache.tephra.persist.TransactionLog;
+import org.apache.tephra.persist.TransactionLogReader;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionStateStorage;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.util.TxUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * This is the central place to manage all active transactions in the system.
+ *
+ * A transaction consists of
+ * <ul>
+ * <li>A write pointer: This is the version used for all writes of that transaction.</li>
+ * <li>A read pointer: All reads under the transaction use this as an upper bound for the version.</li>
+ * <li>A set of excluded versions: These are the write versions of other transactions that must be excluded from
+ * reads, because those transactions are still in progress, or they failed but couldn't be properly rolled back.</li>
+ * </ul>
+ * To use the transaction system, a client must follow this sequence of steps:
+ * <ol>
+ * <li>Request a new transaction.</li>
+ * <li>Use the transaction to read and write datasets. Datasets are encouraged to cache the writes of the
+ * transaction in memory, to reduce the cost of rollback in case the transaction fails. </li>
+ * <li>Check whether the transaction has conflicts. For this, the set of change keys are submitted via canCommit(),
+ * and the transaction manager verifies that none of these keys are in conflict with other transactions that
+ * committed since the start of this transaction.</li>
+ * <li>If the transaction has conflicts:
+ * <ol>
+ * <li>Roll back the changes in every dataset that was changed. This can happen in-memory if the
+ * changes were cached.</li>
+ * <li>Abort the transaction to remove it from the active transactions.</li>
+ * </ol>
+ * <li>If the transaction has no conflicts:</li>
+ * <ol>
+ * <li>Persist all datasets changes to storage.</li>
+ * <li>Commit the transaction. This will repeat the conflict detection, because more overlapping transactions
+ * may have committed since the first conflict check.</li>
+ * <li>If the transaction has conflicts:</li>
+ * <ol>
+ * <li>Roll back the changes in every dataset that was changed. This is more expensive because
+ * changes must be undone in persistent storage.</li>
+ * <li>Abort the transaction to remove it from the active transactions.</li>
+ * </ol>
+ * </ol>
+ * </ol>
+ * Transactions may be short or long-running. A short transaction is started with a timeout, and if it is not
+ * committed before that timeout, it is invalidated and excluded from future reads. A long-running transaction has
+ * no timeout and will remain active until it is committed or aborted. Long transactions are typically used in
+ * map/reduce jobs and can produce enormous amounts of changes. Therefore, long transactions do not participate in
+ * conflict detection (they would almost always have conflicts). We also assume that the changes of long transactions
+ * are not tracked, and therefore cannot be rolled back. Hence, when a long transaction is aborted, it remains in the
+ * list of excluded transactions to make its writes invisible.
+ */
+public class TransactionManager extends AbstractService {
+ // todo: optimize heavily
+
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionManager.class);
+
+ // poll every 1 second to check whether a snapshot is needed
+ private static final long SNAPSHOT_POLL_INTERVAL = 1000L;
+
+ //poll every 10 second to emit metrics
+ private static final long METRICS_POLL_INTERVAL = 10000L;
+
+ private static final long[] NO_INVALID_TX = { };
+
+ // Transactions that are in progress, with their info.
+ private final NavigableMap<Long, InProgressTx> inProgress = new ConcurrentSkipListMap<Long, InProgressTx>();
+
+ // the list of transactions that are invalid (not properly committed/aborted, or timed out)
+ // TODO: explain usage of two arrays
+ private final LongArrayList invalid = new LongArrayList();
+ private long[] invalidArray = NO_INVALID_TX;
+
+ // todo: use moving array instead (use Long2ObjectMap<byte[]> in fastutil)
+ // todo: should this be consolidated with inProgress?
+ // commit time next writePointer -> changes made by this tx
+ private final NavigableMap<Long, Set<ChangeId>> committedChangeSets =
+ new ConcurrentSkipListMap<Long, Set<ChangeId>>();
+ // not committed yet
+ private final Map<Long, Set<ChangeId>> committingChangeSets = Maps.newConcurrentMap();
+
+ private long readPointer;
+ private long lastWritePointer;
+ private MetricsCollector txMetricsCollector;
+
+ private final TransactionStateStorage persistor;
+
+ private final int cleanupInterval;
+ private final int defaultTimeout;
+ private final int defaultLongTimeout;
+ private DaemonThreadExecutor cleanupThread = null;
+
+ private volatile TransactionLog currentLog;
+
+ // timestamp of the last completed snapshot
+ private long lastSnapshotTime;
+ // frequency in millis to perform snapshots
+ private final long snapshotFrequencyInSeconds;
+ // number of most recent snapshots to retain
+ private final int snapshotRetainCount;
+ private DaemonThreadExecutor snapshotThread;
+ private DaemonThreadExecutor metricsThread;
+
+ // lock guarding change of the current transaction log
+ private final ReentrantReadWriteLock logLock = new ReentrantReadWriteLock();
+ private final Lock logReadLock = logLock.readLock();
+ private final Lock logWriteLock = logLock.writeLock();
+
+ // fudge factor (in milliseconds) used when interpreting transactions as LONG based on expiration
+ // TODO: REMOVE WITH txnBackwardsCompatCheck()
+ private final long longTimeoutTolerance;
+
+ public TransactionManager(Configuration config) {
+ this(config, new NoOpTransactionStateStorage(new SnapshotCodecProvider(config)), new DefaultMetricsCollector());
+ }
+
+ @Inject
+ public TransactionManager(Configuration conf, @Nonnull TransactionStateStorage persistor,
+ MetricsCollector txMetricsCollector) {
+ this.persistor = persistor;
+ cleanupInterval = conf.getInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL,
+ TxConstants.Manager.DEFAULT_TX_CLEANUP_INTERVAL);
+ defaultTimeout = conf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT,
+ TxConstants.Manager.DEFAULT_TX_TIMEOUT);
+ defaultLongTimeout = conf.getInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
+ TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT);
+ snapshotFrequencyInSeconds = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
+ TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL);
+ // must always keep at least 1 snapshot
+ snapshotRetainCount = Math.max(conf.getInt(TxConstants.Manager.CFG_TX_SNAPSHOT_RETAIN,
+ TxConstants.Manager.DEFAULT_TX_SNAPSHOT_RETAIN), 1);
+
+ // intentionally not using a constant, as this config should not be exposed
+ // TODO: REMOVE WITH txnBackwardsCompatCheck()
+ longTimeoutTolerance = conf.getLong("data.tx.long.timeout.tolerance", 10000);
+
+ //
+ this.txMetricsCollector = txMetricsCollector;
+ this.txMetricsCollector.configure(conf);
+ clear();
+ }
+
+ private void clear() {
+ invalid.clear();
+ invalidArray = NO_INVALID_TX;
+ inProgress.clear();
+ committedChangeSets.clear();
+ committingChangeSets.clear();
+ lastWritePointer = 0;
+ readPointer = 0;
+ lastSnapshotTime = 0;
+ }
+
+ private boolean isStopping() {
+ return State.STOPPING.equals(state());
+ }
+
+ @Override
+ public synchronized void doStart() {
+ LOG.info("Starting transaction manager.");
+ txMetricsCollector.start();
+ // start up the persistor
+ persistor.startAndWait();
+ try {
+ persistor.setupStorage();
+ } catch (IOException e) {
+ Throwables.propagate(e);
+ }
+ // establish defaults in case there is no persistence
+ clear();
+ // attempt to recover state from last run
+ recoverState();
+ // start the periodic cleanup thread
+ startCleanupThread();
+ startSnapshotThread();
+ startMetricsThread();
+ // initialize the WAL if we did not force a snapshot in recoverState()
+ initLog();
+ // initialize next write pointer if needed
+ if (lastWritePointer == 0) {
+ lastWritePointer = getNextWritePointer();
+ readPointer = lastWritePointer;
+ }
+
+ notifyStarted();
+ }
+
+ private void initLog() {
+ if (currentLog == null) {
+ try {
+ currentLog = persistor.createLog(System.currentTimeMillis());
+ } catch (IOException ioe) {
+ throw Throwables.propagate(ioe);
+ }
+ }
+ }
+
+ private void startCleanupThread() {
+ if (cleanupInterval <= 0 || defaultTimeout <= 0) {
+ return;
+ }
+ LOG.info("Starting periodic timed-out transaction cleanup every " + cleanupInterval +
+ " seconds with default timeout of " + defaultTimeout + " seconds.");
+ this.cleanupThread = new DaemonThreadExecutor("tx-clean-timeout") {
+ @Override
+ public void doRun() {
+ cleanupTimedOutTransactions();
+ }
+
+ @Override
+ public long getSleepMillis() {
+ return cleanupInterval * 1000;
+ }
+ };
+ cleanupThread.start();
+ }
+
+ private void startSnapshotThread() {
+ if (snapshotFrequencyInSeconds > 0) {
+ LOG.info("Starting periodic snapshot thread, frequency = " + snapshotFrequencyInSeconds +
+ " seconds, location = " + persistor.getLocation());
+ this.snapshotThread = new DaemonThreadExecutor("tx-snapshot") {
+ @Override
+ public void doRun() {
+ long currentTime = System.currentTimeMillis();
+ if (lastSnapshotTime < (currentTime - snapshotFrequencyInSeconds * 1000)) {
+ try {
+ doSnapshot(false);
+ } catch (IOException ioe) {
+ LOG.error("Periodic snapshot failed!", ioe);
+ }
+ }
+ }
+
+ @Override
+ protected void onShutdown() {
+ // perform a final snapshot
+ try {
+ LOG.info("Writing final snapshot prior to shutdown");
+ doSnapshot(true);
+ } catch (IOException ioe) {
+ LOG.error("Failed performing final snapshot on shutdown", ioe);
+ }
+ }
+
+ @Override
+ public long getSleepMillis() {
+ return SNAPSHOT_POLL_INTERVAL;
+ }
+ };
+ snapshotThread.start();
+ }
+ }
+
+ // Emits Transaction Data structures size as metrics
+ private void startMetricsThread() {
+ LOG.info("Starting periodic Metrics Emitter thread, frequency = " + METRICS_POLL_INTERVAL);
+ this.metricsThread = new DaemonThreadExecutor("tx-metrics") {
+ @Override
+ public void doRun() {
+ txMetricsCollector.gauge("committing.size", committingChangeSets.size());
+ txMetricsCollector.gauge("committed.size", committedChangeSets.size());
+ txMetricsCollector.gauge("inprogress.size", inProgress.size());
+ txMetricsCollector.gauge("invalid.size", invalidArray.length);
+ }
+
+ @Override
+ protected void onShutdown() {
+ // perform a final metrics emit
+ txMetricsCollector.gauge("committing.size", committingChangeSets.size());
+ txMetricsCollector.gauge("committed.size", committedChangeSets.size());
+ txMetricsCollector.gauge("inprogress.size", inProgress.size());
+ txMetricsCollector.gauge("invalid.size", invalidArray.length);
+ }
+
+ @Override
+ public long getSleepMillis() {
+ return METRICS_POLL_INTERVAL;
+ }
+ };
+ metricsThread.start();
+ }
+
+ private void cleanupTimedOutTransactions() {
+ List<TransactionEdit> invalidEdits = null;
+ this.logReadLock.lock();
+ try {
+ synchronized (this) {
+ if (!isRunning()) {
+ return;
+ }
+
+ long currentTime = System.currentTimeMillis();
+ List<Long> timedOut = Lists.newArrayList();
+ for (Map.Entry<Long, InProgressTx> tx : inProgress.entrySet()) {
+ long expiration = tx.getValue().getExpiration();
+ if (expiration >= 0L && currentTime > expiration) {
+ // timed out, remember tx id (can't remove while iterating over entries)
+ timedOut.add(tx.getKey());
+ LOG.info("Tx invalid list: added tx {} because of timeout", tx.getKey());
+ } else if (expiration < 0) {
+ LOG.warn("Transaction {} has negative expiration time {}. Likely cause is the transaction was not " +
+ "migrated correctly, this transaction will be expired immediately",
+ tx.getKey(), expiration);
+ timedOut.add(tx.getKey());
+ }
+ }
+ if (!timedOut.isEmpty()) {
+ invalidEdits = Lists.newArrayListWithCapacity(timedOut.size());
+ invalid.addAll(timedOut);
+ for (long tx : timedOut) {
+ committingChangeSets.remove(tx);
+ inProgress.remove(tx);
+ invalidEdits.add(TransactionEdit.createInvalid(tx));
+ }
+
+ // todo: find a more efficient way to keep this sorted. Could it just be an array?
+ Collections.sort(invalid);
+ invalidArray = invalid.toLongArray();
+ LOG.info("Invalidated {} transactions due to timeout.", timedOut.size());
+ }
+ }
+ if (invalidEdits != null) {
+ appendToLog(invalidEdits);
+ }
+ } finally {
+ this.logReadLock.unlock();
+ }
+ }
+
+ public synchronized TransactionSnapshot getSnapshot() throws IOException {
+ TransactionSnapshot snapshot = null;
+ if (!isRunning() && !isStopping()) {
+ return null;
+ }
+
+ long now = System.currentTimeMillis();
+ // avoid duplicate snapshots at same timestamp
+ if (now == lastSnapshotTime || (currentLog != null && now == currentLog.getTimestamp())) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ } catch (InterruptedException ie) { }
+ }
+ // copy in memory state
+ snapshot = getCurrentState();
+
+ LOG.debug("Starting snapshot of transaction state with timestamp {}", snapshot.getTimestamp());
+ LOG.debug("Returning snapshot of state: " + snapshot);
+ return snapshot;
+ }
+
+ /**
+ * Take a snapshot of the transaction state and serialize it into the given output stream.
+ * @return whether a snapshot was taken.
+ */
+ public boolean takeSnapshot(OutputStream out) throws IOException {
+ TransactionSnapshot snapshot = getSnapshot();
+ if (snapshot != null) {
+ persistor.writeSnapshot(out, snapshot);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void doSnapshot(boolean closing) throws IOException {
+ long snapshotTime = 0L;
+ TransactionSnapshot snapshot = null;
+ TransactionLog oldLog = null;
+ try {
+ this.logWriteLock.lock();
+ try {
+ synchronized (this) {
+ snapshot = getSnapshot();
+ if (snapshot == null && !closing) {
+ return;
+ }
+ if (snapshot != null) {
+ snapshotTime = snapshot.getTimestamp();
+ }
+
+ // roll WAL
+ oldLog = currentLog;
+ if (!closing) {
+ currentLog = persistor.createLog(snapshot.getTimestamp());
+ }
+ }
+ // there may not be an existing log on startup
+ if (oldLog != null) {
+ oldLog.close();
+ }
+ } finally {
+ this.logWriteLock.unlock();
+ }
+
+ // save snapshot
+ if (snapshot != null) {
+ persistor.writeSnapshot(snapshot);
+ lastSnapshotTime = snapshotTime;
+
+ // clean any obsoleted snapshots and WALs
+ long oldestRetainedTimestamp = persistor.deleteOldSnapshots(snapshotRetainCount);
+ persistor.deleteLogsOlderThan(oldestRetainedTimestamp);
+ }
+ } catch (IOException ioe) {
+ abortService("Snapshot (timestamp " + snapshotTime + ") failed due to: " + ioe.getMessage(), ioe);
+ }
+ }
+
+ public synchronized TransactionSnapshot getCurrentState() {
+ return TransactionSnapshot.copyFrom(System.currentTimeMillis(), readPointer, lastWritePointer,
+ invalid, inProgress, committingChangeSets, committedChangeSets);
+ }
+
+ public synchronized void recoverState() {
+ try {
+ TransactionSnapshot lastSnapshot = persistor.getLatestSnapshot();
+ // if we failed before a snapshot could complete, we might not have one to restore
+ if (lastSnapshot != null) {
+ restoreSnapshot(lastSnapshot);
+ }
+ // replay any WALs since the last snapshot
+ Collection<TransactionLog> logs = persistor.getLogsSince(lastSnapshotTime);
+ if (logs != null) {
+ replayLogs(logs);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to read back transaction state:", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ /**
+ * Restore the initial in-memory transaction state from a snapshot.
+ */
+ private void restoreSnapshot(TransactionSnapshot snapshot) {
+ LOG.info("Restoring transaction state from snapshot at " + snapshot.getTimestamp());
+ Preconditions.checkState(lastSnapshotTime == 0, "lastSnapshotTime has been set!");
+ Preconditions.checkState(readPointer == 0, "readPointer has been set!");
+ Preconditions.checkState(lastWritePointer == 0, "lastWritePointer has been set!");
+ Preconditions.checkState(invalid.isEmpty(), "invalid list should be empty!");
+ Preconditions.checkState(inProgress.isEmpty(), "inProgress map should be empty!");
+ Preconditions.checkState(committingChangeSets.isEmpty(), "committingChangeSets should be empty!");
+ Preconditions.checkState(committedChangeSets.isEmpty(), "committedChangeSets should be empty!");
+ LOG.info("Restoring snapshot of state: " + snapshot);
+
+ lastSnapshotTime = snapshot.getTimestamp();
+ readPointer = snapshot.getReadPointer();
+ lastWritePointer = snapshot.getWritePointer();
+ invalid.addAll(snapshot.getInvalid());
+ inProgress.putAll(txnBackwardsCompatCheck(defaultLongTimeout, longTimeoutTolerance, snapshot.getInProgress()));
+ committingChangeSets.putAll(snapshot.getCommittingChangeSets());
+ committedChangeSets.putAll(snapshot.getCommittedChangeSets());
+ }
+
+ /**
+ * Check if in-progress transactions need to be migrated to have expiration time and type, if so do the migration.
+ * This is required for backwards compatibility, when long running transactions were represented
+ * with expiration time -1. This can be removed when we stop supporting SnapshotCodec version 1.
+ */
+ public static Map<Long, InProgressTx> txnBackwardsCompatCheck(int defaultLongTimeout, long longTimeoutTolerance,
+ Map<Long, InProgressTx> inProgress) {
+ for (Map.Entry<Long, InProgressTx> entry : inProgress.entrySet()) {
+ long writePointer = entry.getKey();
+ long expiration = entry.getValue().getExpiration();
+ // LONG transactions will either have a negative expiration or expiration set to the long timeout
+ // use a fudge factor on the expiration check, since expiraton is set based on system time, not the write pointer
+ if (entry.getValue().getType() == null &&
+ (expiration < 0 ||
+ (getTxExpirationFromWritePointer(writePointer, defaultLongTimeout) - expiration
+ < longTimeoutTolerance))) {
+ // handle null expiration
+ long newExpiration = getTxExpirationFromWritePointer(writePointer, defaultLongTimeout);
+ InProgressTx compatTx =
+ new InProgressTx(entry.getValue().getVisibilityUpperBound(), newExpiration, TransactionType.LONG,
+ entry.getValue().getCheckpointWritePointers());
+ entry.setValue(compatTx);
+ } else if (entry.getValue().getType() == null) {
+ InProgressTx compatTx =
+ new InProgressTx(entry.getValue().getVisibilityUpperBound(), entry.getValue().getExpiration(),
+ TransactionType.SHORT, entry.getValue().getCheckpointWritePointers());
+ entry.setValue(compatTx);
+ }
+ }
+ return inProgress;
+ }
+
+ /**
+ * Resets the state of the transaction manager.
+ */
+ public void resetState() {
+ this.logWriteLock.lock();
+ try {
+ // Take a snapshot before resetting the state, for debugging purposes
+ doSnapshot(false);
+ // Clear the state
+ clear();
+ // Take another snapshot: if no snapshot is taken after clearing the state
+ // and the manager is restarted, we will recover from the snapshot taken
+ // before resetting the state, which would be really bad
+ // This call will also init a new WAL
+ doSnapshot(false);
+ } catch (IOException e) {
+ LOG.error("Snapshot failed when resetting state!", e);
+ e.printStackTrace();
+ } finally {
+ this.logWriteLock.unlock();
+ }
+ }
+
+ /**
+ * Replay all logged edits from the given transaction logs.
+ */
+ private void replayLogs(Collection<TransactionLog> logs) {
+ for (TransactionLog log : logs) {
+ LOG.info("Replaying edits from transaction log " + log.getName());
+ int editCnt = 0;
+ try {
+ TransactionLogReader reader = log.getReader();
+ // reader may be null in the case of an empty file
+ if (reader == null) {
+ continue;
+ }
+ TransactionEdit edit = null;
+ while ((edit = reader.next()) != null) {
+ editCnt++;
+ switch (edit.getState()) {
+ case INPROGRESS:
+ long expiration = edit.getExpiration();
+ TransactionType type = edit.getType();
+ // Check if transaction needs to be migrated to have expiration and type. Previous version of
+ // long running transactions were represented with expiration time as -1.
+ // This can be removed when we stop supporting TransactionEditCodecV2.
+ if (expiration < 0) {
+ expiration = getTxExpirationFromWritePointer(edit.getWritePointer(), defaultLongTimeout);
+ type = TransactionType.LONG;
+ } else if (type == null) {
+ type = TransactionType.SHORT;
+ }
+ addInProgressAndAdvance(edit.getWritePointer(), edit.getVisibilityUpperBound(),
+ expiration, type);
+ break;
+ case COMMITTING:
+ addCommittingChangeSet(edit.getWritePointer(), edit.getChanges());
+ break;
+ case COMMITTED:
+ // TODO: need to reconcile usage of transaction id v/s write pointer TEPHRA-140
+ long transactionId = edit.getWritePointer();
+ long[] checkpointPointers = edit.getCheckpointPointers();
+ long writePointer = checkpointPointers == null || checkpointPointers.length == 0 ?
+ transactionId : checkpointPointers[checkpointPointers.length - 1];
+ doCommit(transactionId, writePointer, edit.getChanges(),
+ edit.getCommitPointer(), edit.getCanCommit());
+ break;
+ case INVALID:
+ doInvalidate(edit.getWritePointer());
+ break;
+ case ABORTED:
+ type = edit.getType();
+ // Check if transaction edit needs to be migrated to have type. Previous versions of
+ // ABORTED edits did not contain type.
+ // This can be removed when we stop supporting TransactionEditCodecV2.
+ if (type == null) {
+ InProgressTx inProgressTx = inProgress.get(edit.getWritePointer());
+ if (inProgressTx != null) {
+ type = inProgressTx.getType();
+ } else {
+ // If transaction is not in-progress, then it has either been already aborted or invalidated.
+ // We cannot determine the transaction's state based on current information, to be safe invalidate it.
+ LOG.warn("Invalidating transaction {} as it's type cannot be determined during replay",
+ edit.getWritePointer());
+ doInvalidate(edit.getWritePointer());
+ break;
+ }
+ }
+ doAbort(edit.getWritePointer(), edit.getCheckpointPointers(), type);
+ break;
+ case TRUNCATE_INVALID_TX:
+ if (edit.getTruncateInvalidTxTime() != 0) {
+ doTruncateInvalidTxBefore(edit.getTruncateInvalidTxTime());
+ } else {
+ doTruncateInvalidTx(edit.getTruncateInvalidTx());
+ }
+ break;
+ case CHECKPOINT:
+ doCheckpoint(edit.getWritePointer(), edit.getParentWritePointer());
+ break;
+ default:
+ // unknown type!
+ throw new IllegalArgumentException("Invalid state for WAL entry: " + edit.getState());
+ }
+ }
+ } catch (IOException ioe) {
+ throw Throwables.propagate(ioe);
+ } catch (InvalidTruncateTimeException e) {
+ throw Throwables.propagate(e);
+ }
+ LOG.info("Read " + editCnt + " edits from log " + log.getName());
+ }
+ }
+
+ @Override
+ public void doStop() {
+ Stopwatch timer = new Stopwatch().start();
+ LOG.info("Shutting down gracefully...");
+ // signal the cleanup thread to stop
+ if (cleanupThread != null) {
+ cleanupThread.shutdown();
+ try {
+ cleanupThread.join(30000L);
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted waiting for cleanup thread to stop");
+ Thread.currentThread().interrupt();
+ }
+ }
+ if (metricsThread != null) {
+ metricsThread.shutdown();
+ try {
+ metricsThread.join(30000L);
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted waiting for cleanup thread to stop");
+ Thread.currentThread().interrupt();
+ }
+ }
+ if (snapshotThread != null) {
+ // this will trigger a final snapshot on stop
+ snapshotThread.shutdown();
+ try {
+ snapshotThread.join(30000L);
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted waiting for snapshot thread to stop");
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ persistor.stopAndWait();
+ txMetricsCollector.stop();
+ timer.stop();
+ LOG.info("Took " + timer + " to stop");
+ notifyStopped();
+ }
+
+ /**
+ * Immediately shuts down the service, without going through the normal close process.
+ * @param message A message describing the source of the failure.
+ * @param error Any exception that caused the failure.
+ */
+ private void abortService(String message, Throwable error) {
+ if (isRunning()) {
+ LOG.error("Aborting transaction manager due to: " + message, error);
+ notifyFailed(error);
+ }
+ }
+
+ private void ensureAvailable() {
+ Preconditions.checkState(isRunning(), "Transaction Manager is not running.");
+ }
+
+ /**
+ * Start a short transaction with the default timeout.
+ */
+ public Transaction startShort() {
+ return startShort(defaultTimeout);
+ }
+
+ /**
+ * Start a short transaction with a given timeout.
+ * @param timeoutInSeconds the time out period in seconds.
+ */
+ public Transaction startShort(int timeoutInSeconds) {
+ Preconditions.checkArgument(timeoutInSeconds > 0, "timeout must be positive but is %s", timeoutInSeconds);
+ txMetricsCollector.rate("start.short");
+ Stopwatch timer = new Stopwatch().start();
+ long expiration = getTxExpiration(timeoutInSeconds);
+ Transaction tx = startTx(expiration, TransactionType.SHORT);
+ txMetricsCollector.histogram("start.short.latency", (int) timer.elapsedMillis());
+ return tx;
+ }
+
+ private static long getTxExpiration(long timeoutInSeconds) {
+ long currentTime = System.currentTimeMillis();
+ return currentTime + TimeUnit.SECONDS.toMillis(timeoutInSeconds);
+ }
+
+ public static long getTxExpirationFromWritePointer(long writePointer, long timeoutInSeconds) {
+ return writePointer / TxConstants.MAX_TX_PER_MS + TimeUnit.SECONDS.toMillis(timeoutInSeconds);
+ }
+
+ private long getNextWritePointer() {
+ // We want to align tx ids with current time. We assume that tx ids are sequential, but not less than
+ // System.currentTimeMillis() * MAX_TX_PER_MS.
+ return Math.max(lastWritePointer + 1, System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS);
+ }
+
+ /**
+ * Start a long transaction. Long transactions and do not participate in conflict detection. Also, aborting a long
+ * transaction moves it to the invalid list because we assume that its writes cannot be rolled back.
+ */
+ public Transaction startLong() {
+ txMetricsCollector.rate("start.long");
+ Stopwatch timer = new Stopwatch().start();
+ long expiration = getTxExpiration(defaultLongTimeout);
+ Transaction tx = startTx(expiration, TransactionType.LONG);
+ txMetricsCollector.histogram("start.long.latency", (int) timer.elapsedMillis());
+ return tx;
+ }
+
+ private Transaction startTx(long expiration, TransactionType type) {
+ Transaction tx = null;
+ long txid;
+ // guard against changes to the transaction log while processing
+ this.logReadLock.lock();
+ try {
+ synchronized (this) {
+ ensureAvailable();
+ txid = getNextWritePointer();
+ tx = createTransaction(txid, type);
+ addInProgressAndAdvance(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type);
+ }
+ // appending to WAL out of global lock for concurrent performance
+ // we should still be able to arrive at the same state even if log entries are out of order
+ appendToLog(TransactionEdit.createStarted(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type));
+ } finally {
+ this.logReadLock.unlock();
+ }
+ return tx;
+ }
+
+ private void addInProgressAndAdvance(long writePointer, long visibilityUpperBound,
+ long expiration, TransactionType type) {
+ inProgress.put(writePointer, new InProgressTx(visibilityUpperBound, expiration, type));
+ advanceWritePointer(writePointer);
+ }
+
+ private void advanceWritePointer(long writePointer) {
+ // don't move the write pointer back if we have out of order transaction log entries
+ if (writePointer > lastWritePointer) {
+ lastWritePointer = writePointer;
+ }
+ }
+
+ public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
+ txMetricsCollector.rate("canCommit");
+ Stopwatch timer = new Stopwatch().start();
+ if (inProgress.get(tx.getTransactionId()) == null) {
+ // invalid transaction, either this has timed out and moved to invalid, or something else is wrong.
+ if (invalid.contains(tx.getTransactionId())) {
+ throw new TransactionNotInProgressException(
+ String.format("canCommit() is called for transaction %d that is not in progress (it is known to be invalid)",
+ tx.getTransactionId()));
+ } else {
+ throw new TransactionNotInProgressException(
+ String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId()));
+ }
+ }
+
+ Set<ChangeId> set = Sets.newHashSetWithExpectedSize(changeIds.size());
+ for (byte[] change : changeIds) {
+ set.add(new ChangeId(change));
+ }
+
+ if (hasConflicts(tx, set)) {
+ return false;
+ }
+ // guard against changes to the transaction log while processing
+ this.logReadLock.lock();
+ try {
+ synchronized (this) {
+ ensureAvailable();
+ addCommittingChangeSet(tx.getTransactionId(), set);
+ }
+ appendToLog(TransactionEdit.createCommitting(tx.getTransactionId(), set));
+ } finally {
+ this.logReadLock.unlock();
+ }
+ txMetricsCollector.histogram("canCommit.latency", (int) timer.elapsedMillis());
+ return true;
+ }
+
+ private void addCommittingChangeSet(long writePointer, Set<ChangeId> changes) {
+ committingChangeSets.put(writePointer, changes);
+ }
+
+ public boolean commit(Transaction tx) throws TransactionNotInProgressException {
+ txMetricsCollector.rate("commit");
+ Stopwatch timer = new Stopwatch().start();
+ Set<ChangeId> changeSet = null;
+ boolean addToCommitted = true;
+ long commitPointer;
+ // guard against changes to the transaction log while processing
+ this.logReadLock.lock();
+ try {
+ synchronized (this) {
+ ensureAvailable();
+ // we record commits at the first not-yet assigned transaction id to simplify clearing out change sets that
+ // are no longer visible by any in-progress transactions
+ commitPointer = lastWritePointer + 1;
+ if (inProgress.get(tx.getTransactionId()) == null) {
+ // invalid transaction, either this has timed out and moved to invalid, or something else is wrong.
+ if (invalid.contains(tx.getTransactionId())) {
+ throw new TransactionNotInProgressException(
+ String.format("canCommit() is called for transaction %d that is not in progress " +
+ "(it is known to be invalid)", tx.getTransactionId()));
+ } else {
+ throw new TransactionNotInProgressException(
+ String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId()));
+ }
+ }
+
+ // these should be atomic
+ // NOTE: whether we succeed or not we don't need to keep changes in committing state: same tx cannot
+ // be attempted to commit twice
+ changeSet = committingChangeSets.remove(tx.getTransactionId());
+
+ if (changeSet != null) {
+ // double-checking if there are conflicts: someone may have committed since canCommit check
+ if (hasConflicts(tx, changeSet)) {
+ return false;
+ }
+ } else {
+ // no changes
+ addToCommitted = false;
+ }
+ doCommit(tx.getTransactionId(), tx.getWritePointer(), changeSet, commitPointer, addToCommitted);
+ }
+ appendToLog(TransactionEdit.createCommitted(tx.getTransactionId(), changeSet, commitPointer, addToCommitted));
+ } finally {
+ this.logReadLock.unlock();
+ }
+ txMetricsCollector.histogram("commit.latency", (int) timer.elapsedMillis());
+ return true;
+ }
+
+ private void doCommit(long transactionId, long writePointer, Set<ChangeId> changes, long commitPointer,
+ boolean addToCommitted) {
+ // In case this method is called when loading a previous WAL, we need to remove the tx from these sets
+ committingChangeSets.remove(transactionId);
+ if (addToCommitted && !changes.isEmpty()) {
+ // No need to add empty changes to the committed change sets, they will never trigger any conflict
+
+ // Record the committed change set with the next writePointer as the commit time.
+ // NOTE: we use current next writePointer as key for the map, hence we may have multiple txs changesets to be
+ // stored under one key
+ Set<ChangeId> changeIds = committedChangeSets.get(commitPointer);
+ if (changeIds != null) {
+ // NOTE: we modify the new set to prevent concurrent modification exception, as other threads (e.g. in
+ // canCommit) use it unguarded
+ changes.addAll(changeIds);
+ }
+ committedChangeSets.put(commitPointer, changes);
+ }
+ // remove from in-progress set, so that it does not get excluded in the future
+ InProgressTx previous = inProgress.remove(transactionId);
+ if (previous == null) {
+ // tx was not in progress! perhaps it timed out and is invalid? try to remove it there.
+ if (invalid.rem(transactionId)) {
+ invalidArray = invalid.toLongArray();
+ LOG.info("Tx invalid list: removed committed tx {}", transactionId);
+ }
+ }
+ // moving read pointer
+ moveReadPointerIfNeeded(writePointer);
+
+ // All committed change sets that are smaller than the earliest started transaction can be removed.
+ // here we ignore transactions that have no timeout, they are long-running and don't participate in
+ // conflict detection.
+ // TODO: for efficiency, can we do this once per-log in replayLogs instead of once per edit?
+ committedChangeSets.headMap(TxUtils.getFirstShortInProgress(inProgress)).clear();
+ }
+
+ public void abort(Transaction tx) {
+ // guard against changes to the transaction log while processing
+ txMetricsCollector.rate("abort");
+ Stopwatch timer = new Stopwatch().start();
+ this.logReadLock.lock();
+ try {
+ synchronized (this) {
+ ensureAvailable();
+ doAbort(tx.getTransactionId(), tx.getCheckpointWritePointers(), tx.getType());
+ }
+ appendToLog(TransactionEdit.createAborted(tx.getTransactionId(), tx.getType(), tx.getCheckpointWritePointers()));
+ txMetricsCollector.histogram("abort.latency", (int) timer.elapsedMillis());
+ } finally {
+ this.logReadLock.unlock();
+ }
+ }
+
+ private void doAbort(long writePointer, long[] checkpointWritePointers, TransactionType type) {
+ committingChangeSets.remove(writePointer);
+
+ if (type == TransactionType.LONG) {
+ // Long running transactions cannot be aborted as their change sets are not saved,
+ // and hence the changes cannot be rolled back. Invalidate the long running transaction instead.
+ doInvalidate(writePointer);
+ return;
+ }
+
+ // makes tx visible (assumes that all operations were rolled back)
+ // remove from in-progress set, so that it does not get excluded in the future
+ InProgressTx removed = inProgress.remove(writePointer);
+ if (removed == null) {
+ // tx was not in progress! perhaps it timed out and is invalid? try to remove it there.
+ if (invalid.rem(writePointer)) {
+ // remove any invalidated checkpoint pointers
+ // this will only be present if the parent write pointer was also invalidated
+ if (checkpointWritePointers != null) {
+ for (int i = 0; i < checkpointWritePointers.length; i++) {
+ invalid.rem(checkpointWritePointers[i]);
+ }
+ }
+ invalidArray = invalid.toLongArray();
+ LOG.info("Tx invalid list: removed aborted tx {}", writePointer);
+ // removed a tx from excludes: must move read pointer
+ moveReadPointerIfNeeded(writePointer);
+ }
+ } else {
+ // removed a tx from excludes: must move read pointer
+ moveReadPointerIfNeeded(writePointer);
+ }
+ }
+
+ public boolean invalidate(long tx) {
+ // guard against changes to the transaction log while processing
+ txMetricsCollector.rate("invalidate");
+ Stopwatch timer = new Stopwatch().start();
+ this.logReadLock.lock();
+ try {
+ boolean success;
+ synchronized (this) {
+ ensureAvailable();
+ success = doInvalidate(tx);
+ }
+ appendToLog(TransactionEdit.createInvalid(tx));
+ txMetricsCollector.histogram("invalidate.latency", (int) timer.elapsedMillis());
+ return success;
+ } finally {
+ this.logReadLock.unlock();
+ }
+ }
+
+ private boolean doInvalidate(long writePointer) {
+ Set<ChangeId> previousChangeSet = committingChangeSets.remove(writePointer);
+ // remove from in-progress set, so that it does not get excluded in the future
+ InProgressTx previous = inProgress.remove(writePointer);
+ // This check is to prevent from invalidating committed transactions
+ if (previous != null || previousChangeSet != null) {
+ // add tx to invalids
+ invalid.add(writePointer);
+ if (previous == null) {
+ LOG.debug("Invalidating tx {} in committing change sets but not in-progress", writePointer);
+ } else {
+ // invalidate any checkpoint write pointers
+ LongArrayList childWritePointers = previous.getCheckpointWritePointers();
+ if (childWritePointers != null) {
+ for (int i = 0; i < childWritePointers.size(); i++) {
+ invalid.add(childWritePointers.get(i));
+ }
+ }
+ }
+ LOG.info("Tx invalid list: added tx {} because of invalidate", writePointer);
+ // todo: find a more efficient way to keep this sorted. Could it just be an array?
+ Collections.sort(invalid);
+ invalidArray = invalid.toLongArray();
+ if (previous != null && !previous.isLongRunning()) {
+ // tx was short-running: must move read pointer
+ moveReadPointerIfNeeded(writePointer);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Removes the given transaction ids from the invalid list.
+ * @param invalidTxIds transaction ids
+ * @return true if invalid list got changed, false otherwise
+ */
+ public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
+ // guard against changes to the transaction log while processing
+ txMetricsCollector.rate("truncateInvalidTx");
+ Stopwatch timer = new Stopwatch().start();
+ this.logReadLock.lock();
+ try {
+ boolean success;
+ synchronized (this) {
+ ensureAvailable();
+ success = doTruncateInvalidTx(invalidTxIds);
+ }
+ appendToLog(TransactionEdit.createTruncateInvalidTx(invalidTxIds));
+ txMetricsCollector.histogram("truncateInvalidTx.latency", (int) timer.elapsedMillis());
+ return success;
+ } finally {
+ this.logReadLock.unlock();
+ }
+ }
+
+ private boolean doTruncateInvalidTx(Set<Long> invalidTxIds) {
+ LOG.info("Removing tx ids {} from invalid list", invalidTxIds);
+ boolean success = invalid.removeAll(invalidTxIds);
+ if (success) {
+ invalidArray = invalid.toLongArray();
+ }
+ return success;
+ }
+
+ /**
+ * Removes all transaction ids started before the given time from invalid list.
+ * @param time time in milliseconds
+ * @return true if invalid list got changed, false otherwise
+ * @throws InvalidTruncateTimeException if there are any in-progress transactions started before given time
+ */
+ public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
+ // guard against changes to the transaction log while processing
+ txMetricsCollector.rate("truncateInvalidTxBefore");
+ Stopwatch timer = new Stopwatch().start();
+ this.logReadLock.lock();
+ try {
+ boolean success;
+ synchronized (this) {
+ ensureAvailable();
+ success = doTruncateInvalidTxBefore(time);
+ }
+ appendToLog(TransactionEdit.createTruncateInvalidTxBefore(time));
+ txMetricsCollector.histogram("truncateInvalidTxBefore.latency", (int) timer.elapsedMillis());
+ return success;
+ } finally {
+ this.logReadLock.unlock();
+ }
+ }
+
+ private boolean doTruncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
+ LOG.info("Removing tx ids before {} from invalid list", time);
+ long truncateWp = time * TxConstants.MAX_TX_PER_MS;
+ // Check if there any in-progress transactions started earlier than truncate time
+ if (inProgress.lowerKey(truncateWp) != null) {
+ throw new InvalidTruncateTimeException("Transactions started earlier than " + time + " are in-progress");
+ }
+
+ // Find all invalid transactions earlier than truncateWp
+ Set<Long> toTruncate = Sets.newHashSet();
+ for (long wp : invalid) {
+ // invalid list is sorted, hence can stop as soon as we reach a wp >= truncateWp
+ if (wp >= truncateWp) {
+ break;
+ }
+ toTruncate.add(wp);
+ }
+ return doTruncateInvalidTx(toTruncate);
+ }
+
+ public Transaction checkpoint(Transaction originalTx) throws TransactionNotInProgressException {
+ txMetricsCollector.rate("checkpoint");
+ Stopwatch timer = new Stopwatch().start();
+
+ Transaction checkpointedTx = null;
+ long txId = originalTx.getTransactionId();
+ long newWritePointer = 0;
+ // guard against changes to the transaction log while processing
+ this.logReadLock.lock();
+ try {
+ synchronized (this) {
+ ensureAvailable();
+ // check that the parent tx is in progress
+ InProgressTx parentTx = inProgress.get(txId);
+ if (parentTx == null) {
+ if (invalid.contains(txId)) {
+ throw new TransactionNotInProgressException(
+ String.format("Transaction %d is not in progress because it was invalidated", txId));
+ } else {
+ throw new TransactionNotInProgressException(
+ String.format("Transaction %d is not in progress", txId));
+ }
+ }
+ newWritePointer = getNextWritePointer();
+ doCheckpoint(newWritePointer, txId);
+ // create a new transaction with the same read snapshot, plus the additional checkpoint write pointer
+ // the same read snapshot is maintained to
+ checkpointedTx = new Transaction(originalTx, newWritePointer,
+ parentTx.getCheckpointWritePointers().toLongArray());
+ }
+ // appending to WAL out of global lock for concurrent performance
+ // we should still be able to arrive at the same state even if log entries are out of order
+ appendToLog(TransactionEdit.createCheckpoint(newWritePointer, txId));
+ } finally {
+ this.logReadLock.unlock();
+ }
+ txMetricsCollector.histogram("checkpoint.latency", (int) timer.elapsedMillis());
+
+ return checkpointedTx;
+ }
+
+ private void doCheckpoint(long newWritePointer, long parentWritePointer) {
+ InProgressTx existingTx = inProgress.get(parentWritePointer);
+ existingTx.addCheckpointWritePointer(newWritePointer);
+ advanceWritePointer(newWritePointer);
+ }
+
+ // hack for exposing important metric
+ public int getExcludedListSize() {
+ return invalid.size() + inProgress.size();
+ }
+
+ /**
+ * @return the size of invalid list
+ */
+ public int getInvalidSize() {
+ return this.invalid.size();
+ }
+
+ int getCommittedSize() {
+ return this.committedChangeSets.size();
+ }
+
+ private boolean hasConflicts(Transaction tx, Set<ChangeId> changeIds) {
+ if (changeIds.isEmpty()) {
+ return false;
+ }
+
+ for (Map.Entry<Long, Set<ChangeId>> changeSet : committedChangeSets.entrySet()) {
+ // If commit time is greater than tx read-pointer,
+ // basically not visible but committed means "tx committed after given tx was started"
+ if (changeSet.getKey() > tx.getTransactionId()) {
+ if (overlap(changeSet.getValue(), changeIds)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean overlap(Set<ChangeId> a, Set<ChangeId> b) {
+ // iterate over the smaller set, and check for every element in the other set
+ if (a.size() > b.size()) {
+ for (ChangeId change : b) {
+ if (a.contains(change)) {
+ return true;
+ }
+ }
+ } else {
+ for (ChangeId change : a) {
+ if (b.contains(change)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void moveReadPointerIfNeeded(long committedWritePointer) {
+ if (committedWritePointer > readPointer) {
+ readPointer = committedWritePointer;
+ }
+ }
+
+ /**
+ * Creates a new Transaction. This method only get called from start transaction, which is already
+ * synchronized.
+ */
+ private Transaction createTransaction(long writePointer, TransactionType type) {
+ // For holding the first in progress short transaction Id (with timeout >= 0).
+ long firstShortTx = Transaction.NO_TX_IN_PROGRESS;
+ LongArrayList inProgressIds = new LongArrayList(inProgress.size());
+ for (Map.Entry<Long, InProgressTx> entry : inProgress.entrySet()) {
+ long txId = entry.getKey();
+ inProgressIds.add(txId);
+ // add any checkpointed write pointers to the in-progress list
+ LongArrayList childIds = entry.getValue().getCheckpointWritePointers();
+ if (childIds != null) {
+ for (int i = 0; i < childIds.size(); i++) {
+ inProgressIds.add(childIds.get(i));
+ }
+ }
+ if (firstShortTx == Transaction.NO_TX_IN_PROGRESS && !entry.getValue().isLongRunning()) {
+ firstShortTx = txId;
+ }
+ }
+
+ return new Transaction(readPointer, writePointer, invalidArray, inProgressIds.toLongArray(), firstShortTx, type);
+ }
+
+ private void appendToLog(TransactionEdit edit) {
+ try {
+ Stopwatch timer = new Stopwatch().start();
+ currentLog.append(edit);
+ txMetricsCollector.rate("wal.append.count");
+ txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsedMillis());
+ } catch (IOException ioe) {
+ abortService("Error appending to transaction log", ioe);
+ }
+ }
+
+ private void appendToLog(List<TransactionEdit> edits) {
+ try {
+ Stopwatch timer = new Stopwatch().start();
+ currentLog.append(edits);
+ txMetricsCollector.rate("wal.append.count", edits.size());
+ txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsedMillis());
+ } catch (IOException ioe) {
+ abortService("Error appending to transaction log", ioe);
+ }
+ }
+
+ /**
+ * Called from the tx service every 10 seconds.
+ * This hack is needed because current metrics system is not flexible when it comes to adding new metrics.
+ */
+ public void logStatistics() {
+ LOG.info("Transaction Statistics: write pointer = " + lastWritePointer +
+ ", invalid = " + invalid.size() +
+ ", in progress = " + inProgress.size() +
+ ", committing = " + committingChangeSets.size() +
+ ", committed = " + committedChangeSets.size());
+ }
+
+ private abstract static class DaemonThreadExecutor extends Thread {
+ private AtomicBoolean stopped = new AtomicBoolean(false);
+
+ public DaemonThreadExecutor(String name) {
+ super(name);
+ setDaemon(true);
+ }
+
+ public void run() {
+ try {
+ while (!isInterrupted() && !stopped.get()) {
+ doRun();
+ synchronized (stopped) {
+ stopped.wait(getSleepMillis());
+ }
+ }
+ } catch (InterruptedException ie) {
+ LOG.info("Interrupted thread " + getName());
+ }
+ // perform any final cleanup
+ onShutdown();
+ LOG.info("Exiting thread " + getName());
+ }
+
+ public abstract void doRun();
+
+ protected abstract long getSleepMillis();
+
+ protected void onShutdown() {
+ }
+
+ public void shutdown() {
+ if (stopped.compareAndSet(false, true)) {
+ synchronized (stopped) {
+ stopped.notifyAll();
+ }
+ }
+ }
+ }
+
+ /**
+ * Represents some of the info on in-progress tx
+ */
+ public static final class InProgressTx {
+ /** the oldest in progress tx at the time of this tx start */
+ private final long visibilityUpperBound;
+ private final long expiration;
+ private final TransactionType type;
+ private LongArrayList checkpointWritePointers = new LongArrayList();
+
+ public InProgressTx(long visibilityUpperBound, long expiration, TransactionType type) {
+ this(visibilityUpperBound, expiration, type, new LongArrayList());
+ }
+
+ public InProgressTx(long visibilityUpperBound, long expiration, TransactionType type,
+ LongArrayList checkpointWritePointers) {
+ this.visibilityUpperBound = visibilityUpperBound;
+ this.expiration = expiration;
+ this.type = type;
+ this.checkpointWritePointers = checkpointWritePointers;
+ }
+
+ // For backwards compatibility when long running txns were represented with -1 expiration
+ @Deprecated
+ public InProgressTx(long visibilityUpperBound, long expiration) {
+ this(visibilityUpperBound, expiration, null);
+ }
+
+ public long getVisibilityUpperBound() {
+ return visibilityUpperBound;
+ }
+
+ public long getExpiration() {
+ return expiration;
+ }
+
+ @Nullable
+ public TransactionType getType() {
+ return type;
+ }
+
+ public boolean isLongRunning() {
+ if (type == null) {
+ // for backwards compatibility when long running txns were represented with -1 expiration
+ return expiration == -1;
+ }
+ return type == TransactionType.LONG;
+ }
+
+ public void addCheckpointWritePointer(long checkpointWritePointer) {
+ checkpointWritePointers.add(checkpointWritePointer);
+ }
+
+ public LongArrayList getCheckpointWritePointers() {
+ return checkpointWritePointers;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || !(o instanceof InProgressTx)) {
+ return false;
+ }
+
+ if (this == o) {
+ return true;
+ }
+
+ InProgressTx other = (InProgressTx) o;
+ return Objects.equal(visibilityUpperBound, other.getVisibilityUpperBound()) &&
+ Objects.equal(expiration, other.getExpiration()) &&
+ Objects.equal(type, other.type) &&
+ Objects.equal(checkpointWritePointers, other.checkpointWritePointers);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(visibilityUpperBound, expiration, type, checkpointWritePointers);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("visibilityUpperBound", visibilityUpperBound)
+ .add("expiration", expiration)
+ .add("type", type)
+ .add("checkpointWritePointers", checkpointWritePointers)
+ .toString();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionServiceMain.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionServiceMain.java b/tephra-core/src/main/java/org/apache/tephra/TransactionServiceMain.java
new file mode 100644
index 0000000..0a9dd1d
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionServiceMain.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.distributed.TransactionService;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.tephra.util.ConfigurationFactory;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * Driver class to start and stop tx in distributed mode.
+ */
+public class TransactionServiceMain {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionServiceMain.class);
+
+ private Configuration conf;
+ private TransactionService txService;
+
+ public static void main(String args[]) throws Exception {
+ TransactionServiceMain instance = new TransactionServiceMain();
+ instance.doMain(args);
+ }
+
+ public TransactionServiceMain() {
+ this(null);
+ }
+
+ public TransactionServiceMain(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * The main method. It simply call methods in the same sequence
+ * as if the program is started by jsvc.
+ */
+ public void doMain(final String[] args) throws Exception {
+ final CountDownLatch shutdownLatch = new CountDownLatch(1);
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ try {
+ TransactionServiceMain.this.stop();
+ } finally {
+ try {
+ TransactionServiceMain.this.destroy();
+ } finally {
+ shutdownLatch.countDown();
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("Exception when shutting down: " + t.getMessage(), t);
+ }
+ }
+ });
+ init(args);
+ start();
+
+ shutdownLatch.await();
+ }
+
+ /**
+ * Invoked by jsvc to initialize the program.
+ */
+ public void init(String[] args) {
+ if (conf == null) {
+ conf = new ConfigurationFactory().get();
+ }
+ }
+
+ /**
+ * Invoked by jsvc to start the program.
+ */
+ public void start() throws Exception {
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ new TransactionModules().getDistributedModules(),
+ new TransactionClientModule()
+ );
+
+ ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
+ zkClientService.startAndWait();
+
+ // start a tx server
+ txService = injector.getInstance(TransactionService.class);
+ try {
+ LOG.info("Starting {}", getClass().getSimpleName());
+ txService.startAndWait();
+ } catch (Exception e) {
+ System.err.println("Failed to start service: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Invoked by jsvc to stop the program.
+ */
+ public void stop() {
+ LOG.info("Stopping {}", getClass().getSimpleName());
+ if (txService == null) {
+ return;
+ }
+ try {
+ if (txService.isRunning()) {
+ txService.stopAndWait();
+ }
+ } catch (Throwable e) {
+ LOG.error("Failed to shutdown transaction service.", e);
+ // because shutdown hooks execute concurrently, the logger may be closed already: thus also print it.
+ System.err.println("Failed to shutdown transaction service: " + e.getMessage());
+ e.printStackTrace(System.err);
+ }
+ }
+
+ /**
+ * Invoked by jsvc for resource cleanup.
+ */
+ public void destroy() { }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionSystemClient.java b/tephra-core/src/main/java/org/apache/tephra/TransactionSystemClient.java
new file mode 100644
index 0000000..9009f84
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionSystemClient.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * Client talking to transaction system.
+ * See also {@link TransactionAware}.
+ * todo: explain Omid.
+ */
+public interface TransactionSystemClient {
+ /**
+ * Starts new short transaction.
+ * @return instance of {@link Transaction}
+ */
+ // TODO: "short" and "long" are very misleading names. Use transaction attributes like "detect conflicts or not", etc.
+ Transaction startShort();
+
+ /**
+ * Starts new short transaction.
+ * @param timeout the timeout for the transaction
+ * @return instance of {@link Transaction}
+ */
+ Transaction startShort(int timeout);
+
+ /**
+ * Starts new long transaction.
+ * @return instance of {@link Transaction}
+ */
+ Transaction startLong();
+
+ // this pre-commit detects conflicts with other transactions committed so far
+ // NOTE: the changes set should not change after this operation, this may help us do some extra optimizations
+ // NOTE: there should be time constraint on how long does it take to commit changes by the client after this operation
+ // is submitted so that we can cleanup related resources
+ // NOTE: as of now you can call this method multiple times, each time the changeSet of tx will be updated. Not sure
+ // if we can call it a feature or a side-affect of implementation. It makes more sense to append changeset, but
+ // before we really need it we don't do it because it will slow down tx manager throughput.
+
+ /**
+ * Checks if transaction with the set of changes can be committed. E.g. it can check conflicts with other changes and
+ * refuse commit if there are conflicts. It is assumed that no other changes will be done in between this method call
+ * and {@link #commit(Transaction)} which may check conflicts again to avoid races.
+ * <p/>
+ * Since we do conflict detection at commit time as well, this may seem redundant. The idea is to check for conflicts
+ * before we persist changes to avoid rollback in case of conflicts as much as possible.
+ * NOTE: in some situations we may want to skip this step to save on RPC with a risk of many rollback ops. So by
+ * default we take safe path.
+ *
+ * @param tx transaction to verify
+ * @param changeIds ids of changes made by transaction
+ * @return true if transaction can be committed otherwise false
+ */
+ boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException;
+
+ /**
+ * Makes transaction visible. It will again check conflicts of changes submitted previously with
+ * {@link #canCommit(Transaction, java.util.Collection)}
+ * @param tx transaction to make visible.
+ * @return true if transaction can be committed otherwise false
+ */
+ boolean commit(Transaction tx) throws TransactionNotInProgressException;
+
+ /**
+ * Makes transaction visible. You should call it only when all changes of this tx are undone.
+ * NOTE: it will not throw {@link TransactionNotInProgressException} if transaction has timed out.
+ * @param tx transaction to make visible.
+ */
+ void abort(Transaction tx);
+
+ /**
+ * Makes transaction invalid. You should call it if not all changes of this tx could be undone.
+ * NOTE: it will not throw {@link TransactionNotInProgressException} if transaction has timed out.
+ * @param tx transaction id to invalidate.
+ * @return true if transaction has been successfully invalidated
+ */
+ boolean invalidate(long tx);
+
+ /**
+ * Performs a checkpoint operation on the current transaction, returning a new Transaction instance with the
+ * updated state. A checkpoint operation assigns a new write pointer for the current transaction.
+ * @param tx the current transaction to checkpoint
+ * @return an updated transaction instance with the new write pointer
+ */
+ Transaction checkpoint(Transaction tx) throws TransactionNotInProgressException;
+
+ /**
+ * Retrieves the state of the transaction manager and send it as a stream. The snapshot will not be persisted.
+ * @return an input stream containing an encoded snapshot of the transaction manager
+ */
+ InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException;
+
+ /**
+ * Return the status of the transaction Manager
+ * @return a String which denotes the status of txManager
+ */
+ String status();
+
+ /**
+ * Resets the state of the transaction manager.
+ */
+ void resetState();
+
+ /**
+ * Removes the given transaction ids from the invalid list.
+ * @param invalidTxIds transaction ids
+ * @return true if invalid list got changed, false otherwise
+ */
+ boolean truncateInvalidTx(Set<Long> invalidTxIds);
+
+ /**
+ * Removes all transaction ids started before the given time from invalid list.
+ * @param time time in milliseconds
+ * @return true if invalid list got changed, false otherwise
+ * @throws InvalidTruncateTimeException if there are any in-progress transactions started before given time
+ */
+ boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException;
+
+ /**
+ * @return the size of invalid list
+ */
+ int getInvalidSize();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TxConstants.java b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
new file mode 100644
index 0000000..7f7fc8c
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import org.apache.tephra.snapshot.DefaultSnapshotCodec;
+import org.apache.tephra.snapshot.SnapshotCodecV2;
+import org.apache.tephra.snapshot.SnapshotCodecV3;
+import org.apache.tephra.snapshot.SnapshotCodecV4;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Transaction system constants
+ */
+public class TxConstants {
+ /**
+ * Defines what level of conflict detection should be used for transactions. {@code ROW} means that only the
+ * table name and row key for each change will be used to determine if the transaction change sets conflict.
+ * {@code COLUMN} means that the table name, row key, column family, and column qualifier will all be used to
+ * identify write conflicts. {@code NONE} means that no conflict detection will be performed, but transaction
+ * clients will still track the current transaction's change set to rollback any persisted changes in the event of
+ * a failure. This should only be used where writes to the same coordinate should never conflict, such as
+ * append-only data. The default value used by {@code TransactionAwareHTable} implementations is {@code COLUMN}.
+ *
+ * <p>
+ * <strong>Note: for a given table, all clients must use the same conflict detection setting!</strong>
+ * Otherwise conflicts will not be flagged correctly.
+ * </p>
+ */
+ public enum ConflictDetection {
+ ROW,
+ COLUMN,
+ NONE
+ }
+
+ /**
+ * Property set for {@code org.apache.hadoop.hbase.HColumnDescriptor} to configure time-to-live on data within
+ * the column family. The value given is in milliseconds. Once a cell's data has surpassed the given value in age,
+ * the cell's data will no longer be visible and may be garbage collected.
+ */
+ public static final String PROPERTY_TTL = "dataset.table.ttl";
+
+ /**
+ * This is how many tx we allow per millisecond, if you care about the system for 100 years:
+ * Long.MAX_VALUE / (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(365 * 100)) =
+ * (as of Feb 20, 2014) 2,028,653. It is safe and convenient to use 1,000,000 as multiplier:
+ * <ul>
+ * <li>
+ * we hardly can do more than 1 billion txs per second
+ * </li>
+ * <li>
+ * long value will not overflow for 200 years
+ * </li>
+ * <li>
+ * makes reading & debugging easier if multiplier is 10^n
+ * </li>
+ * </ul>
+ */
+ public static final long MAX_TX_PER_MS = 1000000;
+
+ /**
+ * Since HBase {@code Delete} operations cannot be undone at the same timestamp, "deleted" data is instead
+ * overwritten with an empty {@code byte[]} to flag it as removed. Cells with empty values will be filtered out
+ * of the results for read operations. If cells with empty values should be included in results (meaning data
+ * cannot be transactionally deleted), then set this configuration property to true.
+ */
+ public static final String ALLOW_EMPTY_VALUES_KEY = "data.tx.allow.empty.values";
+ public static final boolean ALLOW_EMPTY_VALUES_DEFAULT = false;
+
+ /**
+ * Key used to set the serialized transaction as an attribute on Get and Scan operations.
+ */
+ public static final String TX_OPERATION_ATTRIBUTE_KEY = "cask.tx";
+ /**
+ * Key used to flag a delete operation as part of a transaction rollback. This is used so that the
+ * {@code TransactionProcessor} coprocessor loaded on a table can differentiate between deletes issued
+ * as part of a normal client operation versus those performed when rolling back a transaction.
+ */
+ public static final String TX_ROLLBACK_ATTRIBUTE_KEY = "cask.tx.rollback";
+
+ /**
+ * Column qualifier used for a special delete marker tombstone, which identifies an entire column family as deleted.
+ */
+ public static final byte[] FAMILY_DELETE_QUALIFIER = new byte[0];
+
+ // Constants for monitoring status
+ public static final String STATUS_OK = "OK";
+ public static final String STATUS_NOTOK = "NOTOK";
+
+ /**
+ * Indicates whether data written before Tephra was enabled on a table should be readable. Reading non-transactional
+ * data can lead to slight performance penalty. Hence it is disabled by default.
+ * @see <a href="https://issues.cask.co/browse/TEPHRA-89">TEPHRA-89</a>
+ */
+ public static final String READ_NON_TX_DATA = "data.tx.read.pre.existing";
+ public static final boolean DEFAULT_READ_NON_TX_DATA = false;
+
+ /**
+ * TransactionManager configuration.
+ */
+ public static final class Manager {
+ // TransactionManager configuration
+ public static final String CFG_DO_PERSIST = "tx.persist";
+ /** Directory in HDFS used for transaction snapshot and log storage. */
+ public static final String CFG_TX_SNAPSHOT_DIR = "data.tx.snapshot.dir";
+ /** Directory on the local filesystem used for transaction snapshot and log storage. */
+ public static final String CFG_TX_SNAPSHOT_LOCAL_DIR = "data.tx.snapshot.local.dir";
+ /** How often to clean up timed out transactions, in seconds, or 0 for no cleanup. */
+ public static final String CFG_TX_CLEANUP_INTERVAL = "data.tx.cleanup.interval";
+ /** The user id to access HDFS if not running in secure HDFS. */
+ public static final String CFG_TX_HDFS_USER = "data.tx.hdfs.user";
+ /** Default value for how often to check in-progress transactions for expiration, in seconds. */
+ public static final int DEFAULT_TX_CLEANUP_INTERVAL = 10;
+ /**
+ * The timeout for a transaction, in seconds. If the transaction is not finished in that time,
+ * it is marked invalid.
+ */
+ public static final String CFG_TX_TIMEOUT = "data.tx.timeout";
+ /** Default value for transaction timeout, in seconds. */
+ public static final int DEFAULT_TX_TIMEOUT = 30;
+ /**
+ * The timeout for a long running transaction, in seconds. If the transaction is not finished in that time,
+ * it is marked invalid.
+ */
+ public static final String CFG_TX_LONG_TIMEOUT = "data.tx.long.timeout";
+ /** Default value for long running transaction timeout, in seconds. */
+ public static final int DEFAULT_TX_LONG_TIMEOUT = (int) TimeUnit.DAYS.toSeconds(1);
+ /** The frequency (in seconds) to perform periodic snapshots, or 0 for no periodic snapshots. */
+ public static final String CFG_TX_SNAPSHOT_INTERVAL = "data.tx.snapshot.interval";
+ /** Default value for frequency of periodic snapshots of transaction state. */
+ public static final long DEFAULT_TX_SNAPSHOT_INTERVAL = 300;
+ /** Number of most recent transaction snapshots to retain. */
+ public static final String CFG_TX_SNAPSHOT_RETAIN = "data.tx.snapshot.retain";
+ /** Default value for number of most recent snapshots to retain. */
+ public static final int DEFAULT_TX_SNAPSHOT_RETAIN = 10;
+ }
+
+ /**
+ * TransactionService configuration.
+ */
+ public static final class Service {
+
+ /** for the zookeeper quorum string for leader election for tx server. */
+ public static final String CFG_DATA_TX_ZOOKEEPER_QUORUM
+ = "data.tx.zookeeper.quorum";
+
+ /** for the name used to announce service availability to discovery service */
+ public static final String CFG_DATA_TX_DISCOVERY_SERVICE_NAME
+ = "data.tx.discovery.service.name";
+
+ /** for the port of the tx server. */
+ public static final String CFG_DATA_TX_BIND_PORT
+ = "data.tx.bind.port";
+
+ /** for the address (hostname) of the tx server. */
+ public static final String CFG_DATA_TX_BIND_ADDRESS
+ = "data.tx.bind.address";
+
+ /** the number of IO threads in the tx service. */
+ public static final String CFG_DATA_TX_SERVER_IO_THREADS
+ = "data.tx.server.io.threads";
+
+ /** the number of handler threads in the tx service. */
+ public static final String CFG_DATA_TX_SERVER_THREADS
+ = "data.tx.server.threads";
+
+ public static final String CFG_DATA_TX_THRIFT_MAX_READ_BUFFER
+ = "data.tx.thrift.max.read.buffer";
+
+ public static final String DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME
+ = "transaction";
+
+ /** default tx service port. */
+ public static final int DEFAULT_DATA_TX_BIND_PORT
+ = 15165;
+
+ /** default tx service address. */
+ public static final String DEFAULT_DATA_TX_BIND_ADDRESS
+ = "0.0.0.0";
+
+ /** default number of handler IO threads in tx service. */
+ public static final int DEFAULT_DATA_TX_SERVER_IO_THREADS
+ = 2;
+
+ /** default number of handler threads in tx service. */
+ public static final int DEFAULT_DATA_TX_SERVER_THREADS
+ = 20;
+
+ /** default thrift max read buffer size */
+ public static final int DEFAULT_DATA_TX_THRIFT_MAX_READ_BUFFER
+ = 16 * 1024 * 1024;
+
+ // Configuration key names and defaults used by tx client.
+
+ /** to specify the tx client socket timeout in ms. */
+ public static final String CFG_DATA_TX_CLIENT_TIMEOUT
+ = "data.tx.client.timeout";
+
+ /** to specify the tx client provider strategy. */
+ public static final String CFG_DATA_TX_CLIENT_PROVIDER
+ = "data.tx.client.provider";
+
+ /** to specify the number of threads for client provider "pool". */
+ public static final String CFG_DATA_TX_CLIENT_COUNT
+ = "data.tx.client.count";
+
+ /** timeout (in milliseconds) for obtaining client from client provider "pool". */
+ public static final String CFG_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS
+ = "data.tx.client.obtain.timeout";
+
+ /** to specify the retry strategy for a failed thrift call. */
+ public static final String CFG_DATA_TX_CLIENT_RETRY_STRATEGY
+ = "data.tx.client.retry.strategy";
+
+ /** to specify the number of times to retry a failed thrift call. */
+ public static final String CFG_DATA_TX_CLIENT_ATTEMPTS
+ = "data.tx.client.retry.attempts";
+
+ /** to specify the initial sleep time for retry strategy backoff. */
+ public static final String CFG_DATA_TX_CLIENT_BACKOFF_INITIAL
+ = "data.tx.client.retry.backoff.initial";
+
+ /** to specify the backoff factor for retry strategy backoff. */
+ public static final String CFG_DATA_TX_CLIENT_BACKOFF_FACTOR
+ = "data.tx.client.retry.backoff.factor";
+
+ /** to specify the sleep time limit for retry strategy backoff. */
+ public static final String CFG_DATA_TX_CLIENT_BACKOFF_LIMIT
+ = "data.tx.client.retry.backoff.limit";
+
+ /** the default tx client socket timeout in milli seconds. */
+ public static final int DEFAULT_DATA_TX_CLIENT_TIMEOUT_MS
+ = 30 * 1000;
+
+ /** default number of tx clients for client provider "pool". */
+ public static final int DEFAULT_DATA_TX_CLIENT_COUNT
+ = 50;
+
+ /** default timeout (in milliseconds) for obtaining client from client provider "pool". */
+ public static final long DEFAULT_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS
+ = TimeUnit.SECONDS.toMillis(3);
+
+ /** default tx client provider strategy. */
+ public static final String DEFAULT_DATA_TX_CLIENT_PROVIDER
+ = "pool";
+
+ /** retry strategy for thrift clients, e.g. backoff, or n-times. */
+ public static final String DEFAULT_DATA_TX_CLIENT_RETRY_STRATEGY
+ = "backoff";
+
+ /** default number of attempts for strategy n-times. */
+ public static final int DEFAULT_DATA_TX_CLIENT_ATTEMPTS
+ = 2;
+
+ /** default initial sleep is 100ms. */
+ public static final int DEFAULT_DATA_TX_CLIENT_BACKOFF_INITIAL
+ = 100;
+
+ /** default backoff factor is 4. */
+ public static final int DEFAULT_DATA_TX_CLIENT_BACKOFF_FACTOR
+ = 4;
+
+ /** default sleep limit is 30 sec. */
+ public static final int DEFAULT_DATA_TX_CLIENT_BACKOFF_LIMIT
+ = 30 * 1000;
+ }
+
+ /**
+ * Configuration properties for metrics reporting
+ */
+ public static final class Metrics {
+ /**
+ * Frequency at which metrics should be reported, in seconds.
+ */
+ public static final String REPORT_PERIOD_KEY = "data.tx.metrics.period";
+ /**
+ * Default report period for metrics, in seconds.
+ */
+ public static final int REPORT_PERIOD_DEFAULT = 60;
+ }
+
+ /**
+ * Configuration properties used by HBase
+ */
+ public static final class HBase {
+ public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+ public static final String ZK_SESSION_TIMEOUT = "zookeeper.session.timeout";
+ public static final int DEFAULT_ZK_SESSION_TIMEOUT = 180 * 1000;
+ }
+
+ /**
+ * Configuration for the TransactionDataJanitor coprocessor.
+ */
+ public static final class DataJanitor {
+ /**
+ * Whether or not the TransactionDataJanitor coprocessor should be enabled on tables.
+ * Disable for testing.
+ */
+ public static final String CFG_TX_JANITOR_ENABLE = "data.tx.janitor.enable";
+ public static final boolean DEFAULT_TX_JANITOR_ENABLE = true;
+ }
+
+ /**
+ * Configuration for the transaction snapshot persistence.
+ */
+ public static final class Persist {
+ /**
+ * The class names of all known transaction snapshot codecs.
+ */
+ public static final String CFG_TX_SNAPHOT_CODEC_CLASSES = "data.tx.snapshot.codecs";
+ public static final Class[] DEFAULT_TX_SNAPHOT_CODEC_CLASSES =
+ { DefaultSnapshotCodec.class, SnapshotCodecV2.class, SnapshotCodecV3.class, SnapshotCodecV4.class };
+ }
+
+ /**
+ * Configuration for transaction log edit entries
+ */
+ public static final class TransactionLog {
+ /**
+ * Key used to denote the number of entries appended.
+ */
+ public static final String NUM_ENTRIES_APPENDED = "count";
+ public static final String VERSION_KEY = "version";
+ public static final byte CURRENT_VERSION = 2;
+ }
+
+}
[43/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/HDFSUtil.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSUtil.java b/tephra-core/src/main/java/co/cask/tephra/persist/HDFSUtil.java
deleted file mode 100644
index a462566..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/HDFSUtil.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.persist;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.lang.reflect.Method;
-
-/**
- * Utility for handling HDFS file lease recovery. This is a copy-n-paste fork of
- * {@link org.apache.hadoop.hbase.util.FSHDFSUtils} from the latest HBase 0.94 version (as of 0.94.12),
- * which contains some additional fixes not present in our current HBase dependency version --
- * mainly checking the return value of the {@code DistributedFileSystem.recoverLease()} call to verify that
- * recovery succeeded.
- */
-public class HDFSUtil {
- private static final Logger LOG = LoggerFactory.getLogger(HDFSUtil.class);
- /**
- * Recover the lease from HDFS, retrying multiple times.
- */
- public void recoverFileLease(final FileSystem fs, final Path p,
- Configuration conf)
- throws IOException {
- // lease recovery not needed for local file system case.
- if (!(fs instanceof DistributedFileSystem)) {
- return;
- }
- recoverDFSFileLease((DistributedFileSystem) fs, p, conf);
- }
-
- /*
- * Run the dfs recover lease. recoverLease is asynchronous. It returns:
- * -false when it starts the lease recovery (i.e. lease recovery not *yet* done)
- * - true when the lease recovery has succeeded or the file is closed.
- * But, we have to be careful. Each time we call recoverLease, it starts the recover lease
- * process over from the beginning. We could put ourselves in a situation where we are
- * doing nothing but starting a recovery, interrupting it to start again, and so on.
- * The findings over in HBASE-8354 have it that the namenode will try to recover the lease
- * on the file's primary node. If all is well, it should return near immediately. But,
- * as is common, it is the very primary node that has crashed and so the namenode will be
- * stuck waiting on a socket timeout before it will ask another datanode to start the
- * recovery. It does not help if we call recoverLease in the meantime and in particular,
- * subsequent to the socket timeout, a recoverLease invocation will cause us to start
- * over from square one (possibly waiting on socket timeout against primary node). So,
- * in the below, we do the following:
- * 1. Call recoverLease.
- * 2. If it returns true, break.
- * 3. If it returns false, wait a few seconds and then call it again.
- * 4. If it returns true, break.
- * 5. If it returns false, wait for what we think the datanode socket timeout is
- * (configurable) and then try again.
- * 6. If it returns true, break.
- * 7. If it returns false, repeat starting at step 5. above.
- *
- * If HDFS-4525 is available, call it every second and we might be able to exit early.
- */
- boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
- final Configuration conf)
- throws IOException {
- LOG.info("Recovering lease on dfs file " + p);
- long startWaiting = System.currentTimeMillis();
- // Default is 15 minutes. It's huge, but the idea is that if we have a major issue, HDFS
- // usually needs 10 minutes before marking the nodes as dead. So we're putting ourselves
- // beyond that limit 'to be safe'.
- long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
- // This setting should be what the cluster dfs heartbeat is set to.
- long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 3000);
- // This should be set to how long it'll take for us to timeout against primary datanode if it
- // is dead. We set it to 61 seconds, 1 second than the default READ_TIMEOUT in HDFS, the
- // default value for DFS_CLIENT_SOCKET_TIMEOUT_KEY.
- long subsequentPause = conf.getInt("hbase.lease.recovery.dfs.timeout", 61 * 1000);
-
- Method isFileClosedMeth = null;
- // whether we need to look for isFileClosed method
- boolean findIsFileClosedMeth = true;
- boolean recovered = false;
- // We break the loop if we succeed the lease recovery, timeout, or we throw an exception.
- for (int nbAttempt = 0; !recovered; nbAttempt++) {
- recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
- if (recovered || checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) {
- break;
- }
- try {
- // On the first time through wait the short 'firstPause'.
- if (nbAttempt == 0) {
- Thread.sleep(firstPause);
- } else {
- // Cycle here until subsequentPause elapses. While spinning, check isFileClosed if
- // available (should be in hadoop 2.0.5... not in hadoop 1 though.
- long localStartWaiting = System.currentTimeMillis();
- while ((System.currentTimeMillis() - localStartWaiting) <
- subsequentPause) {
- Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
- if (findIsFileClosedMeth) {
- try {
- isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
- new Class[]{ Path.class });
- } catch (NoSuchMethodException nsme) {
- LOG.debug("isFileClosed not available");
- } finally {
- findIsFileClosedMeth = false;
- }
- }
- if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
- recovered = true;
- break;
- }
- }
- }
- } catch (InterruptedException ie) {
- InterruptedIOException iioe = new InterruptedIOException();
- iioe.initCause(ie);
- throw iioe;
- }
- }
- return recovered;
- }
-
- boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
- final int nbAttempt, final Path p, final long startWaiting) {
- if (recoveryTimeout < System.currentTimeMillis()) {
- LOG.warn("Cannot recoverLease after trying for " +
- conf.getInt("hbase.lease.recovery.timeout", 900000) +
- "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
- getLogMessageDetail(nbAttempt, p, startWaiting));
- return true;
- }
- return false;
- }
-
- /**
- * Try to recover the lease.
- * @param dfs The filesystem instance.
- * @param nbAttempt Count number of this attempt.
- * @param p Path of the file to recover.
- * @param startWaiting Timestamp of when we started attempting to recover the file lease.
- * @return True if dfs#recoverLease came by true.
- * @throws java.io.FileNotFoundException
- */
- boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
- final long startWaiting)
- throws FileNotFoundException {
- boolean recovered = false;
- try {
- recovered = dfs.recoverLease(p);
- LOG.info("recoverLease=" + recovered + ", " +
- getLogMessageDetail(nbAttempt, p, startWaiting));
- } catch (IOException e) {
- if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
- // This exception comes out instead of FNFE, fix it
- throw new FileNotFoundException("The given file wasn't found at " + p);
- } else if (e instanceof FileNotFoundException) {
- throw (FileNotFoundException) e;
- }
- LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
- }
- return recovered;
- }
-
- /**
- * @param nbAttempt Attempt number for the lease recovery.
- * @param p Path of the file to recover.
- * @param startWaiting Timestamp of when we started attempting to recover the file lease.
- * @return Detail to append to any log message around lease recovering.
- */
- private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
- return "attempt=" + nbAttempt + " on file=" + p + " after " +
- (System.currentTimeMillis() - startWaiting) + "ms";
- }
-
- /**
- * Call HDFS-4525 isFileClosed if it is available.
- * @param dfs Filesystem instance to use.
- * @param m Method instance to call.
- * @param p Path of the file to check is closed.
- * @return True if file is closed.
- */
- private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
- try {
- return (Boolean) m.invoke(dfs, p);
- } catch (SecurityException e) {
- LOG.warn("No access", e);
- } catch (Exception e) {
- LOG.warn("Failed invocation for " + p.toString(), e);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionLog.java b/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionLog.java
deleted file mode 100644
index 590aaa5..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionLog.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.metrics.MetricsCollector;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-/**
- * Reads and writes transaction logs against files in the local filesystem.
- */
-public class LocalFileTransactionLog extends AbstractTransactionLog {
- private final File logFile;
-
- /**
- * Creates a new transaction log using the given file instance.
- * @param logFile The log file to use.
- */
- public LocalFileTransactionLog(File logFile, long timestamp, MetricsCollector metricsCollector) {
- super(timestamp, metricsCollector);
- this.logFile = logFile;
- }
-
- @Override
- public String getName() {
- return logFile.getAbsolutePath();
- }
-
- @Override
- protected TransactionLogWriter createWriter() throws IOException {
- return new LogWriter(logFile);
- }
-
- @Override
- public TransactionLogReader getReader() throws IOException {
- return new LogReader(logFile);
- }
-
- private static final class LogWriter implements TransactionLogWriter {
- private final FileOutputStream fos;
- private final DataOutputStream out;
-
- public LogWriter(File logFile) throws IOException {
- this.fos = new FileOutputStream(logFile);
- this.out = new DataOutputStream(new BufferedOutputStream(fos, LocalFileTransactionStateStorage.BUFFER_SIZE));
- }
-
- @Override
- public void append(Entry entry) throws IOException {
- entry.write(out);
- }
-
- @Override
- public void commitMarker(int count) throws IOException {
- // skip for local file
- }
-
- @Override
- public void sync() throws IOException {
- out.flush();
- }
-
- @Override
- public void close() throws IOException {
- out.flush();
- out.close();
- fos.close();
- }
- }
-
- private static final class LogReader implements TransactionLogReader {
- private final FileInputStream fin;
- private final DataInputStream in;
- private Entry reuseEntry = new Entry();
-
- public LogReader(File logFile) throws IOException {
- this.fin = new FileInputStream(logFile);
- this.in = new DataInputStream(new BufferedInputStream(fin, LocalFileTransactionStateStorage.BUFFER_SIZE));
- }
-
- @Override
- public TransactionEdit next() throws IOException {
- Entry entry = new Entry();
- try {
- entry.readFields(in);
- } catch (EOFException eofe) {
- // signal end of file by returning null
- return null;
- }
- return entry.getEdit();
- }
-
- @Override
- public TransactionEdit next(TransactionEdit reuse) throws IOException {
- try {
- reuseEntry.getKey().readFields(in);
- reuse.readFields(in);
- } catch (EOFException eofe) {
- // signal end of file by returning null
- return null;
- }
- return reuse;
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- fin.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionStateStorage.java b/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionStateStorage.java
deleted file mode 100644
index 4c3539d..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/LocalFileTransactionStateStorage.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.metrics.MetricsCollector;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.io.Closeables;
-import com.google.common.io.Files;
-import com.google.common.primitives.Longs;
-import com.google.inject.Inject;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import javax.annotation.Nullable;
-
-/**
- * Persists transaction snapshots and write-ahead logs to files on the local filesystem.
- */
-public class LocalFileTransactionStateStorage extends AbstractTransactionStateStorage {
- private static final String TMP_SNAPSHOT_FILE_PREFIX = ".in-progress.";
- private static final String SNAPSHOT_FILE_PREFIX = "snapshot.";
- private static final String LOG_FILE_PREFIX = "txlog.";
- private static final Logger LOG = LoggerFactory.getLogger(LocalFileTransactionStateStorage.class);
- static final int BUFFER_SIZE = 16384;
-
- private static final FilenameFilter SNAPSHOT_FILE_FILTER = new FilenameFilter() {
- @Override
- public boolean accept(File file, String s) {
- return s.startsWith(SNAPSHOT_FILE_PREFIX);
- }
- };
-
- private final String configuredSnapshotDir;
- private final MetricsCollector metricsCollector;
- private File snapshotDir;
-
- @Inject
- public LocalFileTransactionStateStorage(Configuration conf, SnapshotCodecProvider codecProvider,
- MetricsCollector metricsCollector) {
- super(codecProvider);
- this.configuredSnapshotDir = conf.get(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR);
- this.metricsCollector = metricsCollector;
- }
-
- @Override
- protected void startUp() throws Exception {
- Preconditions.checkState(configuredSnapshotDir != null,
- "Snapshot directory is not configured. Please set " + TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR +
- " in configuration.");
- snapshotDir = new File(configuredSnapshotDir);
- }
-
- @Override
- protected void shutDown() throws Exception {
- // nothing to do
- }
-
- @Override
- public String getLocation() {
- return snapshotDir.getAbsolutePath();
- }
-
- @Override
- public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
- // save the snapshot to a temporary file
- File snapshotTmpFile = new File(snapshotDir, TMP_SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
- LOG.debug("Writing snapshot to temporary file {}", snapshotTmpFile);
- OutputStream out = Files.newOutputStreamSupplier(snapshotTmpFile).getOutput();
- boolean threw = true;
- try {
- codecProvider.encode(out, snapshot);
- threw = false;
- } finally {
- Closeables.close(out, threw);
- }
-
- // move the temporary file into place with the correct filename
- File finalFile = new File(snapshotDir, SNAPSHOT_FILE_PREFIX + snapshot.getTimestamp());
- if (!snapshotTmpFile.renameTo(finalFile)) {
- throw new IOException("Failed renaming temporary snapshot file " + snapshotTmpFile.getName() + " to " +
- finalFile.getName());
- }
-
- LOG.debug("Completed snapshot to file {}", finalFile);
- }
-
- @Override
- public TransactionSnapshot getLatestSnapshot() throws IOException {
- InputStream is = getLatestSnapshotInputStream();
- if (is == null) {
- return null;
- }
- try {
- return readSnapshotFile(is);
- } finally {
- is.close();
- }
- }
-
- @Override
- public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
- InputStream is = getLatestSnapshotInputStream();
- if (is == null) {
- return null;
- }
- try {
- return codecProvider.decodeTransactionVisibilityState(is);
- } finally {
- is.close();
- }
- }
-
- private InputStream getLatestSnapshotInputStream() throws IOException {
- File[] snapshotFiles = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER);
- TimestampedFilename mostRecent = null;
- for (File file : snapshotFiles) {
- TimestampedFilename tsFile = new TimestampedFilename(file);
- if (mostRecent == null || tsFile.compareTo(mostRecent) > 0) {
- mostRecent = tsFile;
- }
- }
-
- if (mostRecent == null) {
- LOG.info("No snapshot files found in {}", snapshotDir.getAbsolutePath());
- return null;
- }
-
- return new FileInputStream(mostRecent.getFile());
- }
-
- private TransactionSnapshot readSnapshotFile(InputStream is) throws IOException {
- return codecProvider.decode(is);
- }
-
- @Override
- public long deleteOldSnapshots(int numberToKeep) throws IOException {
- File[] snapshotFiles = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER);
- if (snapshotFiles.length == 0) {
- return -1;
- }
- TimestampedFilename[] snapshotFilenames = new TimestampedFilename[snapshotFiles.length];
- for (int i = 0; i < snapshotFiles.length; i++) {
- snapshotFilenames[i] = new TimestampedFilename(snapshotFiles[i]);
- }
- Arrays.sort(snapshotFilenames, Collections.reverseOrder());
- if (snapshotFilenames.length <= numberToKeep) {
- // nothing to delete, just return the oldest timestamp
- return snapshotFilenames[snapshotFilenames.length - 1].getTimestamp();
- }
- int toRemoveCount = snapshotFilenames.length - numberToKeep;
- TimestampedFilename[] toRemove = new TimestampedFilename[toRemoveCount];
- System.arraycopy(snapshotFilenames, numberToKeep, toRemove, 0, toRemoveCount);
- int removedCnt = 0;
- for (int i = 0; i < toRemove.length; i++) {
- File currentFile = toRemove[i].getFile();
- LOG.debug("Removing old snapshot file {}", currentFile.getAbsolutePath());
- if (!toRemove[i].getFile().delete()) {
- LOG.error("Failed deleting snapshot file {}", currentFile.getAbsolutePath());
- } else {
- removedCnt++;
- }
- }
- long oldestTimestamp = snapshotFilenames[numberToKeep - 1].getTimestamp();
- LOG.info("Removed {} out of {} expected snapshot files older than {}", removedCnt, toRemoveCount, oldestTimestamp);
- return oldestTimestamp;
- }
-
- @Override
- public List<String> listSnapshots() throws IOException {
- File[] snapshots = snapshotDir.listFiles(SNAPSHOT_FILE_FILTER);
- return Lists.transform(Arrays.asList(snapshots), new Function<File, String>() {
- @Nullable
- @Override
- public String apply(@Nullable File input) {
- return input.getName();
- }
- });
- }
-
- @Override
- public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
- File[] logFiles = snapshotDir.listFiles(new LogFileFilter(timestamp, Long.MAX_VALUE));
- TimestampedFilename[] timestampedFiles = new TimestampedFilename[logFiles.length];
- for (int i = 0; i < logFiles.length; i++) {
- timestampedFiles[i] = new TimestampedFilename(logFiles[i]);
- }
- // logs need to be processed in ascending order
- Arrays.sort(timestampedFiles);
- return Lists.transform(Arrays.asList(timestampedFiles), new Function<TimestampedFilename, TransactionLog>() {
- @Nullable
- @Override
- public TransactionLog apply(@Nullable TimestampedFilename input) {
- return new LocalFileTransactionLog(input.getFile(), input.getTimestamp(), metricsCollector);
- }
- });
- }
-
- @Override
- public TransactionLog createLog(long timestamp) throws IOException {
- File newLogFile = new File(snapshotDir, LOG_FILE_PREFIX + timestamp);
- LOG.info("Creating new transaction log at {}", newLogFile.getAbsolutePath());
- return new LocalFileTransactionLog(newLogFile, timestamp, metricsCollector);
- }
-
- @Override
- public void deleteLogsOlderThan(long timestamp) throws IOException {
- File[] logFiles = snapshotDir.listFiles(new LogFileFilter(0, timestamp));
- int removedCnt = 0;
- for (File file : logFiles) {
- LOG.debug("Removing old transaction log {}", file.getPath());
- if (file.delete()) {
- removedCnt++;
- } else {
- LOG.warn("Failed to remove log file {}", file.getAbsolutePath());
- }
- }
- LOG.debug("Removed {} transaction logs older than {}", removedCnt, timestamp);
- }
-
- @Override
- public void setupStorage() throws IOException {
- // create the directory if it doesn't exist
- if (!snapshotDir.exists()) {
- if (!snapshotDir.mkdirs()) {
- throw new IOException("Failed to create directory " + configuredSnapshotDir +
- " for transaction snapshot storage");
- }
- } else {
- Preconditions.checkState(snapshotDir.isDirectory(),
- "Configured snapshot directory " + configuredSnapshotDir + " is not a directory!");
- Preconditions.checkState(snapshotDir.canWrite(), "Configured snapshot directory " +
- configuredSnapshotDir + " exists but is not writable!");
- }
- }
-
- @Override
- public List<String> listLogs() throws IOException {
- File[] logs = snapshotDir.listFiles(new LogFileFilter(0, Long.MAX_VALUE));
- return Lists.transform(Arrays.asList(logs), new Function<File, String>() {
- @Nullable
- @Override
- public String apply(@Nullable File input) {
- return input.getName();
- }
- });
- }
-
- private static class LogFileFilter implements FilenameFilter {
- private final long startTime;
- private final long endTime;
-
- public LogFileFilter(long startTime, long endTime) {
- this.startTime = startTime;
- this.endTime = endTime;
- }
-
- @Override
- public boolean accept(File file, String s) {
- if (s.startsWith(LOG_FILE_PREFIX)) {
- String[] parts = s.split("\\.");
- if (parts.length == 2) {
- try {
- long fileTime = Long.parseLong(parts[1]);
- return fileTime >= startTime && fileTime < endTime;
- } catch (NumberFormatException ignored) {
- LOG.warn("Filename {} did not match the expected pattern prefix.<timestamp>", s);
- }
- }
- }
- return false;
- }
- }
-
- /**
- * Represents a filename composed of a prefix and a ".timestamp" suffix. This is useful for manipulating both
- * snapshot and transaction log filenames.
- */
- private static class TimestampedFilename implements Comparable<TimestampedFilename> {
- private File file;
- private String prefix;
- private long timestamp;
-
- public TimestampedFilename(File file) {
- this.file = file;
- String[] parts = file.getName().split("\\.");
- if (parts.length != 2) {
- throw new IllegalArgumentException("Filename " + file.getName() +
- " did not match the expected pattern prefix.timestamp");
- }
- prefix = parts[0];
- timestamp = Long.parseLong(parts[1]);
- }
-
- public File getFile() {
- return file;
- }
-
- public String getPrefix() {
- return prefix;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public int compareTo(TimestampedFilename other) {
- int res = prefix.compareTo(other.getPrefix());
- if (res == 0) {
- res = Longs.compare(timestamp, other.getTimestamp());
- }
- return res;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/NoOpTransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/NoOpTransactionStateStorage.java b/tephra-core/src/main/java/co/cask/tephra/persist/NoOpTransactionStateStorage.java
deleted file mode 100644
index 3b6d0b7..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/NoOpTransactionStateStorage.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.snapshot.SnapshotCodec;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import com.google.common.util.concurrent.AbstractIdleService;
-import com.google.inject.Inject;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Minimal {@link TransactionStateStorage} implementation that does nothing, i.e. does not maintain any actual state.
- */
-public class NoOpTransactionStateStorage extends AbstractIdleService implements TransactionStateStorage {
-
- private final SnapshotCodec codec;
-
- @Inject
- public NoOpTransactionStateStorage(SnapshotCodecProvider codecProvider) {
- codec = codecProvider;
- }
-
- @Override
- protected void startUp() throws Exception {
- }
-
- @Override
- protected void shutDown() throws Exception {
- }
-
- @Override
- public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException {
- codec.encode(out, snapshot);
- }
-
- @Override
- public void writeSnapshot(TransactionSnapshot snapshot) throws IOException {
- }
-
- @Override
- public TransactionSnapshot getLatestSnapshot() throws IOException {
- return null;
- }
-
- @Override
- public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException {
- return null;
- }
-
- @Override
- public long deleteOldSnapshots(int numberToKeep) throws IOException {
- return 0;
- }
-
- @Override
- public List<String> listSnapshots() throws IOException {
- return new ArrayList<>(0);
- }
-
- @Override
- public List<TransactionLog> getLogsSince(long timestamp) throws IOException {
- return new ArrayList<>(0);
- }
-
- @Override
- public TransactionLog createLog(long timestamp) throws IOException {
- return new NoOpTransactionLog();
- }
-
- @Override
- public void deleteLogsOlderThan(long timestamp) throws IOException {
- }
-
- @Override
- public void setupStorage() throws IOException {
- }
-
- @Override
- public List<String> listLogs() throws IOException {
- return new ArrayList<>(0);
- }
-
- @Override
- public String getLocation() {
- return "no-op";
- }
-
- private static class NoOpTransactionLog implements TransactionLog {
- private long timestamp = System.currentTimeMillis();
-
- @Override
- public String getName() {
- return "no-op";
- }
-
- @Override
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public void append(TransactionEdit edit) throws IOException {
- }
-
- @Override
- public void append(List<TransactionEdit> edits) throws IOException {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public TransactionLogReader getReader() {
- return new TransactionLogReader() {
- @Override
- public TransactionEdit next() {
- return null;
- }
-
- @Override
- public TransactionEdit next(TransactionEdit reuse) {
- return null;
- }
-
- @Override
- public void close() {
- }
- };
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEdit.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEdit.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEdit.java
deleted file mode 100644
index 405bbfd..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEdit.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import com.google.common.base.Objects;
-import com.google.common.collect.Sets;
-import org.apache.hadoop.io.Writable;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Set;
-
-/**
- * Represents a transaction state change in the {@link TransactionLog}.
- */
-public class TransactionEdit implements Writable {
-
- /**
- * The possible state changes for a transaction.
- */
- public enum State {
- INPROGRESS, COMMITTING, COMMITTED, INVALID, ABORTED, MOVE_WATERMARK, TRUNCATE_INVALID_TX, CHECKPOINT
- }
-
- private long writePointer;
-
- /**
- * stores the value of visibility upper bound
- * (see {@link TransactionManager.InProgressTx#getVisibilityUpperBound()})
- * for edit of {@link State#INPROGRESS} only
- */
- private long visibilityUpperBound;
- private long commitPointer;
- private long expirationDate;
- private State state;
- private Set<ChangeId> changes;
- /** Whether or not the COMMITTED change should be fully committed. */
- private boolean canCommit;
- private TransactionType type;
- private Set<Long> truncateInvalidTx;
- private long truncateInvalidTxTime;
- private long parentWritePointer;
- private long[] checkpointPointers;
-
- // for Writable
- public TransactionEdit() {
- this.changes = Sets.newHashSet();
- this.truncateInvalidTx = Sets.newHashSet();
- }
-
- // package private for testing
- TransactionEdit(long writePointer, long visibilityUpperBound, State state, long expirationDate,
- Set<ChangeId> changes, long commitPointer, boolean canCommit, TransactionType type,
- Set<Long> truncateInvalidTx, long truncateInvalidTxTime, long parentWritePointer,
- long[] checkpointPointers) {
- this.writePointer = writePointer;
- this.visibilityUpperBound = visibilityUpperBound;
- this.state = state;
- this.expirationDate = expirationDate;
- this.changes = changes != null ? changes : Collections.<ChangeId>emptySet();
- this.commitPointer = commitPointer;
- this.canCommit = canCommit;
- this.type = type;
- this.truncateInvalidTx = truncateInvalidTx != null ? truncateInvalidTx : Collections.<Long>emptySet();
- this.truncateInvalidTxTime = truncateInvalidTxTime;
- this.parentWritePointer = parentWritePointer;
- this.checkpointPointers = checkpointPointers;
- }
-
- /**
- * Returns the transaction write pointer assigned for the state change.
- */
- public long getWritePointer() {
- return writePointer;
- }
-
- void setWritePointer(long writePointer) {
- this.writePointer = writePointer;
- }
-
- public long getVisibilityUpperBound() {
- return visibilityUpperBound;
- }
-
- void setVisibilityUpperBound(long visibilityUpperBound) {
- this.visibilityUpperBound = visibilityUpperBound;
- }
-
- /**
- * Returns the type of state change represented.
- */
- public State getState() {
- return state;
- }
-
- void setState(State state) {
- this.state = state;
- }
-
- /**
- * Returns any expiration timestamp (in milliseconds) associated with the state change. This should only
- * be populated for changes of type {@link State#INPROGRESS}.
- */
- public long getExpiration() {
- return expirationDate;
- }
-
- void setExpiration(long expirationDate) {
- this.expirationDate = expirationDate;
- }
-
- /**
- * @return the set of changed row keys associated with the state change. This is only populated for edits
- * of type {@link State#COMMITTING} or {@link State#COMMITTED}.
- */
- public Set<ChangeId> getChanges() {
- return changes;
- }
-
- void setChanges(Set<ChangeId> changes) {
- this.changes = changes;
- }
-
- /**
- * Returns the write pointer used to commit the row key change set. This is only populated for edits of type
- * {@link State#COMMITTED}.
- */
- public long getCommitPointer() {
- return commitPointer;
- }
-
- void setCommitPointer(long commitPointer) {
- this.commitPointer = commitPointer;
- }
-
- /**
- * Returns whether or not the transaction should be moved to the committed set. This is only populated for edits
- * of type {@link State#COMMITTED}.
- */
- public boolean getCanCommit() {
- return canCommit;
- }
-
- void setCanCommit(boolean canCommit) {
- this.canCommit = canCommit;
- }
-
- /**
- * Returns the transaction type. This is only populated for edits of type {@link State#INPROGRESS} or
- * {@link State#ABORTED}.
- */
- public TransactionType getType() {
- return type;
- }
-
- void setType(TransactionType type) {
- this.type = type;
- }
-
- /**
- * Returns the transaction ids to be removed from invalid transaction list. This is only populated for
- * edits of type {@link State#TRUNCATE_INVALID_TX}
- */
- public Set<Long> getTruncateInvalidTx() {
- return truncateInvalidTx;
- }
-
- void setTruncateInvalidTx(Set<Long> truncateInvalidTx) {
- this.truncateInvalidTx = truncateInvalidTx;
- }
-
- /**
- * Returns the time until which the invalid transactions need to be truncated from invalid transaction list.
- * This is only populated for edits of type {@link State#TRUNCATE_INVALID_TX}
- */
- public long getTruncateInvalidTxTime() {
- return truncateInvalidTxTime;
- }
-
- void setTruncateInvalidTxTime(long truncateInvalidTxTime) {
- this.truncateInvalidTxTime = truncateInvalidTxTime;
- }
-
- /**
- * Returns the parent write pointer for a checkpoint operation. This is only populated for edits of type
- * {@link State#CHECKPOINT}
- */
- public long getParentWritePointer() {
- return parentWritePointer;
- }
-
- void setParentWritePointer(long parentWritePointer) {
- this.parentWritePointer = parentWritePointer;
- }
-
- /**
- * Returns the checkpoint write pointers for the edit. This is only populated for edits of type
- * {@link State#ABORTED}.
- */
- public long[] getCheckpointPointers() {
- return checkpointPointers;
- }
-
- void setCheckpointPointers(long[] checkpointPointers) {
- this.checkpointPointers = checkpointPointers;
- }
-
- /**
- * Creates a new instance in the {@link State#INPROGRESS} state.
- */
- public static TransactionEdit createStarted(long writePointer, long visibilityUpperBound,
- long expirationDate, TransactionType type) {
- return new TransactionEdit(writePointer, visibilityUpperBound, State.INPROGRESS,
- expirationDate, null, 0L, false, type, null, 0L, 0L, null);
- }
-
- /**
- * Creates a new instance in the {@link State#COMMITTING} state.
- */
- public static TransactionEdit createCommitting(long writePointer, Set<ChangeId> changes) {
- return new TransactionEdit(writePointer, 0L, State.COMMITTING, 0L, changes, 0L, false, null, null, 0L, 0L, null);
- }
-
- /**
- * Creates a new instance in the {@link State#COMMITTED} state.
- */
- public static TransactionEdit createCommitted(long writePointer, Set<ChangeId> changes, long nextWritePointer,
- boolean canCommit) {
- return new TransactionEdit(writePointer, 0L, State.COMMITTED, 0L, changes, nextWritePointer, canCommit, null,
- null, 0L, 0L, null);
- }
-
- /**
- * Creates a new instance in the {@link State#ABORTED} state.
- */
- public static TransactionEdit createAborted(long writePointer, TransactionType type, long[] checkpointPointers) {
- return new TransactionEdit(writePointer, 0L, State.ABORTED, 0L, null, 0L, false, type, null, 0L, 0L,
- checkpointPointers);
- }
-
- /**
- * Creates a new instance in the {@link State#INVALID} state.
- */
- public static TransactionEdit createInvalid(long writePointer) {
- return new TransactionEdit(writePointer, 0L, State.INVALID, 0L, null, 0L, false, null, null, 0L, 0L, null);
- }
-
- /**
- * Creates a new instance in the {@link State#MOVE_WATERMARK} state.
- */
- public static TransactionEdit createMoveWatermark(long writePointer) {
- return new TransactionEdit(writePointer, 0L, State.MOVE_WATERMARK, 0L, null, 0L, false, null, null, 0L, 0L, null);
- }
-
- /**
- * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state.
- */
- public static TransactionEdit createTruncateInvalidTx(Set<Long> truncateInvalidTx) {
- return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, truncateInvalidTx,
- 0L, 0L, null);
- }
-
- /**
- * Creates a new instance in the {@link State#TRUNCATE_INVALID_TX} state.
- */
- public static TransactionEdit createTruncateInvalidTxBefore(long truncateInvalidTxTime) {
- return new TransactionEdit(0L, 0L, State.TRUNCATE_INVALID_TX, 0L, null, 0L, false, null, null,
- truncateInvalidTxTime, 0L, null);
- }
-
- /**
- * Creates a new instance in the {@link State#CHECKPOINT} state.
- */
- public static TransactionEdit createCheckpoint(long writePointer, long parentWritePointer) {
- return new TransactionEdit(writePointer, 0L, State.CHECKPOINT, 0L, null, 0L, false, null, null, 0L,
- parentWritePointer, null);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- TransactionEditCodecs.encode(this, out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- TransactionEditCodecs.decode(this, in);
- }
-
- @Override
- public final boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof TransactionEdit)) {
- return false;
- }
-
- TransactionEdit that = (TransactionEdit) o;
-
- return Objects.equal(this.writePointer, that.writePointer) &&
- Objects.equal(this.visibilityUpperBound, that.visibilityUpperBound) &&
- Objects.equal(this.commitPointer, that.commitPointer) &&
- Objects.equal(this.expirationDate, that.expirationDate) &&
- Objects.equal(this.state, that.state) &&
- Objects.equal(this.changes, that.changes) &&
- Objects.equal(this.canCommit, that.canCommit) &&
- Objects.equal(this.type, that.type) &&
- Objects.equal(this.truncateInvalidTx, that.truncateInvalidTx) &&
- Objects.equal(this.truncateInvalidTxTime, that.truncateInvalidTxTime) &&
- Objects.equal(this.parentWritePointer, that.parentWritePointer) &&
- Arrays.equals(this.checkpointPointers, that.checkpointPointers);
- }
-
- @Override
- public final int hashCode() {
- return Objects.hashCode(writePointer, visibilityUpperBound, commitPointer, expirationDate, state, changes,
- canCommit, type, parentWritePointer, checkpointPointers);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("writePointer", writePointer)
- .add("visibilityUpperBound", visibilityUpperBound)
- .add("commitPointer", commitPointer)
- .add("expiration", expirationDate)
- .add("state", state)
- .add("changesSize", changes != null ? changes.size() : 0)
- .add("canCommit", canCommit)
- .add("type", type)
- .add("truncateInvalidTx", truncateInvalidTx)
- .add("truncateInvalidTxTime", truncateInvalidTxTime)
- .add("parentWritePointer", parentWritePointer)
- .add("checkpointPointers", checkpointPointers)
- .toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEditCodecs.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEditCodecs.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEditCodecs.java
deleted file mode 100644
index e18b73f..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionEditCodecs.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.TransactionType;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Sets;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-/**
- * Utilities to handle encoding and decoding of {@link TransactionEdit} entries, while maintaining compatibility
- * with older versions of the serialized data.
- */
-public class TransactionEditCodecs {
-
- private static final TransactionEditCodec[] ALL_CODECS = {
- new TransactionEditCodecV1(),
- new TransactionEditCodecV2(),
- new TransactionEditCodecV3(),
- new TransactionEditCodecV4()
- };
-
- private static final SortedMap<Byte, TransactionEditCodec> CODECS = new TreeMap<>();
- static {
- for (TransactionEditCodec codec : ALL_CODECS) {
- CODECS.put(codec.getVersion(), codec);
- }
- }
-
- /**
- * Deserializes the encoded data from the given input stream, setting the values as fields
- * on the given {@code TransactionEdit} instances. This method expects first value in the
- * {code DataInput} to be a byte representing the codec version used to serialize the instance.
- *
- * @param dest the transaction edit to populate with the deserialized values
- * @param in the input stream containing the encoded data
- * @throws IOException if an error occurs while deserializing from the input stream
- */
- public static void decode(TransactionEdit dest, DataInput in) throws IOException {
- byte version = in.readByte();
- TransactionEditCodec codec = CODECS.get(version);
- if (codec == null) {
- throw new IOException("TransactionEdit was serialized with an unknown codec version " + version +
- ". Was it written with a newer version of Tephra?");
- }
- codec.decode(dest, in);
- }
-
- /**
- * Serializes the given {@code TransactionEdit} instance with the latest available codec.
- * This will first write out the version of the codec used to serialize the instance so that
- * the correct codec can be used when calling {@link #decode(TransactionEdit, DataInput)}.
- *
- * @param src the transaction edit to serialize
- * @param out the output stream to contain the data
- * @throws IOException if an error occurs while serializing to the output stream
- */
- public static void encode(TransactionEdit src, DataOutput out) throws IOException {
- TransactionEditCodec latestCodec = CODECS.get(CODECS.firstKey());
- out.writeByte(latestCodec.getVersion());
- latestCodec.encode(src, out);
- }
-
- /**
- * Encodes the given transaction edit using a specific codec. Note that this is only exposed
- * for use by tests.
- */
- @VisibleForTesting
- static void encode(TransactionEdit src, DataOutput out, TransactionEditCodec codec) throws IOException {
- out.writeByte(codec.getVersion());
- codec.encode(src, out);
- }
-
- /**
- * Defines the interface used for encoding and decoding {@link TransactionEdit} instances to and from
- * binary representations.
- */
- interface TransactionEditCodec {
- /**
- * Reads the encoded values from the data input stream and sets the fields in the given {@code TransactionEdit}
- * instance.
- *
- * @param dest the instance on which to set all the deserialized values
- * @param in the input stream containing the serialized data
- * @throws IOException if an error occurs while deserializing the data
- */
- void decode(TransactionEdit dest, DataInput in) throws IOException;
-
- /**
- * Writes all the field values from the {@code TransactionEdit} instance in serialized form to the data
- * output stream.
- *
- * @param src the instance to serialize to the stream
- * @param out the output stream to contain the data
- * @throws IOException if an error occurs while serializing the data
- */
- void encode(TransactionEdit src, DataOutput out) throws IOException;
-
- /**
- * Returns the version number for this codec. Each codec should use a unique version number, with the newest
- * codec having the lowest number.
- */
- byte getVersion();
- }
-
-
- // package-private for unit-test access
- static class TransactionEditCodecV1 implements TransactionEditCodec {
- @Override
- public void decode(TransactionEdit dest, DataInput in) throws IOException {
- dest.setWritePointer(in.readLong());
- int stateIdx = in.readInt();
- try {
- dest.setState(TransactionEdit.State.values()[stateIdx]);
- } catch (ArrayIndexOutOfBoundsException e) {
- throw new IOException("State enum ordinal value is out of range: " + stateIdx);
- }
- dest.setExpiration(in.readLong());
- dest.setCommitPointer(in.readLong());
- dest.setCanCommit(in.readBoolean());
- int changeSize = in.readInt();
- Set<ChangeId> changes = Sets.newHashSet();
- for (int i = 0; i < changeSize; i++) {
- int currentLength = in.readInt();
- byte[] currentBytes = new byte[currentLength];
- in.readFully(currentBytes);
- changes.add(new ChangeId(currentBytes));
- }
- dest.setChanges(changes);
- // 1st version did not store this info. It is safe to set firstInProgress to 0, it may decrease performance until
- // this tx is finished, but correctness will be preserved.
- dest.setVisibilityUpperBound(0);
- }
-
- /** @deprecated use {@link TransactionEditCodecs.TransactionEditCodecV4} instead, it is still here for
- * unit-tests only */
- @Override
- @Deprecated
- public void encode(TransactionEdit src, DataOutput out) throws IOException {
- out.writeLong(src.getWritePointer());
- // use ordinal for predictable size, though this does not support evolution
- out.writeInt(src.getState().ordinal());
- out.writeLong(src.getExpiration());
- out.writeLong(src.getCommitPointer());
- out.writeBoolean(src.getCanCommit());
- Set<ChangeId> changes = src.getChanges();
- if (changes == null) {
- out.writeInt(0);
- } else {
- out.writeInt(changes.size());
- for (ChangeId c : changes) {
- byte[] cKey = c.getKey();
- out.writeInt(cKey.length);
- out.write(cKey);
- }
- }
- // NOTE: we didn't have visibilityUpperBound in V1, it was added in V2
- // we didn't have transaction type, truncateInvalidTx and truncateInvalidTxTime in V1 and V2,
- // it was added in V3
- }
-
- @Override
- public byte getVersion() {
- return -1;
- }
- }
-
- // package-private for unit-test access
- static class TransactionEditCodecV2 extends TransactionEditCodecV1 implements TransactionEditCodec {
- @Override
- public void decode(TransactionEdit dest, DataInput in) throws IOException {
- super.decode(dest, in);
- dest.setVisibilityUpperBound(in.readLong());
- }
-
- /** @deprecated use {@link TransactionEditCodecs.TransactionEditCodecV4} instead, it is still here for
- * unit-tests only */
- @Override
- public void encode(TransactionEdit src, DataOutput out) throws IOException {
- super.encode(src, out);
- out.writeLong(src.getVisibilityUpperBound());
- // NOTE: we didn't have transaction type, truncateInvalidTx and truncateInvalidTxTime in V1 and V2,
- // it was added in V3
- }
-
- @Override
- public byte getVersion() {
- return -2;
- }
- }
-
- // TODO: refactor to avoid duplicate code among different version of codecs
- // package-private for unit-test access
- static class TransactionEditCodecV3 extends TransactionEditCodecV2 implements TransactionEditCodec {
- @Override
- public void decode(TransactionEdit dest, DataInput in) throws IOException {
- super.decode(dest, in);
- int typeIdx = in.readInt();
- // null transaction type is represented as -1
- if (typeIdx < 0) {
- dest.setType(null);
- } else {
- try {
- dest.setType(TransactionType.values()[typeIdx]);
- } catch (ArrayIndexOutOfBoundsException e) {
- throw new IOException("Type enum ordinal value is out of range: " + typeIdx);
- }
- }
-
- int truncateInvalidTxSize = in.readInt();
- Set<Long> truncateInvalidTx = emptySet(dest.getTruncateInvalidTx());
- for (int i = 0; i < truncateInvalidTxSize; i++) {
- truncateInvalidTx.add(in.readLong());
- }
- dest.setTruncateInvalidTx(truncateInvalidTx);
- dest.setTruncateInvalidTxTime(in.readLong());
- }
-
- private <T> Set<T> emptySet(Set<T> set) {
- if (set == null) {
- return Sets.newHashSet();
- }
- set.clear();
- return set;
- }
-
- @Override
- public void encode(TransactionEdit src, DataOutput out) throws IOException {
- super.encode(src, out);
- // null transaction type is represented as -1
- if (src.getType() == null) {
- out.writeInt(-1);
- } else {
- out.writeInt(src.getType().ordinal());
- }
-
- Set<Long> truncateInvalidTx = src.getTruncateInvalidTx();
- if (truncateInvalidTx == null) {
- out.writeInt(0);
- } else {
- out.writeInt(truncateInvalidTx.size());
- for (long id : truncateInvalidTx) {
- out.writeLong(id);
- }
- }
- out.writeLong(src.getTruncateInvalidTxTime());
- }
-
- @Override
- public byte getVersion() {
- return -3;
- }
- }
-
- static class TransactionEditCodecV4 extends TransactionEditCodecV3 {
- @Override
- public void decode(TransactionEdit dest, DataInput in) throws IOException {
- super.decode(dest, in);
- dest.setParentWritePointer(in.readLong());
- int checkpointPointersLen = in.readInt();
- if (checkpointPointersLen >= 0) {
- long[] checkpointPointers = new long[checkpointPointersLen];
- for (int i = 0; i < checkpointPointersLen; i++) {
- checkpointPointers[i] = in.readLong();
- }
- dest.setCheckpointPointers(checkpointPointers);
- }
- }
-
- @Override
- public void encode(TransactionEdit src, DataOutput out) throws IOException {
- super.encode(src, out);
- out.writeLong(src.getParentWritePointer());
- long[] checkpointPointers = src.getCheckpointPointers();
- if (checkpointPointers == null) {
- out.writeInt(-1);
- } else {
- out.writeInt(checkpointPointers.length);
- for (int i = 0; i < checkpointPointers.length; i++) {
- out.writeLong(checkpointPointers[i]);
- }
- }
- }
-
- @Override
- public byte getVersion() {
- return -4;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLog.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLog.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLog.java
deleted file mode 100644
index f76a8f1..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLog.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Represents a log of transaction state changes.
- */
-public interface TransactionLog {
-
- String getName();
-
- long getTimestamp();
-
- void append(TransactionEdit edit) throws IOException;
-
- void append(List<TransactionEdit> edits) throws IOException;
-
- void close() throws IOException;
-
- TransactionLogReader getReader() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogReader.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogReader.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogReader.java
deleted file mode 100644
index 3a3eaca..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogReader.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * Represents a reader for {@link TransactionLog} instances.
- */
-public interface TransactionLogReader extends Closeable {
- /**
- * Returns the next {@code TransactionEdit} from the log file, based on the current position, or {@code null}
- * if the end of the file has been reached.
- */
- TransactionEdit next() throws IOException;
-
- /**
- * Populates {@code reuse} with the next {@code TransactionEdit}, based on the reader's current position in the
- * log file.
- * @param reuse The {@code TransactionEdit} instance to populate with the log entry data.
- * @return The {@code TransactionEdit} instance, or {@code null} if the end of the file has been reached.
- * @throws IOException If an error is encountered reading the log data.
- */
- TransactionEdit next(TransactionEdit reuse) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogWriter.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogWriter.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogWriter.java
deleted file mode 100644
index 67d9aaf..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionLogWriter.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * Common interface for transaction log writers used by classes extending {@link AbstractTransactionLog}.
- */
-public interface TransactionLogWriter extends Closeable {
- /**
- * Adds a new transaction entry to the log. Note that this does not guarantee that the entry has been flushed
- * to persistent storage until {@link #sync()} has been called.
- *
- * @param entry The transaction edit to append.
- * @throws IOException If an error occurs while writing the edit to storage.
- */
- void append(AbstractTransactionLog.Entry entry) throws IOException;
-
- /**
- * Makes an entry of number of transaction entries that will follow in that log in a single sync.
- *
- * @param count Number of transaction entries.
- * @throws IOException If an error occurs while writing the count to storage.
- */
- void commitMarker(int count) throws IOException;
-
- /**
- * Syncs any pending transaction edits added through {@link #append(AbstractTransactionLog.Entry)},
- * but not yet flushed to durable storage.
- *
- * @throws IOException If an error occurs while flushing the outstanding edits.
- */
- void sync() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionSnapshot.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionSnapshot.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionSnapshot.java
deleted file mode 100644
index 22bc449..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionSnapshot.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.TransactionManager;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.TreeMap;
-
-/**
- * Represents an in-memory snapshot of the full transaction state.
- */
-public class TransactionSnapshot implements TransactionVisibilityState {
- private long timestamp;
- private long readPointer;
- private long writePointer;
- private Collection<Long> invalid;
- private NavigableMap<Long, TransactionManager.InProgressTx> inProgress;
- private Map<Long, Set<ChangeId>> committingChangeSets;
- private Map<Long, Set<ChangeId>> committedChangeSets;
-
- public TransactionSnapshot(long timestamp, long readPointer, long writePointer, Collection<Long> invalid,
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress,
- Map<Long, Set<ChangeId>> committing, Map<Long, Set<ChangeId>> committed) {
- this(timestamp, readPointer, writePointer, invalid, inProgress);
- this.committingChangeSets = committing;
- this.committedChangeSets = committed;
- }
-
- public TransactionSnapshot(long timestamp, long readPointer, long writePointer, Collection<Long> invalid,
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress) {
- this.timestamp = timestamp;
- this.readPointer = readPointer;
- this.writePointer = writePointer;
- this.invalid = invalid;
- this.inProgress = inProgress;
- this.committingChangeSets = Collections.emptyMap();
- this.committedChangeSets = Collections.emptyMap();
- }
-
- @Override
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public long getReadPointer() {
- return readPointer;
- }
-
- @Override
- public long getWritePointer() {
- return writePointer;
- }
-
- @Override
- public Collection<Long> getInvalid() {
- return invalid;
- }
-
- @Override
- public NavigableMap<Long, TransactionManager.InProgressTx> getInProgress() {
- return inProgress;
- }
-
- @Override
- public long getVisibilityUpperBound() {
- // the readPointer of the oldest in-progress tx is the oldest in use
- // todo: potential problem with not moving visibility upper bound for the whole duration of long-running tx
- Map.Entry<Long, TransactionManager.InProgressTx> firstInProgress = inProgress.firstEntry();
- if (firstInProgress == null) {
- // using readPointer as smallest visible when non txs are there
- return readPointer;
- }
- return firstInProgress.getValue().getVisibilityUpperBound();
- }
-
- /**
- * Returns a map of transaction write pointer to sets of changed row keys for transactions that had called
- * {@code InMemoryTransactionManager.canCommit(Transaction, Collection)} but not yet called
- * {@code InMemoryTransactionManager.commit(Transaction)} at the time of the snapshot.
- *
- * @return a map of transaction write pointer to set of changed row keys.
- */
- public Map<Long, Set<ChangeId>> getCommittingChangeSets() {
- return committingChangeSets;
- }
-
- /**
- * Returns a map of transaction write pointer to set of changed row keys for transaction that had successfully called
- * {@code InMemoryTransactionManager.commit(Transaction)} at the time of the snapshot.
- *
- * @return a map of transaction write pointer to set of changed row keys.
- */
- public Map<Long, Set<ChangeId>> getCommittedChangeSets() {
- return committedChangeSets;
- }
-
- /**
- * Checks that this instance matches another {@code TransactionSnapshot} instance. Note that the equality check
- * ignores the snapshot timestamp value, but includes all other properties.
- *
- * @param obj the other instance to check for equality.
- * @return {@code true} if the instances are equal, {@code false} if not.
- */
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof TransactionSnapshot)) {
- return false;
- }
- TransactionSnapshot other = (TransactionSnapshot) obj;
- return readPointer == other.readPointer &&
- writePointer == other.writePointer &&
- invalid.equals(other.invalid) &&
- inProgress.equals(other.inProgress) &&
- committingChangeSets.equals(other.committingChangeSets) &&
- committedChangeSets.equals(other.committedChangeSets);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("timestamp", timestamp)
- .add("readPointer", readPointer)
- .add("writePointer", writePointer)
- .add("invalidSize", invalid.size())
- .add("inProgressSize", inProgress.size())
- .add("committingSize", committingChangeSets.size())
- .add("committedSize", committedChangeSets.size())
- .toString();
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(readPointer, writePointer, invalid, inProgress, committingChangeSets, committedChangeSets);
- }
-
- /**
- * Creates a new {@code TransactionSnapshot} instance with copies of all of the individual collections.
- * @param readPointer current transaction read pointer
- * @param writePointer current transaction write pointer
- * @param invalid current list of invalid write pointers
- * @param inProgress current map of in-progress write pointers to expiration timestamps
- * @param committing current map of write pointers to change sets which have passed {@code canCommit()} but not
- * yet committed
- * @param committed current map of write pointers to change sets which have committed
- * @return a new {@code TransactionSnapshot} instance
- */
- public static TransactionSnapshot copyFrom(long snapshotTime, long readPointer,
- long writePointer, Collection<Long> invalid,
- NavigableMap<Long, TransactionManager.InProgressTx> inProgress,
- Map<Long, Set<ChangeId>> committing,
- NavigableMap<Long, Set<ChangeId>> committed) {
- // copy invalid IDs
- Collection<Long> invalidCopy = Lists.newArrayList(invalid);
- // copy in-progress IDs and expirations
- NavigableMap<Long, TransactionManager.InProgressTx> inProgressCopy = Maps.newTreeMap(inProgress);
-
- // for committing and committed maps, we need to copy each individual Set as well to prevent modification
- Map<Long, Set<ChangeId>> committingCopy = Maps.newHashMap();
- for (Map.Entry<Long, Set<ChangeId>> entry : committing.entrySet()) {
- committingCopy.put(entry.getKey(), new HashSet<>(entry.getValue()));
- }
-
- NavigableMap<Long, Set<ChangeId>> committedCopy = new TreeMap<>();
- for (Map.Entry<Long, Set<ChangeId>> entry : committed.entrySet()) {
- committedCopy.put(entry.getKey(), new HashSet<>(entry.getValue()));
- }
-
- return new TransactionSnapshot(snapshotTime, readPointer, writePointer,
- invalidCopy, inProgressCopy, committingCopy, committedCopy);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionStateStorage.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionStateStorage.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionStateStorage.java
deleted file mode 100644
index 0acb9bb..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionStateStorage.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import com.google.common.util.concurrent.Service;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.List;
-
-/**
- * Defines the common contract for persisting transaction state changes.
- */
-public interface TransactionStateStorage extends Service {
-
- /**
- * Persists a snapshot of transaction state to an output stream.
- */
- public void writeSnapshot(OutputStream out, TransactionSnapshot snapshot) throws IOException;
-
- /**
- * Persists a snapshot of transaction state.
- */
- public void writeSnapshot(TransactionSnapshot snapshot) throws IOException;
-
- /**
- * Returns the most recent snapshot that has been successfully written. Note that this may return {@code null}
- * if no completed snapshot files are found.
- */
- public TransactionSnapshot getLatestSnapshot() throws IOException;
-
- /**
- * Returns the most recent transaction visibility state that has been successfully written.
- * Note that this may return {@code null} if no completed snapshot files are found.
- * @return {@link TransactionVisibilityState}
- */
- public TransactionVisibilityState getLatestTransactionVisibilityState() throws IOException;
-
- /**
- * Removes any snapshots prior to the {@code numberToKeep} most recent.
- *
- * @param numberToKeep The number of most recent snapshots to keep.
- * @throws IOException If an error occurs while deleting old snapshots.
- * @return The timestamp of the oldest snapshot kept.
- */
- public long deleteOldSnapshots(int numberToKeep) throws IOException;
-
- /**
- * Returns the (non-qualified) names of available snapshots.
- */
- public List<String> listSnapshots() throws IOException;
-
- /**
- * Returns all {@link TransactionLog}s with a timestamp greater than or equal to the given timestamp. Note that
- * the returned list is guaranteed to be sorted in ascending timestamp order.
- */
- public List<TransactionLog> getLogsSince(long timestamp) throws IOException;
-
- /**
- * Creates a new {@link TransactionLog}.
- */
- public TransactionLog createLog(long timestamp) throws IOException;
-
- /**
- * Returns the (non-qualified) names of available logs.
- */
- public List<String> listLogs() throws IOException;
-
- /**
- * Removes any transaction logs with a timestamp older than the given value. Logs must be removed based on timestamp
- * to ensure we can fully recover state based on a given snapshot.
- * @param timestamp The timestamp to delete up to. Logs with a timestamp less than this value will be removed.
- * @throws IOException If an error occurs while removing logs.
- */
- public void deleteLogsOlderThan(long timestamp) throws IOException;
-
- /**
- * Create the directories required for the transaction state stage.
- * @throws IOException If an error occurred during the creation of required directories for transaction state storage.
- */
- public void setupStorage() throws IOException;
-
- /**
- * Returns a string representation of the location used for persistence.
- */
- public String getLocation();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/TransactionVisibilityState.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionVisibilityState.java b/tephra-core/src/main/java/co/cask/tephra/persist/TransactionVisibilityState.java
deleted file mode 100644
index 68d17c3..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/TransactionVisibilityState.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.persist;
-
-import co.cask.tephra.TransactionManager;
-
-import java.util.Collection;
-import java.util.NavigableMap;
-
-/**
- * Transaction Visibility state contains information required by TransactionProcessor CoProcessor
- * to determine cell visibility.
- */
-public interface TransactionVisibilityState {
-
- /**
- * Returns the timestamp from when this snapshot was created.
- */
- long getTimestamp();
-
- /**
- * Returns the read pointer at the time of the snapshot.
- */
- long getReadPointer();
-
- /**
- * Returns the next write pointer at the time of the snapshot.
- */
- long getWritePointer();
-
- /**
- * Returns the list of invalid write pointers at the time of the snapshot.
- */
- Collection<Long> getInvalid();
-
- /**
- * Returns the map of write pointers to in-progress transactions at the time of the snapshot.
- */
- NavigableMap<Long, TransactionManager.InProgressTx> getInProgress();
-
- /**
- * @return transaction id {@code X} such that any of the transactions newer than {@code X} might be invisible to
- * some of the currently in-progress transactions or to those that will be started <p>
- * NOTE: the returned tx id can be invalid.
- */
- long getVisibilityUpperBound();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/persist/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/persist/package-info.java b/tephra-core/src/main/java/co/cask/tephra/persist/package-info.java
deleted file mode 100644
index c94f0fe..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/persist/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains interfaces and implementations for persisting transaction state.
- */
-package co.cask.tephra.persist;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/rpc/RPCServiceHandler.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/rpc/RPCServiceHandler.java b/tephra-core/src/main/java/co/cask/tephra/rpc/RPCServiceHandler.java
deleted file mode 100644
index 664bfe3..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/rpc/RPCServiceHandler.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.rpc;
-
-/**
- * Defines lifecycle interface for all rpc handlers.
- */
-public interface RPCServiceHandler {
-
- void init() throws Exception;
-
- void destroy() throws Exception;
-}
[18/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionFilters.java b/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionFilters.java
deleted file mode 100644
index fa29b49..0000000
--- a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionFilters.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98.coprocessor;
-
-import co.cask.tephra.Transaction;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Factory class for providing {@link Filter} instances.
- */
-public class TransactionFilters {
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
- }
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessor.java
deleted file mode 100644
index 4ef041a..0000000
--- a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionProcessor.java
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionCodec;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreScanner;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-
-/**
- * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
- * for transactions:
- * <ul>
- * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
- * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
- * or expired due to TTL.</li>
- * </ul>
- *
- * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
- * or on all user tables by adding the following to hbase-site.xml:
- * {@code
- * <property>
- * <name>hbase.coprocessor.region.classes</name>
- * <value>co.cask.tephra.hbase98.coprocessor.TransactionProcessor</value>
- * </property>
- * }
- * </p>
- *
- * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
- * as an attribute:
- * {@code
- * Transaction t = ...;
- * Get get = new Get(...);
- * TransactionCodec codec = new TransactionCodec();
- * codec.addToOperation(get, t);
- * }
- * </p>
- */
-public class TransactionProcessor extends BaseRegionObserver {
- private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
-
- private TransactionStateCache cache;
- private final TransactionCodec txCodec;
- protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
- protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
-
- public TransactionProcessor() {
- this.txCodec = new TransactionCodec();
- }
-
- /* RegionObserver implementation */
-
- @Override
- public void start(CoprocessorEnvironment e) throws IOException {
- if (e instanceof RegionCoprocessorEnvironment) {
- RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
- Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
- this.cache = cacheSupplier.get();
-
- HTableDescriptor tableDesc = env.getRegion().getTableDesc();
- for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
- String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
- long ttl = 0;
- if (columnTTL != null) {
- try {
- ttl = Long.parseLong(columnTTL);
- LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
- } catch (NumberFormatException nfe) {
- LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
- ", value = " + columnTTL);
- }
- }
- ttlByFamily.put(columnDesc.getName(), ttl);
- }
-
- this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
- TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
- this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
- if (readNonTxnData) {
- LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
- }
- }
- }
-
- protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
- return new TransactionStateCacheSupplier(env.getConfiguration());
- }
-
- @Override
- public void stop(CoprocessorEnvironment e) throws IOException {
- // nothing to do
- }
-
- @Override
- public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
- throws IOException {
- Transaction tx = getFromOperation(get);
- if (tx != null) {
- projectFamilyDeletes(get);
- get.setMaxVersions();
- get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
- get.setFilter(newFilter);
- }
- }
-
- @Override
- public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
- Durability durability) throws IOException {
- // Translate deletes into our own delete tombstones
- // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
- // us to rollback the changes (by a real delete) if the transaction fails
-
- // Deletes that are part of a transaction rollback do not need special handling.
- // They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
- return;
- }
-
- // Other deletes are client-initiated and need to be translated into our own tombstones
- // TODO: this should delegate to the DeleteStrategy implementation.
- Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
- for (byte[] family : delete.getFamilyCellMap().keySet()) {
- List<Cell> familyCells = delete.getFamilyCellMap().get(family);
- if (isFamilyDelete(familyCells)) {
- deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- } else {
- int cellSize = familyCells.size();
- for (int i = 0; i < cellSize; i++) {
- Cell cell = familyCells.get(i);
- deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
- HConstants.EMPTY_BYTE_ARRAY);
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
- }
- e.getEnvironment().getRegion().put(deleteMarkers);
- // skip normal delete handling
- e.bypass();
- }
-
- private boolean isFamilyDelete(List<Cell> familyCells) {
- return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
- }
-
- @Override
- public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
- throws IOException {
- Transaction tx = getFromOperation(scan);
- if (tx != null) {
- projectFamilyDeletes(scan);
- scan.setMaxVersions();
- scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
- TxUtils.getMaxVisibleTimestamp(tx));
- Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
- scan.setFilter(newFilter);
- }
- return s;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any scan operation.
- * @param scan The original scan request
- * @return The modified scan request with the family delete qualifiers represented
- */
- private Scan projectFamilyDeletes(Scan scan) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return scan;
- }
-
- /**
- * Ensures that family delete markers are present in the columns requested for any get operation.
- * @param get The original get request
- * @return The modified get request with the family delete qualifiers represented
- */
- private Get projectFamilyDeletes(Get get) {
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
- NavigableSet<byte[]> columns = entry.getValue();
- // wildcard scans will automatically include the delete marker, so only need to add it when we have
- // explicit columns listed
- if (columns != null && !columns.isEmpty()) {
- get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
- }
- }
- return get;
- }
-
- @Override
- public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- KeyValueScanner memstoreScanner, InternalScanner scanner)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
- Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
- HConstants.OLDEST_TIMESTAMP);
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
- CompactionRequest request)
- throws IOException {
- return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners,
- scanType, earliestPutTs);
- }
-
- protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
- TransactionVisibilityState snapshot, Store store,
- List<? extends KeyValueScanner> scanners, ScanType type,
- long earliestPutTs) throws IOException {
- if (snapshot == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
- ", no current transaction state found, defaulting to normal " + action + " scanner");
- }
- return null;
- }
-
- // construct a dummy transaction from the latest snapshot
- Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
- Scan scan = new Scan();
- // need to see all versions, since we filter out excludes and applications may rely on multiple versions
- scan.setMaxVersions();
- scan.setFilter(
- new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
- snapshot.getInvalid(),
- getTransactionFilter(dummyTx, type, null)));
-
- return new StoreScanner(store, store.getScanInfo(), scan, scanners,
- type, store.getSmallestReadPoint(), earliestPutTs);
- }
-
- private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
- byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
- if (encoded != null) {
- return txCodec.decode(encoded);
- }
- return null;
- }
-
- /**
- * Derived classes can override this method to customize the filter used to return data visible for the current
- * transaction.
- *
- * @param tx the current transaction to apply
- * @param type the type of scan being performed
- */
- protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
- return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
- }
-
- /**
- * Filter used to include cells visible to in-progress transactions on flush and commit.
- */
- static class IncludeInProgressFilter extends FilterBase {
- private final long visibilityUpperBound;
- private final Set<Long> invalidIds;
- private final Filter txFilter;
-
- public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
- this.visibilityUpperBound = upperBound;
- this.invalidIds = Sets.newHashSet(invalids);
- this.txFilter = transactionFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- // include all cells visible to in-progress transactions, except for those already marked as invalid
- long ts = cell.getTimestamp();
- if (ts > visibilityUpperBound) {
- // include everything that could still be in-progress except invalids
- if (invalidIds.contains(ts)) {
- return ReturnCode.SKIP;
- }
- return ReturnCode.INCLUDE;
- }
- return txFilter.filterKeyValue(cell);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java
deleted file mode 100644
index 34c20bf..0000000
--- a/tephra-hbase-compat-0.98/src/main/java/co/cask/tephra/hbase98/coprocessor/TransactionVisibilityFilter.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase98.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-/**
- * Applies filtering of data based on transactional visibility (HBase 0.98+ specific version).
- * Note: this is intended for server-side use only, as additional properties need to be set on
- * any {@code Scan} or {@code Get} operation performed.
- */
-public class TransactionVisibilityFilter extends FilterBase {
- private final Transaction tx;
- // oldest visible timestamp by column family, used to apply TTL when reading
- private final Map<byte[], Long> oldestTsByFamily;
- // if false, empty values will be interpreted as deletes
- private final boolean allowEmptyValues;
- // whether or not we can remove delete markers
- // these can only be safely removed when we are traversing all storefiles
- private final boolean clearDeletes;
- // optional sub-filter to apply to visible cells
- private final Filter cellFilter;
-
- // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
- private byte[] currentFamily = new byte[0];
- private long currentOldestTs;
-
- private DeleteTracker deleteTracker = new DeleteTracker();
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType) {
- this(tx, ttlByFamily, allowEmptyValues, scanType, null);
- }
-
- /**
- * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
- *
- * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
- * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
- * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
- * these will be interpreted as "delete" markers and the column will be filtered out
- * @param scanType the type of scan operation being performed
- * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
- * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
- * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
- */
- public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
- ScanType scanType, @Nullable Filter cellFilter) {
- this.tx = tx;
- this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
- long familyTTL = ttlEntry.getValue();
- oldestTsByFamily.put(ttlEntry.getKey(),
- familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
- }
- this.allowEmptyValues = allowEmptyValues;
- this.clearDeletes =
- scanType == ScanType.COMPACT_DROP_DELETES ||
- (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
- this.cellFilter = cellFilter;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (!CellUtil.matchingFamily(cell, currentFamily)) {
- // column family changed
- currentFamily = CellUtil.cloneFamily(cell);
- Long familyOldestTs = oldestTsByFamily.get(currentFamily);
- currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
- deleteTracker.reset();
- }
- // need to apply TTL for the column family here
- long kvTimestamp = cell.getTimestamp();
- if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
- // passed TTL for this column, seek to next
- return ReturnCode.NEXT_COL;
- } else if (tx.isVisible(kvTimestamp)) {
- // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
- // cell is visible
- // visibility SNAPSHOT_ALL needs all matches
- return runSubFilter(ReturnCode.INCLUDE, cell);
- }
- if (DeleteTracker.isFamilyDelete(cell)) {
- deleteTracker.addFamilyDelete(cell);
- if (clearDeletes) {
- return ReturnCode.NEXT_COL;
- } else {
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // check if masked by family delete
- if (deleteTracker.isDeleted(cell)) {
- return ReturnCode.NEXT_COL;
- }
- // check for column delete
- if (isColumnDelete(cell)) {
- if (clearDeletes) {
- // skip "deleted" cell
- return ReturnCode.NEXT_COL;
- } else {
- // keep the marker but skip any remaining versions
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- }
- }
- // cell is visible
- // as soon as we find a KV to include we can move to the next column
- return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
- } else {
- return ReturnCode.SKIP;
- }
- }
-
- private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
- if (cellFilter != null) {
- ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
- return determineReturnCode(txFilterCode, subFilterCode);
- }
- return txFilterCode;
- }
-
- /**
- * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
- * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
- * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
- * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
- *
- * @param txFilterCode return code from TransactionVisibilityFilter
- * @param subFilterCode return code from sub-filter
- * @return final return code
- */
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- // Return the more restrictive of the two filter responses
- switch (subFilterCode) {
- case INCLUDE:
- return txFilterCode;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
-
- @Override
- public boolean filterRow() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRow();
- }
- return super.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- // Convert Tephra deletes back into HBase deletes
- if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
- if (DeleteTracker.isFamilyDelete(cell)) {
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
- KeyValue.Type.DeleteFamily);
- } else if (isColumnDelete(cell)) {
- // Note: in some cases KeyValue.Type.Delete is used in Delete object,
- // and in some other cases KeyValue.Type.DeleteColumn is used.
- // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
- // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
- // work in both cases.
- return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
- cell.getTimestamp(), KeyValue.Type.DeleteColumn);
- }
- }
- return cell;
- }
-
- @Override
- public void reset() throws IOException {
- deleteTracker.reset();
- if (cellFilter != null) {
- cellFilter.reset();
- }
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterRowKey(buffer, offset, length);
- }
- return super.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- if (cellFilter != null) {
- return cellFilter.filterAllRemaining();
- }
- return super.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- if (cellFilter != null) {
- cellFilter.filterRowCells(kvs);
- } else {
- super.filterRowCells(kvs);
- }
- }
-
- @Override
- public boolean hasFilterRow() {
- if (cellFilter != null) {
- return cellFilter.hasFilterRow();
- }
- return super.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextKeyHint(currentKV);
- }
- return super.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- if (cellFilter != null) {
- return cellFilter.getNextCellHint(currentKV);
- }
- return super.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- if (cellFilter != null) {
- return cellFilter.isFamilyEssential(name);
- }
- return super.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return super.toByteArray();
- }
-
- private boolean isColumnDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
- }
-
- private static final class DeleteTracker {
- private long familyDeleteTs;
-
- public static boolean isFamilyDelete(Cell cell) {
- return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
- CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
- CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
- }
-
- public void addFamilyDelete(Cell delete) {
- this.familyDeleteTs = delete.getTimestamp();
- }
-
- public boolean isDeleted(Cell cell) {
- return cell.getTimestamp() <= familyDeleteTs;
- }
-
- public void reset() {
- this.familyDeleteTs = 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/HBase98ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/HBase98ConfigurationProvider.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/HBase98ConfigurationProvider.java
new file mode 100644
index 0000000..f529783
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/HBase98ConfigurationProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.tephra.util.ConfigurationProvider;
+
+/**
+ * HBase 0.98 version of {@link ConfigurationProvider}.
+ */
+public class HBase98ConfigurationProvider extends ConfigurationProvider {
+ @Override
+ public Configuration get() {
+ return HBaseConfiguration.create();
+ }
+
+ @Override
+ public Configuration get(Configuration baseConf) {
+ return HBaseConfiguration.create(baseConf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/SecondaryIndexTable.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/SecondaryIndexTable.java
new file mode 100644
index 0000000..14a43a9
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/SecondaryIndexTable.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase98;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.TransactionContext;
+import org.apache.tephra.TransactionFailureException;
+import org.apache.tephra.distributed.TransactionServiceClient;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A Transactional SecondaryIndexTable.
+ */
+public class SecondaryIndexTable {
+ private byte[] secondaryIndex;
+ private TransactionAwareHTable transactionAwareHTable;
+ private TransactionAwareHTable secondaryIndexTable;
+ private TransactionContext transactionContext;
+ private final TableName secondaryIndexTableName;
+ private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
+ private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
+ private static final byte[] DELIMITER = new byte[] {0};
+
+ public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
+ byte[] secondaryIndex) {
+ secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
+ HTable secondaryIndexHTable = null;
+ HBaseAdmin hBaseAdmin = null;
+ try {
+ hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
+ if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
+ hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
+ }
+ secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ } finally {
+ try {
+ hBaseAdmin.close();
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ }
+
+ this.secondaryIndex = secondaryIndex;
+ this.transactionAwareHTable = new TransactionAwareHTable(hTable);
+ this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
+ this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
+ secondaryIndexTable);
+ }
+
+ public Result get(Get get) throws IOException {
+ return get(Collections.singletonList(get))[0];
+ }
+
+ public Result[] get(List<Get> gets) throws IOException {
+ try {
+ transactionContext.start();
+ Result[] result = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return result;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public Result[] getByIndex(byte[] value) throws IOException {
+ try {
+ transactionContext.start();
+ Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
+ scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
+ ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
+
+ ArrayList<Get> gets = new ArrayList<Get>();
+ for (Result result : indexScanner) {
+ for (Cell cell : result.listCells()) {
+ gets.add(new Get(cell.getValue()));
+ }
+ }
+ Result[] results = transactionAwareHTable.get(gets);
+ transactionContext.finish();
+ return results;
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ return null;
+ }
+
+ public void put(Put put) throws IOException {
+ put(Collections.singletonList(put));
+ }
+
+
+ public void put(List<Put> puts) throws IOException {
+ try {
+ transactionContext.start();
+ ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
+ for (Put put : puts) {
+ List<Put> indexPuts = new ArrayList<Put>();
+ Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
+ for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
+ for (KeyValue value : family.getValue()) {
+ if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
+ secondaryIndex, 0, secondaryIndex.length)) {
+ byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
+ Bytes.add(value.getValue(), DELIMITER,
+ value.getRow()));
+ Put indexPut = new Put(secondaryRow);
+ indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
+ indexPuts.add(indexPut);
+ }
+ }
+ }
+ secondaryIndexPuts.addAll(indexPuts);
+ }
+ transactionAwareHTable.put(puts);
+ secondaryIndexTable.put(secondaryIndexPuts);
+ transactionContext.finish();
+ } catch (Exception e) {
+ try {
+ transactionContext.abort();
+ } catch (TransactionFailureException e1) {
+ throw new IOException("Could not rollback transaction", e1);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java
new file mode 100644
index 0000000..54babd1
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java
@@ -0,0 +1,642 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase98;
+
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.tephra.AbstractTransactionAwareTable;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionAware;
+import org.apache.tephra.TxConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+/**
+ * A Transaction Aware HTable implementation for HBase 0.98. Operations are committed as usual,
+ * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
+ * was started.
+ */
+public class TransactionAwareHTable extends AbstractTransactionAwareTable
+ implements HTableInterface, TransactionAware {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
+ private final HTableInterface hTable;
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ */
+ public TransactionAwareHTable(HTableInterface hTable) {
+ this(hTable, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
+ this(hTable, conflictLevel, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
+ this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
+ boolean allowNonTransactional) {
+ super(conflictLevel, allowNonTransactional);
+ this.hTable = hTable;
+ }
+
+ /* AbstractTransactionAwareTable implementation */
+
+ @Override
+ protected byte[] getTableKey() {
+ return getTableName();
+ }
+
+ @Override
+ protected boolean doCommit() throws IOException {
+ hTable.flushCommits();
+ return true;
+ }
+
+ @Override
+ protected boolean doRollback() throws Exception {
+ try {
+ // pre-size arraylist of deletes
+ int size = 0;
+ for (Set<ActionChange> cs : changeSets.values()) {
+ size += cs.size();
+ }
+ List<Delete> rollbackDeletes = new ArrayList<>(size);
+ for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
+ long transactionTimestamp = entry.getKey();
+ for (ActionChange change : entry.getValue()) {
+ byte[] row = change.getRow();
+ byte[] family = change.getFamily();
+ byte[] qualifier = change.getQualifier();
+ Delete rollbackDelete = new Delete(row);
+ rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ switch (conflictLevel) {
+ case ROW:
+ case NONE:
+ // issue family delete for the tx write pointer
+ rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
+ break;
+ case COLUMN:
+ if (family != null && qualifier == null) {
+ rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
+ } else if (family != null && qualifier != null) {
+ rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
+ }
+ rollbackDeletes.add(rollbackDelete);
+ }
+ }
+ hTable.delete(rollbackDeletes);
+ return true;
+ } finally {
+ try {
+ hTable.flushCommits();
+ } catch (Exception e) {
+ LOG.error("Could not flush HTable commits", e);
+ }
+ tx = null;
+ changeSets.clear();
+ }
+ }
+
+ /* HTableInterface implementation */
+
+ @Override
+ public byte[] getTableName() {
+ return hTable.getTableName();
+ }
+
+ @Override
+ public TableName getName() {
+ return hTable.getName();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hTable.getConfiguration();
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return hTable.getTableDescriptor();
+ }
+
+ @Override
+ public boolean exists(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.exists(transactionalizeAction(get));
+ }
+
+ @Override
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.exists(transactionalizedGets);
+ }
+
+ @Override
+ public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batch(transactionalizeActions(actions), results);
+ }
+
+ @Override
+ public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batch(transactionalizeActions(actions));
+ }
+
+ @Override
+ public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
+ IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batchCallback(transactionalizeActions(actions), results, callback);
+ }
+
+ @Override
+ public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
+ InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batchCallback(transactionalizeActions(actions), callback);
+ }
+
+ @Override
+ public Result get(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.get(transactionalizeAction(get));
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.get(transactionalizedGets);
+ }
+
+ @Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.getRowOrBefore(row, family);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan scan) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addColumn(family, qualifier);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public void put(Put put) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Put txPut = transactionalizeAction(put);
+ hTable.put(txPut);
+ }
+
+ @Override
+ public void put(List<Put> puts) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
+ for (Put put : puts) {
+ Put txPut = transactionalizeAction(put);
+ transactionalizedPuts.add(txPut);
+ }
+ hTable.put(transactionalizedPuts);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndPut(row, family, qualifier, value, put);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public void delete(Delete delete) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.delete(transactionalizeAction(delete));
+ }
+
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
+ for (Delete delete : deletes) {
+ Delete txDelete = transactionalizeAction(delete);
+ transactionalizedDeletes.add(txDelete);
+ }
+ hTable.delete(transactionalizedDeletes);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndDelete(row, family, qualifier, value, delete);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
+ CompareFilter.CompareOp compareOp, byte[] value, RowMutations rowMutations)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndMutate(row, family, qualifier, compareOp, value, rowMutations);
+ }
+
+ throw new UnsupportedOperationException("checkAndMutate operation is not supported transactionally");
+ }
+
+ @Override
+ public void mutateRow(RowMutations rm) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ RowMutations transactionalMutations = new RowMutations();
+ for (Mutation mutation : rm.getMutations()) {
+ if (mutation instanceof Put) {
+ transactionalMutations.add(transactionalizeAction((Put) mutation));
+ } else if (mutation instanceof Delete) {
+ transactionalMutations.add(transactionalizeAction((Delete) mutation));
+ }
+ }
+ hTable.mutateRow(transactionalMutations);
+ }
+
+ @Override
+ public Result append(Append append) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.append(append);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public Result increment(Increment increment) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.increment(increment);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean isAutoFlush() {
+ return hTable.isAutoFlush();
+ }
+
+ @Override
+ public void flushCommits() throws IOException {
+ hTable.flushCommits();
+ }
+
+ @Override
+ public void close() throws IOException {
+ hTable.close();
+ }
+
+ @Override
+ public CoprocessorRpcChannel coprocessorService(byte[] row) {
+ return hTable.coprocessorService(row);
+ }
+
+ @Override
+ public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable)
+ throws ServiceException, Throwable {
+ return hTable.coprocessorService(service, startKey, endKey, callable);
+ }
+
+ @Override
+ public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable, Batch.Callback<R> callback)
+ throws ServiceException, Throwable {
+ hTable.coprocessorService(service, startKey, endKey, callable, callback);
+ }
+
+ @Override
+ public <R extends Message> Map<byte[], R> batchCoprocessorService(
+ MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey,
+ R responsePrototype) throws ServiceException, Throwable {
+ return hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
+ }
+
+ @Override
+ public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
+ Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
+ throws ServiceException, Throwable {
+ hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush) {
+ setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ hTable.setAutoFlush(autoFlush, clearBufferOnFail);
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ hTable.setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public long getWriteBufferSize() {
+ return hTable.getWriteBufferSize();
+ }
+
+ @Override
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ hTable.setWriteBufferSize(writeBufferSize);
+ }
+
+ // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
+
+ private Get transactionalizeAction(Get get) throws IOException {
+ addToOperation(get, tx);
+ return get;
+ }
+
+ private Scan transactionalizeAction(Scan scan) throws IOException {
+ addToOperation(scan, tx);
+ return scan;
+ }
+
+ private Put transactionalizeAction(Put put) throws IOException {
+ Put txPut = new Put(put.getRow(), tx.getWritePointer());
+ Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
+ if (!familyMap.isEmpty()) {
+ for (Map.Entry<byte[], List<Cell>> family : familyMap) {
+ List<Cell> familyValues = family.getValue();
+ if (!familyValues.isEmpty()) {
+ for (Cell value : familyValues) {
+ txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
+ addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
+ txPut.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txPut.setDurability(put.getDurability());
+ addToOperation(txPut, tx);
+ return txPut;
+ }
+
+ private Delete transactionalizeAction(Delete delete) throws IOException {
+ long transactionTimestamp = tx.getWritePointer();
+
+ byte[] deleteRow = delete.getRow();
+ Delete txDelete = new Delete(deleteRow, transactionTimestamp);
+
+ Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
+ if (familyToDelete.isEmpty()) {
+ // perform a row delete if we are using row-level conflict detection
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // Row delete leaves delete markers in all column families of the table
+ // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
+ for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
+ // no need to identify individual columns deleted
+ addToChangeSet(deleteRow, columnDescriptor.getName(), null);
+ }
+ } else {
+ Result result = get(new Get(delete.getRow()));
+ // Delete everything
+ NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
+ for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
+ }
+ }
+ }
+ } else {
+ for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
+ byte[] family = familyEntry.getKey();
+ List<Cell> entries = familyEntry.getValue();
+ boolean isFamilyDelete = false;
+ if (entries.size() == 1) {
+ Cell cell = entries.get(0);
+ isFamilyDelete = CellUtil.isDeleteFamily(cell);
+ }
+ if (isFamilyDelete) {
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // no need to identify individual columns deleted
+ txDelete.deleteFamily(family);
+ addToChangeSet(deleteRow, family, null);
+ } else {
+ Result result = get(new Get(delete.getRow()).addFamily(family));
+ // Delete entire family
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, family, column.getKey());
+ }
+ }
+ } else {
+ for (Cell value : entries) {
+ txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
+ addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ txDelete.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txDelete.setDurability(delete.getDurability());
+ return txDelete;
+ }
+
+ private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
+ List<Row> transactionalizedActions = new ArrayList<>(actions.size());
+ for (Row action : actions) {
+ if (action instanceof Get) {
+ transactionalizedActions.add(transactionalizeAction((Get) action));
+ } else if (action instanceof Put) {
+ transactionalizedActions.add(transactionalizeAction((Put) action));
+ } else if (action instanceof Delete) {
+ transactionalizedActions.add(transactionalizeAction((Delete) action));
+ } else {
+ transactionalizedActions.add(action);
+ }
+ }
+ return transactionalizedActions;
+ }
+
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilter.java
new file mode 100644
index 0000000..ceeef6f
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/CellSkipFilter.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
+ * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
+ * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
+ * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
+ * Please see TEPHRA-169 for more details.
+ */
+public class CellSkipFilter extends FilterBase {
+ private final Filter filter;
+ // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
+ private KeyValue skipColumn = null;
+
+ public CellSkipFilter(Filter filter) {
+ this.filter = filter;
+ }
+
+ /**
+ * Determines whether the current cell should be skipped. The cell will be skipped
+ * if the previous keyvalue had the same key as the current cell. This means filter already responded
+ * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
+ * @param cell the {@link Cell} to be tested for skipping
+ * @return true is current cell should be skipped, false otherwise
+ */
+ private boolean skipCellVersion(Cell cell) {
+ return skipColumn != null
+ && skipColumn.matchingRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
+ && skipColumn.matchingColumn(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (skipCellVersion(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+
+ ReturnCode code = filter.filterKeyValue(cell);
+ if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
+ // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
+ skipColumn = KeyValue.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ } else {
+ skipColumn = null;
+ }
+ return code;
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ return filter.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ return filter.transformCell(cell);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ filter.reset();
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ return filter.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ return filter.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ filter.filterRowCells(kvs);
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ return filter.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ return filter.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ return filter.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ return filter.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return filter.toByteArray();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionFilters.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionFilters.java
new file mode 100644
index 0000000..889c70a
--- /dev/null
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionFilters.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase98.coprocessor;
+
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.tephra.Transaction;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Factory class for providing {@link Filter} instances.
+ */
+public class TransactionFilters {
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+ }
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+ }
+}
[49/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionManager.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionManager.java b/tephra-core/src/main/java/co/cask/tephra/TransactionManager.java
deleted file mode 100644
index 3f51956..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionManager.java
+++ /dev/null
@@ -1,1398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.metrics.DefaultMetricsCollector;
-import co.cask.tephra.metrics.MetricsCollector;
-import co.cask.tephra.persist.NoOpTransactionStateStorage;
-import co.cask.tephra.persist.TransactionEdit;
-import co.cask.tephra.persist.TransactionLog;
-import co.cask.tephra.persist.TransactionLogReader;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionStateStorage;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.AbstractService;
-import com.google.inject.Inject;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-/**
- * This is the central place to manage all active transactions in the system.
- *
- * A transaction consists of
- * <ul>
- * <li>A write pointer: This is the version used for all writes of that transaction.</li>
- * <li>A read pointer: All reads under the transaction use this as an upper bound for the version.</li>
- * <li>A set of excluded versions: These are the write versions of other transactions that must be excluded from
- * reads, because those transactions are still in progress, or they failed but couldn't be properly rolled back.</li>
- * </ul>
- * To use the transaction system, a client must follow this sequence of steps:
- * <ol>
- * <li>Request a new transaction.</li>
- * <li>Use the transaction to read and write datasets. Datasets are encouraged to cache the writes of the
- * transaction in memory, to reduce the cost of rollback in case the transaction fails. </li>
- * <li>Check whether the transaction has conflicts. For this, the set of change keys are submitted via canCommit(),
- * and the transaction manager verifies that none of these keys are in conflict with other transactions that
- * committed since the start of this transaction.</li>
- * <li>If the transaction has conflicts:
- * <ol>
- * <li>Roll back the changes in every dataset that was changed. This can happen in-memory if the
- * changes were cached.</li>
- * <li>Abort the transaction to remove it from the active transactions.</li>
- * </ol>
- * <li>If the transaction has no conflicts:</li>
- * <ol>
- * <li>Persist all datasets changes to storage.</li>
- * <li>Commit the transaction. This will repeat the conflict detection, because more overlapping transactions
- * may have committed since the first conflict check.</li>
- * <li>If the transaction has conflicts:</li>
- * <ol>
- * <li>Roll back the changes in every dataset that was changed. This is more expensive because
- * changes must be undone in persistent storage.</li>
- * <li>Abort the transaction to remove it from the active transactions.</li>
- * </ol>
- * </ol>
- * </ol>
- * Transactions may be short or long-running. A short transaction is started with a timeout, and if it is not
- * committed before that timeout, it is invalidated and excluded from future reads. A long-running transaction has
- * no timeout and will remain active until it is committed or aborted. Long transactions are typically used in
- * map/reduce jobs and can produce enormous amounts of changes. Therefore, long transactions do not participate in
- * conflict detection (they would almost always have conflicts). We also assume that the changes of long transactions
- * are not tracked, and therefore cannot be rolled back. Hence, when a long transaction is aborted, it remains in the
- * list of excluded transactions to make its writes invisible.
- */
-public class TransactionManager extends AbstractService {
- // todo: optimize heavily
-
- private static final Logger LOG = LoggerFactory.getLogger(TransactionManager.class);
-
- // poll every 1 second to check whether a snapshot is needed
- private static final long SNAPSHOT_POLL_INTERVAL = 1000L;
-
- //poll every 10 second to emit metrics
- private static final long METRICS_POLL_INTERVAL = 10000L;
-
- private static final long[] NO_INVALID_TX = { };
-
- // Transactions that are in progress, with their info.
- private final NavigableMap<Long, InProgressTx> inProgress = new ConcurrentSkipListMap<Long, InProgressTx>();
-
- // the list of transactions that are invalid (not properly committed/aborted, or timed out)
- // TODO: explain usage of two arrays
- private final LongArrayList invalid = new LongArrayList();
- private long[] invalidArray = NO_INVALID_TX;
-
- // todo: use moving array instead (use Long2ObjectMap<byte[]> in fastutil)
- // todo: should this be consolidated with inProgress?
- // commit time next writePointer -> changes made by this tx
- private final NavigableMap<Long, Set<ChangeId>> committedChangeSets =
- new ConcurrentSkipListMap<Long, Set<ChangeId>>();
- // not committed yet
- private final Map<Long, Set<ChangeId>> committingChangeSets = Maps.newConcurrentMap();
-
- private long readPointer;
- private long lastWritePointer;
- private MetricsCollector txMetricsCollector;
-
- private final TransactionStateStorage persistor;
-
- private final int cleanupInterval;
- private final int defaultTimeout;
- private final int defaultLongTimeout;
- private DaemonThreadExecutor cleanupThread = null;
-
- private volatile TransactionLog currentLog;
-
- // timestamp of the last completed snapshot
- private long lastSnapshotTime;
- // frequency in millis to perform snapshots
- private final long snapshotFrequencyInSeconds;
- // number of most recent snapshots to retain
- private final int snapshotRetainCount;
- private DaemonThreadExecutor snapshotThread;
- private DaemonThreadExecutor metricsThread;
-
- // lock guarding change of the current transaction log
- private final ReentrantReadWriteLock logLock = new ReentrantReadWriteLock();
- private final Lock logReadLock = logLock.readLock();
- private final Lock logWriteLock = logLock.writeLock();
-
- // fudge factor (in milliseconds) used when interpreting transactions as LONG based on expiration
- // TODO: REMOVE WITH txnBackwardsCompatCheck()
- private final long longTimeoutTolerance;
-
- public TransactionManager(Configuration config) {
- this(config, new NoOpTransactionStateStorage(new SnapshotCodecProvider(config)), new DefaultMetricsCollector());
- }
-
- @Inject
- public TransactionManager(Configuration conf, @Nonnull TransactionStateStorage persistor,
- MetricsCollector txMetricsCollector) {
- this.persistor = persistor;
- cleanupInterval = conf.getInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL,
- TxConstants.Manager.DEFAULT_TX_CLEANUP_INTERVAL);
- defaultTimeout = conf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT,
- TxConstants.Manager.DEFAULT_TX_TIMEOUT);
- defaultLongTimeout = conf.getInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
- TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT);
- snapshotFrequencyInSeconds = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
- TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL);
- // must always keep at least 1 snapshot
- snapshotRetainCount = Math.max(conf.getInt(TxConstants.Manager.CFG_TX_SNAPSHOT_RETAIN,
- TxConstants.Manager.DEFAULT_TX_SNAPSHOT_RETAIN), 1);
-
- // intentionally not using a constant, as this config should not be exposed
- // TODO: REMOVE WITH txnBackwardsCompatCheck()
- longTimeoutTolerance = conf.getLong("data.tx.long.timeout.tolerance", 10000);
-
- //
- this.txMetricsCollector = txMetricsCollector;
- this.txMetricsCollector.configure(conf);
- clear();
- }
-
- private void clear() {
- invalid.clear();
- invalidArray = NO_INVALID_TX;
- inProgress.clear();
- committedChangeSets.clear();
- committingChangeSets.clear();
- lastWritePointer = 0;
- readPointer = 0;
- lastSnapshotTime = 0;
- }
-
- private boolean isStopping() {
- return State.STOPPING.equals(state());
- }
-
- @Override
- public synchronized void doStart() {
- LOG.info("Starting transaction manager.");
- txMetricsCollector.start();
- // start up the persistor
- persistor.startAndWait();
- try {
- persistor.setupStorage();
- } catch (IOException e) {
- Throwables.propagate(e);
- }
- // establish defaults in case there is no persistence
- clear();
- // attempt to recover state from last run
- recoverState();
- // start the periodic cleanup thread
- startCleanupThread();
- startSnapshotThread();
- startMetricsThread();
- // initialize the WAL if we did not force a snapshot in recoverState()
- initLog();
- // initialize next write pointer if needed
- if (lastWritePointer == 0) {
- lastWritePointer = getNextWritePointer();
- readPointer = lastWritePointer;
- }
-
- notifyStarted();
- }
-
- private void initLog() {
- if (currentLog == null) {
- try {
- currentLog = persistor.createLog(System.currentTimeMillis());
- } catch (IOException ioe) {
- throw Throwables.propagate(ioe);
- }
- }
- }
-
- private void startCleanupThread() {
- if (cleanupInterval <= 0 || defaultTimeout <= 0) {
- return;
- }
- LOG.info("Starting periodic timed-out transaction cleanup every " + cleanupInterval +
- " seconds with default timeout of " + defaultTimeout + " seconds.");
- this.cleanupThread = new DaemonThreadExecutor("tx-clean-timeout") {
- @Override
- public void doRun() {
- cleanupTimedOutTransactions();
- }
-
- @Override
- public long getSleepMillis() {
- return cleanupInterval * 1000;
- }
- };
- cleanupThread.start();
- }
-
- private void startSnapshotThread() {
- if (snapshotFrequencyInSeconds > 0) {
- LOG.info("Starting periodic snapshot thread, frequency = " + snapshotFrequencyInSeconds +
- " seconds, location = " + persistor.getLocation());
- this.snapshotThread = new DaemonThreadExecutor("tx-snapshot") {
- @Override
- public void doRun() {
- long currentTime = System.currentTimeMillis();
- if (lastSnapshotTime < (currentTime - snapshotFrequencyInSeconds * 1000)) {
- try {
- doSnapshot(false);
- } catch (IOException ioe) {
- LOG.error("Periodic snapshot failed!", ioe);
- }
- }
- }
-
- @Override
- protected void onShutdown() {
- // perform a final snapshot
- try {
- LOG.info("Writing final snapshot prior to shutdown");
- doSnapshot(true);
- } catch (IOException ioe) {
- LOG.error("Failed performing final snapshot on shutdown", ioe);
- }
- }
-
- @Override
- public long getSleepMillis() {
- return SNAPSHOT_POLL_INTERVAL;
- }
- };
- snapshotThread.start();
- }
- }
-
- // Emits Transaction Data structures size as metrics
- private void startMetricsThread() {
- LOG.info("Starting periodic Metrics Emitter thread, frequency = " + METRICS_POLL_INTERVAL);
- this.metricsThread = new DaemonThreadExecutor("tx-metrics") {
- @Override
- public void doRun() {
- txMetricsCollector.gauge("committing.size", committingChangeSets.size());
- txMetricsCollector.gauge("committed.size", committedChangeSets.size());
- txMetricsCollector.gauge("inprogress.size", inProgress.size());
- txMetricsCollector.gauge("invalid.size", invalidArray.length);
- }
-
- @Override
- protected void onShutdown() {
- // perform a final metrics emit
- txMetricsCollector.gauge("committing.size", committingChangeSets.size());
- txMetricsCollector.gauge("committed.size", committedChangeSets.size());
- txMetricsCollector.gauge("inprogress.size", inProgress.size());
- txMetricsCollector.gauge("invalid.size", invalidArray.length);
- }
-
- @Override
- public long getSleepMillis() {
- return METRICS_POLL_INTERVAL;
- }
- };
- metricsThread.start();
- }
-
- private void cleanupTimedOutTransactions() {
- List<TransactionEdit> invalidEdits = null;
- this.logReadLock.lock();
- try {
- synchronized (this) {
- if (!isRunning()) {
- return;
- }
-
- long currentTime = System.currentTimeMillis();
- List<Long> timedOut = Lists.newArrayList();
- for (Map.Entry<Long, InProgressTx> tx : inProgress.entrySet()) {
- long expiration = tx.getValue().getExpiration();
- if (expiration >= 0L && currentTime > expiration) {
- // timed out, remember tx id (can't remove while iterating over entries)
- timedOut.add(tx.getKey());
- LOG.info("Tx invalid list: added tx {} because of timeout", tx.getKey());
- } else if (expiration < 0) {
- LOG.warn("Transaction {} has negative expiration time {}. Likely cause is the transaction was not " +
- "migrated correctly, this transaction will be expired immediately",
- tx.getKey(), expiration);
- timedOut.add(tx.getKey());
- }
- }
- if (!timedOut.isEmpty()) {
- invalidEdits = Lists.newArrayListWithCapacity(timedOut.size());
- invalid.addAll(timedOut);
- for (long tx : timedOut) {
- committingChangeSets.remove(tx);
- inProgress.remove(tx);
- invalidEdits.add(TransactionEdit.createInvalid(tx));
- }
-
- // todo: find a more efficient way to keep this sorted. Could it just be an array?
- Collections.sort(invalid);
- invalidArray = invalid.toLongArray();
- LOG.info("Invalidated {} transactions due to timeout.", timedOut.size());
- }
- }
- if (invalidEdits != null) {
- appendToLog(invalidEdits);
- }
- } finally {
- this.logReadLock.unlock();
- }
- }
-
- public synchronized TransactionSnapshot getSnapshot() throws IOException {
- TransactionSnapshot snapshot = null;
- if (!isRunning() && !isStopping()) {
- return null;
- }
-
- long now = System.currentTimeMillis();
- // avoid duplicate snapshots at same timestamp
- if (now == lastSnapshotTime || (currentLog != null && now == currentLog.getTimestamp())) {
- try {
- TimeUnit.MILLISECONDS.sleep(1);
- } catch (InterruptedException ie) { }
- }
- // copy in memory state
- snapshot = getCurrentState();
-
- LOG.debug("Starting snapshot of transaction state with timestamp {}", snapshot.getTimestamp());
- LOG.debug("Returning snapshot of state: " + snapshot);
- return snapshot;
- }
-
- /**
- * Take a snapshot of the transaction state and serialize it into the given output stream.
- * @return whether a snapshot was taken.
- */
- public boolean takeSnapshot(OutputStream out) throws IOException {
- TransactionSnapshot snapshot = getSnapshot();
- if (snapshot != null) {
- persistor.writeSnapshot(out, snapshot);
- return true;
- } else {
- return false;
- }
- }
-
- private void doSnapshot(boolean closing) throws IOException {
- long snapshotTime = 0L;
- TransactionSnapshot snapshot = null;
- TransactionLog oldLog = null;
- try {
- this.logWriteLock.lock();
- try {
- synchronized (this) {
- snapshot = getSnapshot();
- if (snapshot == null && !closing) {
- return;
- }
- if (snapshot != null) {
- snapshotTime = snapshot.getTimestamp();
- }
-
- // roll WAL
- oldLog = currentLog;
- if (!closing) {
- currentLog = persistor.createLog(snapshot.getTimestamp());
- }
- }
- // there may not be an existing log on startup
- if (oldLog != null) {
- oldLog.close();
- }
- } finally {
- this.logWriteLock.unlock();
- }
-
- // save snapshot
- if (snapshot != null) {
- persistor.writeSnapshot(snapshot);
- lastSnapshotTime = snapshotTime;
-
- // clean any obsoleted snapshots and WALs
- long oldestRetainedTimestamp = persistor.deleteOldSnapshots(snapshotRetainCount);
- persistor.deleteLogsOlderThan(oldestRetainedTimestamp);
- }
- } catch (IOException ioe) {
- abortService("Snapshot (timestamp " + snapshotTime + ") failed due to: " + ioe.getMessage(), ioe);
- }
- }
-
- public synchronized TransactionSnapshot getCurrentState() {
- return TransactionSnapshot.copyFrom(System.currentTimeMillis(), readPointer, lastWritePointer,
- invalid, inProgress, committingChangeSets, committedChangeSets);
- }
-
- public synchronized void recoverState() {
- try {
- TransactionSnapshot lastSnapshot = persistor.getLatestSnapshot();
- // if we failed before a snapshot could complete, we might not have one to restore
- if (lastSnapshot != null) {
- restoreSnapshot(lastSnapshot);
- }
- // replay any WALs since the last snapshot
- Collection<TransactionLog> logs = persistor.getLogsSince(lastSnapshotTime);
- if (logs != null) {
- replayLogs(logs);
- }
- } catch (IOException e) {
- LOG.error("Unable to read back transaction state:", e);
- throw Throwables.propagate(e);
- }
- }
-
- /**
- * Restore the initial in-memory transaction state from a snapshot.
- */
- private void restoreSnapshot(TransactionSnapshot snapshot) {
- LOG.info("Restoring transaction state from snapshot at " + snapshot.getTimestamp());
- Preconditions.checkState(lastSnapshotTime == 0, "lastSnapshotTime has been set!");
- Preconditions.checkState(readPointer == 0, "readPointer has been set!");
- Preconditions.checkState(lastWritePointer == 0, "lastWritePointer has been set!");
- Preconditions.checkState(invalid.isEmpty(), "invalid list should be empty!");
- Preconditions.checkState(inProgress.isEmpty(), "inProgress map should be empty!");
- Preconditions.checkState(committingChangeSets.isEmpty(), "committingChangeSets should be empty!");
- Preconditions.checkState(committedChangeSets.isEmpty(), "committedChangeSets should be empty!");
- LOG.info("Restoring snapshot of state: " + snapshot);
-
- lastSnapshotTime = snapshot.getTimestamp();
- readPointer = snapshot.getReadPointer();
- lastWritePointer = snapshot.getWritePointer();
- invalid.addAll(snapshot.getInvalid());
- inProgress.putAll(txnBackwardsCompatCheck(defaultLongTimeout, longTimeoutTolerance, snapshot.getInProgress()));
- committingChangeSets.putAll(snapshot.getCommittingChangeSets());
- committedChangeSets.putAll(snapshot.getCommittedChangeSets());
- }
-
- /**
- * Check if in-progress transactions need to be migrated to have expiration time and type, if so do the migration.
- * This is required for backwards compatibility, when long running transactions were represented
- * with expiration time -1. This can be removed when we stop supporting SnapshotCodec version 1.
- */
- public static Map<Long, InProgressTx> txnBackwardsCompatCheck(int defaultLongTimeout, long longTimeoutTolerance,
- Map<Long, InProgressTx> inProgress) {
- for (Map.Entry<Long, InProgressTx> entry : inProgress.entrySet()) {
- long writePointer = entry.getKey();
- long expiration = entry.getValue().getExpiration();
- // LONG transactions will either have a negative expiration or expiration set to the long timeout
- // use a fudge factor on the expiration check, since expiraton is set based on system time, not the write pointer
- if (entry.getValue().getType() == null &&
- (expiration < 0 ||
- (getTxExpirationFromWritePointer(writePointer, defaultLongTimeout) - expiration
- < longTimeoutTolerance))) {
- // handle null expiration
- long newExpiration = getTxExpirationFromWritePointer(writePointer, defaultLongTimeout);
- InProgressTx compatTx =
- new InProgressTx(entry.getValue().getVisibilityUpperBound(), newExpiration, TransactionType.LONG,
- entry.getValue().getCheckpointWritePointers());
- entry.setValue(compatTx);
- } else if (entry.getValue().getType() == null) {
- InProgressTx compatTx =
- new InProgressTx(entry.getValue().getVisibilityUpperBound(), entry.getValue().getExpiration(),
- TransactionType.SHORT, entry.getValue().getCheckpointWritePointers());
- entry.setValue(compatTx);
- }
- }
- return inProgress;
- }
-
- /**
- * Resets the state of the transaction manager.
- */
- public void resetState() {
- this.logWriteLock.lock();
- try {
- // Take a snapshot before resetting the state, for debugging purposes
- doSnapshot(false);
- // Clear the state
- clear();
- // Take another snapshot: if no snapshot is taken after clearing the state
- // and the manager is restarted, we will recover from the snapshot taken
- // before resetting the state, which would be really bad
- // This call will also init a new WAL
- doSnapshot(false);
- } catch (IOException e) {
- LOG.error("Snapshot failed when resetting state!", e);
- e.printStackTrace();
- } finally {
- this.logWriteLock.unlock();
- }
- }
-
- /**
- * Replay all logged edits from the given transaction logs.
- */
- private void replayLogs(Collection<TransactionLog> logs) {
- for (TransactionLog log : logs) {
- LOG.info("Replaying edits from transaction log " + log.getName());
- int editCnt = 0;
- try {
- TransactionLogReader reader = log.getReader();
- // reader may be null in the case of an empty file
- if (reader == null) {
- continue;
- }
- TransactionEdit edit = null;
- while ((edit = reader.next()) != null) {
- editCnt++;
- switch (edit.getState()) {
- case INPROGRESS:
- long expiration = edit.getExpiration();
- TransactionType type = edit.getType();
- // Check if transaction needs to be migrated to have expiration and type. Previous version of
- // long running transactions were represented with expiration time as -1.
- // This can be removed when we stop supporting TransactionEditCodecV2.
- if (expiration < 0) {
- expiration = getTxExpirationFromWritePointer(edit.getWritePointer(), defaultLongTimeout);
- type = TransactionType.LONG;
- } else if (type == null) {
- type = TransactionType.SHORT;
- }
- addInProgressAndAdvance(edit.getWritePointer(), edit.getVisibilityUpperBound(),
- expiration, type);
- break;
- case COMMITTING:
- addCommittingChangeSet(edit.getWritePointer(), edit.getChanges());
- break;
- case COMMITTED:
- // TODO: need to reconcile usage of transaction id v/s write pointer TEPHRA-140
- long transactionId = edit.getWritePointer();
- long[] checkpointPointers = edit.getCheckpointPointers();
- long writePointer = checkpointPointers == null || checkpointPointers.length == 0 ?
- transactionId : checkpointPointers[checkpointPointers.length - 1];
- doCommit(transactionId, writePointer, edit.getChanges(),
- edit.getCommitPointer(), edit.getCanCommit());
- break;
- case INVALID:
- doInvalidate(edit.getWritePointer());
- break;
- case ABORTED:
- type = edit.getType();
- // Check if transaction edit needs to be migrated to have type. Previous versions of
- // ABORTED edits did not contain type.
- // This can be removed when we stop supporting TransactionEditCodecV2.
- if (type == null) {
- InProgressTx inProgressTx = inProgress.get(edit.getWritePointer());
- if (inProgressTx != null) {
- type = inProgressTx.getType();
- } else {
- // If transaction is not in-progress, then it has either been already aborted or invalidated.
- // We cannot determine the transaction's state based on current information, to be safe invalidate it.
- LOG.warn("Invalidating transaction {} as it's type cannot be determined during replay",
- edit.getWritePointer());
- doInvalidate(edit.getWritePointer());
- break;
- }
- }
- doAbort(edit.getWritePointer(), edit.getCheckpointPointers(), type);
- break;
- case TRUNCATE_INVALID_TX:
- if (edit.getTruncateInvalidTxTime() != 0) {
- doTruncateInvalidTxBefore(edit.getTruncateInvalidTxTime());
- } else {
- doTruncateInvalidTx(edit.getTruncateInvalidTx());
- }
- break;
- case CHECKPOINT:
- doCheckpoint(edit.getWritePointer(), edit.getParentWritePointer());
- break;
- default:
- // unknown type!
- throw new IllegalArgumentException("Invalid state for WAL entry: " + edit.getState());
- }
- }
- } catch (IOException ioe) {
- throw Throwables.propagate(ioe);
- } catch (InvalidTruncateTimeException e) {
- throw Throwables.propagate(e);
- }
- LOG.info("Read " + editCnt + " edits from log " + log.getName());
- }
- }
-
- @Override
- public void doStop() {
- Stopwatch timer = new Stopwatch().start();
- LOG.info("Shutting down gracefully...");
- // signal the cleanup thread to stop
- if (cleanupThread != null) {
- cleanupThread.shutdown();
- try {
- cleanupThread.join(30000L);
- } catch (InterruptedException ie) {
- LOG.warn("Interrupted waiting for cleanup thread to stop");
- Thread.currentThread().interrupt();
- }
- }
- if (metricsThread != null) {
- metricsThread.shutdown();
- try {
- metricsThread.join(30000L);
- } catch (InterruptedException ie) {
- LOG.warn("Interrupted waiting for cleanup thread to stop");
- Thread.currentThread().interrupt();
- }
- }
- if (snapshotThread != null) {
- // this will trigger a final snapshot on stop
- snapshotThread.shutdown();
- try {
- snapshotThread.join(30000L);
- } catch (InterruptedException ie) {
- LOG.warn("Interrupted waiting for snapshot thread to stop");
- Thread.currentThread().interrupt();
- }
- }
-
- persistor.stopAndWait();
- txMetricsCollector.stop();
- timer.stop();
- LOG.info("Took " + timer + " to stop");
- notifyStopped();
- }
-
- /**
- * Immediately shuts down the service, without going through the normal close process.
- * @param message A message describing the source of the failure.
- * @param error Any exception that caused the failure.
- */
- private void abortService(String message, Throwable error) {
- if (isRunning()) {
- LOG.error("Aborting transaction manager due to: " + message, error);
- notifyFailed(error);
- }
- }
-
- private void ensureAvailable() {
- Preconditions.checkState(isRunning(), "Transaction Manager is not running.");
- }
-
- /**
- * Start a short transaction with the default timeout.
- */
- public Transaction startShort() {
- return startShort(defaultTimeout);
- }
-
- /**
- * Start a short transaction with a given timeout.
- * @param timeoutInSeconds the time out period in seconds.
- */
- public Transaction startShort(int timeoutInSeconds) {
- Preconditions.checkArgument(timeoutInSeconds > 0, "timeout must be positive but is %s", timeoutInSeconds);
- txMetricsCollector.rate("start.short");
- Stopwatch timer = new Stopwatch().start();
- long expiration = getTxExpiration(timeoutInSeconds);
- Transaction tx = startTx(expiration, TransactionType.SHORT);
- txMetricsCollector.histogram("start.short.latency", (int) timer.elapsedMillis());
- return tx;
- }
-
- private static long getTxExpiration(long timeoutInSeconds) {
- long currentTime = System.currentTimeMillis();
- return currentTime + TimeUnit.SECONDS.toMillis(timeoutInSeconds);
- }
-
- public static long getTxExpirationFromWritePointer(long writePointer, long timeoutInSeconds) {
- return writePointer / TxConstants.MAX_TX_PER_MS + TimeUnit.SECONDS.toMillis(timeoutInSeconds);
- }
-
- private long getNextWritePointer() {
- // We want to align tx ids with current time. We assume that tx ids are sequential, but not less than
- // System.currentTimeMillis() * MAX_TX_PER_MS.
- return Math.max(lastWritePointer + 1, System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS);
- }
-
- /**
- * Start a long transaction. Long transactions and do not participate in conflict detection. Also, aborting a long
- * transaction moves it to the invalid list because we assume that its writes cannot be rolled back.
- */
- public Transaction startLong() {
- txMetricsCollector.rate("start.long");
- Stopwatch timer = new Stopwatch().start();
- long expiration = getTxExpiration(defaultLongTimeout);
- Transaction tx = startTx(expiration, TransactionType.LONG);
- txMetricsCollector.histogram("start.long.latency", (int) timer.elapsedMillis());
- return tx;
- }
-
- private Transaction startTx(long expiration, TransactionType type) {
- Transaction tx = null;
- long txid;
- // guard against changes to the transaction log while processing
- this.logReadLock.lock();
- try {
- synchronized (this) {
- ensureAvailable();
- txid = getNextWritePointer();
- tx = createTransaction(txid, type);
- addInProgressAndAdvance(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type);
- }
- // appending to WAL out of global lock for concurrent performance
- // we should still be able to arrive at the same state even if log entries are out of order
- appendToLog(TransactionEdit.createStarted(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type));
- } finally {
- this.logReadLock.unlock();
- }
- return tx;
- }
-
- private void addInProgressAndAdvance(long writePointer, long visibilityUpperBound,
- long expiration, TransactionType type) {
- inProgress.put(writePointer, new InProgressTx(visibilityUpperBound, expiration, type));
- advanceWritePointer(writePointer);
- }
-
- private void advanceWritePointer(long writePointer) {
- // don't move the write pointer back if we have out of order transaction log entries
- if (writePointer > lastWritePointer) {
- lastWritePointer = writePointer;
- }
- }
-
- public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException {
- txMetricsCollector.rate("canCommit");
- Stopwatch timer = new Stopwatch().start();
- if (inProgress.get(tx.getTransactionId()) == null) {
- // invalid transaction, either this has timed out and moved to invalid, or something else is wrong.
- if (invalid.contains(tx.getTransactionId())) {
- throw new TransactionNotInProgressException(
- String.format("canCommit() is called for transaction %d that is not in progress (it is known to be invalid)",
- tx.getTransactionId()));
- } else {
- throw new TransactionNotInProgressException(
- String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId()));
- }
- }
-
- Set<ChangeId> set = Sets.newHashSetWithExpectedSize(changeIds.size());
- for (byte[] change : changeIds) {
- set.add(new ChangeId(change));
- }
-
- if (hasConflicts(tx, set)) {
- return false;
- }
- // guard against changes to the transaction log while processing
- this.logReadLock.lock();
- try {
- synchronized (this) {
- ensureAvailable();
- addCommittingChangeSet(tx.getTransactionId(), set);
- }
- appendToLog(TransactionEdit.createCommitting(tx.getTransactionId(), set));
- } finally {
- this.logReadLock.unlock();
- }
- txMetricsCollector.histogram("canCommit.latency", (int) timer.elapsedMillis());
- return true;
- }
-
- private void addCommittingChangeSet(long writePointer, Set<ChangeId> changes) {
- committingChangeSets.put(writePointer, changes);
- }
-
- public boolean commit(Transaction tx) throws TransactionNotInProgressException {
- txMetricsCollector.rate("commit");
- Stopwatch timer = new Stopwatch().start();
- Set<ChangeId> changeSet = null;
- boolean addToCommitted = true;
- long commitPointer;
- // guard against changes to the transaction log while processing
- this.logReadLock.lock();
- try {
- synchronized (this) {
- ensureAvailable();
- // we record commits at the first not-yet assigned transaction id to simplify clearing out change sets that
- // are no longer visible by any in-progress transactions
- commitPointer = lastWritePointer + 1;
- if (inProgress.get(tx.getTransactionId()) == null) {
- // invalid transaction, either this has timed out and moved to invalid, or something else is wrong.
- if (invalid.contains(tx.getTransactionId())) {
- throw new TransactionNotInProgressException(
- String.format("canCommit() is called for transaction %d that is not in progress " +
- "(it is known to be invalid)", tx.getTransactionId()));
- } else {
- throw new TransactionNotInProgressException(
- String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId()));
- }
- }
-
- // these should be atomic
- // NOTE: whether we succeed or not we don't need to keep changes in committing state: same tx cannot
- // be attempted to commit twice
- changeSet = committingChangeSets.remove(tx.getTransactionId());
-
- if (changeSet != null) {
- // double-checking if there are conflicts: someone may have committed since canCommit check
- if (hasConflicts(tx, changeSet)) {
- return false;
- }
- } else {
- // no changes
- addToCommitted = false;
- }
- doCommit(tx.getTransactionId(), tx.getWritePointer(), changeSet, commitPointer, addToCommitted);
- }
- appendToLog(TransactionEdit.createCommitted(tx.getTransactionId(), changeSet, commitPointer, addToCommitted));
- } finally {
- this.logReadLock.unlock();
- }
- txMetricsCollector.histogram("commit.latency", (int) timer.elapsedMillis());
- return true;
- }
-
- private void doCommit(long transactionId, long writePointer, Set<ChangeId> changes, long commitPointer,
- boolean addToCommitted) {
- // In case this method is called when loading a previous WAL, we need to remove the tx from these sets
- committingChangeSets.remove(transactionId);
- if (addToCommitted && !changes.isEmpty()) {
- // No need to add empty changes to the committed change sets, they will never trigger any conflict
-
- // Record the committed change set with the next writePointer as the commit time.
- // NOTE: we use current next writePointer as key for the map, hence we may have multiple txs changesets to be
- // stored under one key
- Set<ChangeId> changeIds = committedChangeSets.get(commitPointer);
- if (changeIds != null) {
- // NOTE: we modify the new set to prevent concurrent modification exception, as other threads (e.g. in
- // canCommit) use it unguarded
- changes.addAll(changeIds);
- }
- committedChangeSets.put(commitPointer, changes);
- }
- // remove from in-progress set, so that it does not get excluded in the future
- InProgressTx previous = inProgress.remove(transactionId);
- if (previous == null) {
- // tx was not in progress! perhaps it timed out and is invalid? try to remove it there.
- if (invalid.rem(transactionId)) {
- invalidArray = invalid.toLongArray();
- LOG.info("Tx invalid list: removed committed tx {}", transactionId);
- }
- }
- // moving read pointer
- moveReadPointerIfNeeded(writePointer);
-
- // All committed change sets that are smaller than the earliest started transaction can be removed.
- // here we ignore transactions that have no timeout, they are long-running and don't participate in
- // conflict detection.
- // TODO: for efficiency, can we do this once per-log in replayLogs instead of once per edit?
- committedChangeSets.headMap(TxUtils.getFirstShortInProgress(inProgress)).clear();
- }
-
- public void abort(Transaction tx) {
- // guard against changes to the transaction log while processing
- txMetricsCollector.rate("abort");
- Stopwatch timer = new Stopwatch().start();
- this.logReadLock.lock();
- try {
- synchronized (this) {
- ensureAvailable();
- doAbort(tx.getTransactionId(), tx.getCheckpointWritePointers(), tx.getType());
- }
- appendToLog(TransactionEdit.createAborted(tx.getTransactionId(), tx.getType(), tx.getCheckpointWritePointers()));
- txMetricsCollector.histogram("abort.latency", (int) timer.elapsedMillis());
- } finally {
- this.logReadLock.unlock();
- }
- }
-
- private void doAbort(long writePointer, long[] checkpointWritePointers, TransactionType type) {
- committingChangeSets.remove(writePointer);
-
- if (type == TransactionType.LONG) {
- // Long running transactions cannot be aborted as their change sets are not saved,
- // and hence the changes cannot be rolled back. Invalidate the long running transaction instead.
- doInvalidate(writePointer);
- return;
- }
-
- // makes tx visible (assumes that all operations were rolled back)
- // remove from in-progress set, so that it does not get excluded in the future
- InProgressTx removed = inProgress.remove(writePointer);
- if (removed == null) {
- // tx was not in progress! perhaps it timed out and is invalid? try to remove it there.
- if (invalid.rem(writePointer)) {
- // remove any invalidated checkpoint pointers
- // this will only be present if the parent write pointer was also invalidated
- if (checkpointWritePointers != null) {
- for (int i = 0; i < checkpointWritePointers.length; i++) {
- invalid.rem(checkpointWritePointers[i]);
- }
- }
- invalidArray = invalid.toLongArray();
- LOG.info("Tx invalid list: removed aborted tx {}", writePointer);
- // removed a tx from excludes: must move read pointer
- moveReadPointerIfNeeded(writePointer);
- }
- } else {
- // removed a tx from excludes: must move read pointer
- moveReadPointerIfNeeded(writePointer);
- }
- }
-
- public boolean invalidate(long tx) {
- // guard against changes to the transaction log while processing
- txMetricsCollector.rate("invalidate");
- Stopwatch timer = new Stopwatch().start();
- this.logReadLock.lock();
- try {
- boolean success;
- synchronized (this) {
- ensureAvailable();
- success = doInvalidate(tx);
- }
- appendToLog(TransactionEdit.createInvalid(tx));
- txMetricsCollector.histogram("invalidate.latency", (int) timer.elapsedMillis());
- return success;
- } finally {
- this.logReadLock.unlock();
- }
- }
-
- private boolean doInvalidate(long writePointer) {
- Set<ChangeId> previousChangeSet = committingChangeSets.remove(writePointer);
- // remove from in-progress set, so that it does not get excluded in the future
- InProgressTx previous = inProgress.remove(writePointer);
- // This check is to prevent from invalidating committed transactions
- if (previous != null || previousChangeSet != null) {
- // add tx to invalids
- invalid.add(writePointer);
- if (previous == null) {
- LOG.debug("Invalidating tx {} in committing change sets but not in-progress", writePointer);
- } else {
- // invalidate any checkpoint write pointers
- LongArrayList childWritePointers = previous.getCheckpointWritePointers();
- if (childWritePointers != null) {
- for (int i = 0; i < childWritePointers.size(); i++) {
- invalid.add(childWritePointers.get(i));
- }
- }
- }
- LOG.info("Tx invalid list: added tx {} because of invalidate", writePointer);
- // todo: find a more efficient way to keep this sorted. Could it just be an array?
- Collections.sort(invalid);
- invalidArray = invalid.toLongArray();
- if (previous != null && !previous.isLongRunning()) {
- // tx was short-running: must move read pointer
- moveReadPointerIfNeeded(writePointer);
- }
- return true;
- }
- return false;
- }
-
- /**
- * Removes the given transaction ids from the invalid list.
- * @param invalidTxIds transaction ids
- * @return true if invalid list got changed, false otherwise
- */
- public boolean truncateInvalidTx(Set<Long> invalidTxIds) {
- // guard against changes to the transaction log while processing
- txMetricsCollector.rate("truncateInvalidTx");
- Stopwatch timer = new Stopwatch().start();
- this.logReadLock.lock();
- try {
- boolean success;
- synchronized (this) {
- ensureAvailable();
- success = doTruncateInvalidTx(invalidTxIds);
- }
- appendToLog(TransactionEdit.createTruncateInvalidTx(invalidTxIds));
- txMetricsCollector.histogram("truncateInvalidTx.latency", (int) timer.elapsedMillis());
- return success;
- } finally {
- this.logReadLock.unlock();
- }
- }
-
- private boolean doTruncateInvalidTx(Set<Long> invalidTxIds) {
- LOG.info("Removing tx ids {} from invalid list", invalidTxIds);
- boolean success = invalid.removeAll(invalidTxIds);
- if (success) {
- invalidArray = invalid.toLongArray();
- }
- return success;
- }
-
- /**
- * Removes all transaction ids started before the given time from invalid list.
- * @param time time in milliseconds
- * @return true if invalid list got changed, false otherwise
- * @throws InvalidTruncateTimeException if there are any in-progress transactions started before given time
- */
- public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
- // guard against changes to the transaction log while processing
- txMetricsCollector.rate("truncateInvalidTxBefore");
- Stopwatch timer = new Stopwatch().start();
- this.logReadLock.lock();
- try {
- boolean success;
- synchronized (this) {
- ensureAvailable();
- success = doTruncateInvalidTxBefore(time);
- }
- appendToLog(TransactionEdit.createTruncateInvalidTxBefore(time));
- txMetricsCollector.histogram("truncateInvalidTxBefore.latency", (int) timer.elapsedMillis());
- return success;
- } finally {
- this.logReadLock.unlock();
- }
- }
-
- private boolean doTruncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
- LOG.info("Removing tx ids before {} from invalid list", time);
- long truncateWp = time * TxConstants.MAX_TX_PER_MS;
- // Check if there any in-progress transactions started earlier than truncate time
- if (inProgress.lowerKey(truncateWp) != null) {
- throw new InvalidTruncateTimeException("Transactions started earlier than " + time + " are in-progress");
- }
-
- // Find all invalid transactions earlier than truncateWp
- Set<Long> toTruncate = Sets.newHashSet();
- for (long wp : invalid) {
- // invalid list is sorted, hence can stop as soon as we reach a wp >= truncateWp
- if (wp >= truncateWp) {
- break;
- }
- toTruncate.add(wp);
- }
- return doTruncateInvalidTx(toTruncate);
- }
-
- public Transaction checkpoint(Transaction originalTx) throws TransactionNotInProgressException {
- txMetricsCollector.rate("checkpoint");
- Stopwatch timer = new Stopwatch().start();
-
- Transaction checkpointedTx = null;
- long txId = originalTx.getTransactionId();
- long newWritePointer = 0;
- // guard against changes to the transaction log while processing
- this.logReadLock.lock();
- try {
- synchronized (this) {
- ensureAvailable();
- // check that the parent tx is in progress
- InProgressTx parentTx = inProgress.get(txId);
- if (parentTx == null) {
- if (invalid.contains(txId)) {
- throw new TransactionNotInProgressException(
- String.format("Transaction %d is not in progress because it was invalidated", txId));
- } else {
- throw new TransactionNotInProgressException(
- String.format("Transaction %d is not in progress", txId));
- }
- }
- newWritePointer = getNextWritePointer();
- doCheckpoint(newWritePointer, txId);
- // create a new transaction with the same read snapshot, plus the additional checkpoint write pointer
- // the same read snapshot is maintained to
- checkpointedTx = new Transaction(originalTx, newWritePointer,
- parentTx.getCheckpointWritePointers().toLongArray());
- }
- // appending to WAL out of global lock for concurrent performance
- // we should still be able to arrive at the same state even if log entries are out of order
- appendToLog(TransactionEdit.createCheckpoint(newWritePointer, txId));
- } finally {
- this.logReadLock.unlock();
- }
- txMetricsCollector.histogram("checkpoint.latency", (int) timer.elapsedMillis());
-
- return checkpointedTx;
- }
-
- private void doCheckpoint(long newWritePointer, long parentWritePointer) {
- InProgressTx existingTx = inProgress.get(parentWritePointer);
- existingTx.addCheckpointWritePointer(newWritePointer);
- advanceWritePointer(newWritePointer);
- }
-
- // hack for exposing important metric
- public int getExcludedListSize() {
- return invalid.size() + inProgress.size();
- }
-
- /**
- * @return the size of invalid list
- */
- public int getInvalidSize() {
- return this.invalid.size();
- }
-
- int getCommittedSize() {
- return this.committedChangeSets.size();
- }
-
- private boolean hasConflicts(Transaction tx, Set<ChangeId> changeIds) {
- if (changeIds.isEmpty()) {
- return false;
- }
-
- for (Map.Entry<Long, Set<ChangeId>> changeSet : committedChangeSets.entrySet()) {
- // If commit time is greater than tx read-pointer,
- // basically not visible but committed means "tx committed after given tx was started"
- if (changeSet.getKey() > tx.getTransactionId()) {
- if (overlap(changeSet.getValue(), changeIds)) {
- return true;
- }
- }
- }
- return false;
- }
-
- private boolean overlap(Set<ChangeId> a, Set<ChangeId> b) {
- // iterate over the smaller set, and check for every element in the other set
- if (a.size() > b.size()) {
- for (ChangeId change : b) {
- if (a.contains(change)) {
- return true;
- }
- }
- } else {
- for (ChangeId change : a) {
- if (b.contains(change)) {
- return true;
- }
- }
- }
- return false;
- }
-
- private void moveReadPointerIfNeeded(long committedWritePointer) {
- if (committedWritePointer > readPointer) {
- readPointer = committedWritePointer;
- }
- }
-
- /**
- * Creates a new Transaction. This method only get called from start transaction, which is already
- * synchronized.
- */
- private Transaction createTransaction(long writePointer, TransactionType type) {
- // For holding the first in progress short transaction Id (with timeout >= 0).
- long firstShortTx = Transaction.NO_TX_IN_PROGRESS;
- LongArrayList inProgressIds = new LongArrayList(inProgress.size());
- for (Map.Entry<Long, InProgressTx> entry : inProgress.entrySet()) {
- long txId = entry.getKey();
- inProgressIds.add(txId);
- // add any checkpointed write pointers to the in-progress list
- LongArrayList childIds = entry.getValue().getCheckpointWritePointers();
- if (childIds != null) {
- for (int i = 0; i < childIds.size(); i++) {
- inProgressIds.add(childIds.get(i));
- }
- }
- if (firstShortTx == Transaction.NO_TX_IN_PROGRESS && !entry.getValue().isLongRunning()) {
- firstShortTx = txId;
- }
- }
-
- return new Transaction(readPointer, writePointer, invalidArray, inProgressIds.toLongArray(), firstShortTx, type);
- }
-
- private void appendToLog(TransactionEdit edit) {
- try {
- Stopwatch timer = new Stopwatch().start();
- currentLog.append(edit);
- txMetricsCollector.rate("wal.append.count");
- txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsedMillis());
- } catch (IOException ioe) {
- abortService("Error appending to transaction log", ioe);
- }
- }
-
- private void appendToLog(List<TransactionEdit> edits) {
- try {
- Stopwatch timer = new Stopwatch().start();
- currentLog.append(edits);
- txMetricsCollector.rate("wal.append.count", edits.size());
- txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsedMillis());
- } catch (IOException ioe) {
- abortService("Error appending to transaction log", ioe);
- }
- }
-
- /**
- * Called from the tx service every 10 seconds.
- * This hack is needed because current metrics system is not flexible when it comes to adding new metrics.
- */
- public void logStatistics() {
- LOG.info("Transaction Statistics: write pointer = " + lastWritePointer +
- ", invalid = " + invalid.size() +
- ", in progress = " + inProgress.size() +
- ", committing = " + committingChangeSets.size() +
- ", committed = " + committedChangeSets.size());
- }
-
- private abstract static class DaemonThreadExecutor extends Thread {
- private AtomicBoolean stopped = new AtomicBoolean(false);
-
- public DaemonThreadExecutor(String name) {
- super(name);
- setDaemon(true);
- }
-
- public void run() {
- try {
- while (!isInterrupted() && !stopped.get()) {
- doRun();
- synchronized (stopped) {
- stopped.wait(getSleepMillis());
- }
- }
- } catch (InterruptedException ie) {
- LOG.info("Interrupted thread " + getName());
- }
- // perform any final cleanup
- onShutdown();
- LOG.info("Exiting thread " + getName());
- }
-
- public abstract void doRun();
-
- protected abstract long getSleepMillis();
-
- protected void onShutdown() {
- }
-
- public void shutdown() {
- if (stopped.compareAndSet(false, true)) {
- synchronized (stopped) {
- stopped.notifyAll();
- }
- }
- }
- }
-
- /**
- * Represents some of the info on in-progress tx
- */
- public static final class InProgressTx {
- /** the oldest in progress tx at the time of this tx start */
- private final long visibilityUpperBound;
- private final long expiration;
- private final TransactionType type;
- private LongArrayList checkpointWritePointers = new LongArrayList();
-
- public InProgressTx(long visibilityUpperBound, long expiration, TransactionType type) {
- this(visibilityUpperBound, expiration, type, new LongArrayList());
- }
-
- public InProgressTx(long visibilityUpperBound, long expiration, TransactionType type,
- LongArrayList checkpointWritePointers) {
- this.visibilityUpperBound = visibilityUpperBound;
- this.expiration = expiration;
- this.type = type;
- this.checkpointWritePointers = checkpointWritePointers;
- }
-
- // For backwards compatibility when long running txns were represented with -1 expiration
- @Deprecated
- public InProgressTx(long visibilityUpperBound, long expiration) {
- this(visibilityUpperBound, expiration, null);
- }
-
- public long getVisibilityUpperBound() {
- return visibilityUpperBound;
- }
-
- public long getExpiration() {
- return expiration;
- }
-
- @Nullable
- public TransactionType getType() {
- return type;
- }
-
- public boolean isLongRunning() {
- if (type == null) {
- // for backwards compatibility when long running txns were represented with -1 expiration
- return expiration == -1;
- }
- return type == TransactionType.LONG;
- }
-
- public void addCheckpointWritePointer(long checkpointWritePointer) {
- checkpointWritePointers.add(checkpointWritePointer);
- }
-
- public LongArrayList getCheckpointWritePointers() {
- return checkpointWritePointers;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == null || !(o instanceof InProgressTx)) {
- return false;
- }
-
- if (this == o) {
- return true;
- }
-
- InProgressTx other = (InProgressTx) o;
- return Objects.equal(visibilityUpperBound, other.getVisibilityUpperBound()) &&
- Objects.equal(expiration, other.getExpiration()) &&
- Objects.equal(type, other.type) &&
- Objects.equal(checkpointWritePointers, other.checkpointWritePointers);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(visibilityUpperBound, expiration, type, checkpointWritePointers);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("visibilityUpperBound", visibilityUpperBound)
- .add("expiration", expiration)
- .add("type", type)
- .add("checkpointWritePointers", checkpointWritePointers)
- .toString();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionServiceMain.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionServiceMain.java b/tephra-core/src/main/java/co/cask/tephra/TransactionServiceMain.java
deleted file mode 100644
index 8437017..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionServiceMain.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.distributed.TransactionService;
-import co.cask.tephra.runtime.ConfigModule;
-import co.cask.tephra.runtime.DiscoveryModules;
-import co.cask.tephra.runtime.TransactionClientModule;
-import co.cask.tephra.runtime.TransactionModules;
-import co.cask.tephra.runtime.ZKModule;
-import co.cask.tephra.util.ConfigurationFactory;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.twill.zookeeper.ZKClientService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.CountDownLatch;
-
-/**
- * Driver class to start and stop tx in distributed mode.
- */
-public class TransactionServiceMain {
-
- private static final Logger LOG = LoggerFactory.getLogger(TransactionServiceMain.class);
-
- private Configuration conf;
- private TransactionService txService;
-
- public static void main(String args[]) throws Exception {
- TransactionServiceMain instance = new TransactionServiceMain();
- instance.doMain(args);
- }
-
- public TransactionServiceMain() {
- this(null);
- }
-
- public TransactionServiceMain(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * The main method. It simply call methods in the same sequence
- * as if the program is started by jsvc.
- */
- public void doMain(final String[] args) throws Exception {
- final CountDownLatch shutdownLatch = new CountDownLatch(1);
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- try {
- TransactionServiceMain.this.stop();
- } finally {
- try {
- TransactionServiceMain.this.destroy();
- } finally {
- shutdownLatch.countDown();
- }
- }
- } catch (Throwable t) {
- LOG.error("Exception when shutting down: " + t.getMessage(), t);
- }
- }
- });
- init(args);
- start();
-
- shutdownLatch.await();
- }
-
- /**
- * Invoked by jsvc to initialize the program.
- */
- public void init(String[] args) {
- if (conf == null) {
- conf = new ConfigurationFactory().get();
- }
- }
-
- /**
- * Invoked by jsvc to start the program.
- */
- public void start() throws Exception {
- Injector injector = Guice.createInjector(
- new ConfigModule(conf),
- new ZKModule(),
- new DiscoveryModules().getDistributedModules(),
- new TransactionModules().getDistributedModules(),
- new TransactionClientModule()
- );
-
- ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
- zkClientService.startAndWait();
-
- // start a tx server
- txService = injector.getInstance(TransactionService.class);
- try {
- LOG.info("Starting {}", getClass().getSimpleName());
- txService.startAndWait();
- } catch (Exception e) {
- System.err.println("Failed to start service: " + e.getMessage());
- }
- }
-
- /**
- * Invoked by jsvc to stop the program.
- */
- public void stop() {
- LOG.info("Stopping {}", getClass().getSimpleName());
- if (txService == null) {
- return;
- }
- try {
- if (txService.isRunning()) {
- txService.stopAndWait();
- }
- } catch (Throwable e) {
- LOG.error("Failed to shutdown transaction service.", e);
- // because shutdown hooks execute concurrently, the logger may be closed already: thus also print it.
- System.err.println("Failed to shutdown transaction service: " + e.getMessage());
- e.printStackTrace(System.err);
- }
- }
-
- /**
- * Invoked by jsvc for resource cleanup.
- */
- public void destroy() { }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TransactionSystemClient.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TransactionSystemClient.java b/tephra-core/src/main/java/co/cask/tephra/TransactionSystemClient.java
deleted file mode 100644
index ca68143..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TransactionSystemClient.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.Set;
-
-/**
- * Client talking to transaction system.
- * See also {@link co.cask.tephra.TransactionAware}.
- * todo: explain Omid.
- */
-public interface TransactionSystemClient {
- /**
- * Starts new short transaction.
- * @return instance of {@link co.cask.tephra.Transaction}
- */
- // TODO: "short" and "long" are very misleading names. Use transaction attributes like "detect conflicts or not", etc.
- Transaction startShort();
-
- /**
- * Starts new short transaction.
- * @param timeout the timeout for the transaction
- * @return instance of {@link Transaction}
- */
- Transaction startShort(int timeout);
-
- /**
- * Starts new long transaction.
- * @return instance of {@link Transaction}
- */
- Transaction startLong();
-
- // this pre-commit detects conflicts with other transactions committed so far
- // NOTE: the changes set should not change after this operation, this may help us do some extra optimizations
- // NOTE: there should be time constraint on how long does it take to commit changes by the client after this operation
- // is submitted so that we can cleanup related resources
- // NOTE: as of now you can call this method multiple times, each time the changeSet of tx will be updated. Not sure
- // if we can call it a feature or a side-affect of implementation. It makes more sense to append changeset, but
- // before we really need it we don't do it because it will slow down tx manager throughput.
-
- /**
- * Checks if transaction with the set of changes can be committed. E.g. it can check conflicts with other changes and
- * refuse commit if there are conflicts. It is assumed that no other changes will be done in between this method call
- * and {@link #commit(Transaction)} which may check conflicts again to avoid races.
- * <p/>
- * Since we do conflict detection at commit time as well, this may seem redundant. The idea is to check for conflicts
- * before we persist changes to avoid rollback in case of conflicts as much as possible.
- * NOTE: in some situations we may want to skip this step to save on RPC with a risk of many rollback ops. So by
- * default we take safe path.
- *
- * @param tx transaction to verify
- * @param changeIds ids of changes made by transaction
- * @return true if transaction can be committed otherwise false
- */
- boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException;
-
- /**
- * Makes transaction visible. It will again check conflicts of changes submitted previously with
- * {@link #canCommit(Transaction, java.util.Collection)}
- * @param tx transaction to make visible.
- * @return true if transaction can be committed otherwise false
- */
- boolean commit(Transaction tx) throws TransactionNotInProgressException;
-
- /**
- * Makes transaction visible. You should call it only when all changes of this tx are undone.
- * NOTE: it will not throw {@link TransactionNotInProgressException} if transaction has timed out.
- * @param tx transaction to make visible.
- */
- void abort(Transaction tx);
-
- /**
- * Makes transaction invalid. You should call it if not all changes of this tx could be undone.
- * NOTE: it will not throw {@link TransactionNotInProgressException} if transaction has timed out.
- * @param tx transaction id to invalidate.
- * @return true if transaction has been successfully invalidated
- */
- boolean invalidate(long tx);
-
- /**
- * Performs a checkpoint operation on the current transaction, returning a new Transaction instance with the
- * updated state. A checkpoint operation assigns a new write pointer for the current transaction.
- * @param tx the current transaction to checkpoint
- * @return an updated transaction instance with the new write pointer
- */
- Transaction checkpoint(Transaction tx) throws TransactionNotInProgressException;
-
- /**
- * Retrieves the state of the transaction manager and send it as a stream. The snapshot will not be persisted.
- * @return an input stream containing an encoded snapshot of the transaction manager
- */
- InputStream getSnapshotInputStream() throws TransactionCouldNotTakeSnapshotException;
-
- /**
- * Return the status of the transaction Manager
- * @return a String which denotes the status of txManager
- */
- String status();
-
- /**
- * Resets the state of the transaction manager.
- */
- void resetState();
-
- /**
- * Removes the given transaction ids from the invalid list.
- * @param invalidTxIds transaction ids
- * @return true if invalid list got changed, false otherwise
- */
- boolean truncateInvalidTx(Set<Long> invalidTxIds);
-
- /**
- * Removes all transaction ids started before the given time from invalid list.
- * @param time time in milliseconds
- * @return true if invalid list got changed, false otherwise
- * @throws InvalidTruncateTimeException if there are any in-progress transactions started before given time
- */
- boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException;
-
- /**
- * @return the size of invalid list
- */
- int getInvalidSize();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/co/cask/tephra/TxConstants.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/co/cask/tephra/TxConstants.java b/tephra-core/src/main/java/co/cask/tephra/TxConstants.java
deleted file mode 100644
index 7c96c85..0000000
--- a/tephra-core/src/main/java/co/cask/tephra/TxConstants.java
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra;
-
-import co.cask.tephra.snapshot.DefaultSnapshotCodec;
-import co.cask.tephra.snapshot.SnapshotCodecV2;
-import co.cask.tephra.snapshot.SnapshotCodecV3;
-import co.cask.tephra.snapshot.SnapshotCodecV4;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Transaction system constants
- */
-public class TxConstants {
- /**
- * Defines what level of conflict detection should be used for transactions. {@code ROW} means that only the
- * table name and row key for each change will be used to determine if the transaction change sets conflict.
- * {@code COLUMN} means that the table name, row key, column family, and column qualifier will all be used to
- * identify write conflicts. {@code NONE} means that no conflict detection will be performed, but transaction
- * clients will still track the current transaction's change set to rollback any persisted changes in the event of
- * a failure. This should only be used where writes to the same coordinate should never conflict, such as
- * append-only data. The default value used by {@code TransactionAwareHTable} implementations is {@code COLUMN}.
- *
- * <p>
- * <strong>Note: for a given table, all clients must use the same conflict detection setting!</strong>
- * Otherwise conflicts will not be flagged correctly.
- * </p>
- */
- public enum ConflictDetection {
- ROW,
- COLUMN,
- NONE
- }
-
- /**
- * Property set for {@code org.apache.hadoop.hbase.HColumnDescriptor} to configure time-to-live on data within
- * the column family. The value given is in milliseconds. Once a cell's data has surpassed the given value in age,
- * the cell's data will no longer be visible and may be garbage collected.
- */
- public static final String PROPERTY_TTL = "dataset.table.ttl";
-
- /**
- * This is how many tx we allow per millisecond, if you care about the system for 100 years:
- * Long.MAX_VALUE / (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(365 * 100)) =
- * (as of Feb 20, 2014) 2,028,653. It is safe and convenient to use 1,000,000 as multiplier:
- * <ul>
- * <li>
- * we hardly can do more than 1 billion txs per second
- * </li>
- * <li>
- * long value will not overflow for 200 years
- * </li>
- * <li>
- * makes reading & debugging easier if multiplier is 10^n
- * </li>
- * </ul>
- */
- public static final long MAX_TX_PER_MS = 1000000;
-
- /**
- * Since HBase {@code Delete} operations cannot be undone at the same timestamp, "deleted" data is instead
- * overwritten with an empty {@code byte[]} to flag it as removed. Cells with empty values will be filtered out
- * of the results for read operations. If cells with empty values should be included in results (meaning data
- * cannot be transactionally deleted), then set this configuration property to true.
- */
- public static final String ALLOW_EMPTY_VALUES_KEY = "data.tx.allow.empty.values";
- public static final boolean ALLOW_EMPTY_VALUES_DEFAULT = false;
-
- /**
- * Key used to set the serialized transaction as an attribute on Get and Scan operations.
- */
- public static final String TX_OPERATION_ATTRIBUTE_KEY = "cask.tx";
- /**
- * Key used to flag a delete operation as part of a transaction rollback. This is used so that the
- * {@code TransactionProcessor} coprocessor loaded on a table can differentiate between deletes issued
- * as part of a normal client operation versus those performed when rolling back a transaction.
- */
- public static final String TX_ROLLBACK_ATTRIBUTE_KEY = "cask.tx.rollback";
-
- /**
- * Column qualifier used for a special delete marker tombstone, which identifies an entire column family as deleted.
- */
- public static final byte[] FAMILY_DELETE_QUALIFIER = new byte[0];
-
- // Constants for monitoring status
- public static final String STATUS_OK = "OK";
- public static final String STATUS_NOTOK = "NOTOK";
-
- /**
- * Indicates whether data written before Tephra was enabled on a table should be readable. Reading non-transactional
- * data can lead to slight performance penalty. Hence it is disabled by default.
- * @see <a href="https://issues.cask.co/browse/TEPHRA-89">TEPHRA-89</a>
- */
- public static final String READ_NON_TX_DATA = "data.tx.read.pre.existing";
- public static final boolean DEFAULT_READ_NON_TX_DATA = false;
-
- /**
- * TransactionManager configuration.
- */
- public static final class Manager {
- // TransactionManager configuration
- public static final String CFG_DO_PERSIST = "tx.persist";
- /** Directory in HDFS used for transaction snapshot and log storage. */
- public static final String CFG_TX_SNAPSHOT_DIR = "data.tx.snapshot.dir";
- /** Directory on the local filesystem used for transaction snapshot and log storage. */
- public static final String CFG_TX_SNAPSHOT_LOCAL_DIR = "data.tx.snapshot.local.dir";
- /** How often to clean up timed out transactions, in seconds, or 0 for no cleanup. */
- public static final String CFG_TX_CLEANUP_INTERVAL = "data.tx.cleanup.interval";
- /** The user id to access HDFS if not running in secure HDFS. */
- public static final String CFG_TX_HDFS_USER = "data.tx.hdfs.user";
- /** Default value for how often to check in-progress transactions for expiration, in seconds. */
- public static final int DEFAULT_TX_CLEANUP_INTERVAL = 10;
- /**
- * The timeout for a transaction, in seconds. If the transaction is not finished in that time,
- * it is marked invalid.
- */
- public static final String CFG_TX_TIMEOUT = "data.tx.timeout";
- /** Default value for transaction timeout, in seconds. */
- public static final int DEFAULT_TX_TIMEOUT = 30;
- /**
- * The timeout for a long running transaction, in seconds. If the transaction is not finished in that time,
- * it is marked invalid.
- */
- public static final String CFG_TX_LONG_TIMEOUT = "data.tx.long.timeout";
- /** Default value for long running transaction timeout, in seconds. */
- public static final int DEFAULT_TX_LONG_TIMEOUT = (int) TimeUnit.DAYS.toSeconds(1);
- /** The frequency (in seconds) to perform periodic snapshots, or 0 for no periodic snapshots. */
- public static final String CFG_TX_SNAPSHOT_INTERVAL = "data.tx.snapshot.interval";
- /** Default value for frequency of periodic snapshots of transaction state. */
- public static final long DEFAULT_TX_SNAPSHOT_INTERVAL = 300;
- /** Number of most recent transaction snapshots to retain. */
- public static final String CFG_TX_SNAPSHOT_RETAIN = "data.tx.snapshot.retain";
- /** Default value for number of most recent snapshots to retain. */
- public static final int DEFAULT_TX_SNAPSHOT_RETAIN = 10;
- }
-
- /**
- * TransactionService configuration.
- */
- public static final class Service {
-
- /** for the zookeeper quorum string for leader election for tx server. */
- public static final String CFG_DATA_TX_ZOOKEEPER_QUORUM
- = "data.tx.zookeeper.quorum";
-
- /** for the name used to announce service availability to discovery service */
- public static final String CFG_DATA_TX_DISCOVERY_SERVICE_NAME
- = "data.tx.discovery.service.name";
-
- /** for the port of the tx server. */
- public static final String CFG_DATA_TX_BIND_PORT
- = "data.tx.bind.port";
-
- /** for the address (hostname) of the tx server. */
- public static final String CFG_DATA_TX_BIND_ADDRESS
- = "data.tx.bind.address";
-
- /** the number of IO threads in the tx service. */
- public static final String CFG_DATA_TX_SERVER_IO_THREADS
- = "data.tx.server.io.threads";
-
- /** the number of handler threads in the tx service. */
- public static final String CFG_DATA_TX_SERVER_THREADS
- = "data.tx.server.threads";
-
- public static final String CFG_DATA_TX_THRIFT_MAX_READ_BUFFER
- = "data.tx.thrift.max.read.buffer";
-
- public static final String DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME
- = "transaction";
-
- /** default tx service port. */
- public static final int DEFAULT_DATA_TX_BIND_PORT
- = 15165;
-
- /** default tx service address. */
- public static final String DEFAULT_DATA_TX_BIND_ADDRESS
- = "0.0.0.0";
-
- /** default number of handler IO threads in tx service. */
- public static final int DEFAULT_DATA_TX_SERVER_IO_THREADS
- = 2;
-
- /** default number of handler threads in tx service. */
- public static final int DEFAULT_DATA_TX_SERVER_THREADS
- = 20;
-
- /** default thrift max read buffer size */
- public static final int DEFAULT_DATA_TX_THRIFT_MAX_READ_BUFFER
- = 16 * 1024 * 1024;
-
- // Configuration key names and defaults used by tx client.
-
- /** to specify the tx client socket timeout in ms. */
- public static final String CFG_DATA_TX_CLIENT_TIMEOUT
- = "data.tx.client.timeout";
-
- /** to specify the tx client provider strategy. */
- public static final String CFG_DATA_TX_CLIENT_PROVIDER
- = "data.tx.client.provider";
-
- /** to specify the number of threads for client provider "pool". */
- public static final String CFG_DATA_TX_CLIENT_COUNT
- = "data.tx.client.count";
-
- /** timeout (in milliseconds) for obtaining client from client provider "pool". */
- public static final String CFG_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS
- = "data.tx.client.obtain.timeout";
-
- /** to specify the retry strategy for a failed thrift call. */
- public static final String CFG_DATA_TX_CLIENT_RETRY_STRATEGY
- = "data.tx.client.retry.strategy";
-
- /** to specify the number of times to retry a failed thrift call. */
- public static final String CFG_DATA_TX_CLIENT_ATTEMPTS
- = "data.tx.client.retry.attempts";
-
- /** to specify the initial sleep time for retry strategy backoff. */
- public static final String CFG_DATA_TX_CLIENT_BACKOFF_INITIAL
- = "data.tx.client.retry.backoff.initial";
-
- /** to specify the backoff factor for retry strategy backoff. */
- public static final String CFG_DATA_TX_CLIENT_BACKOFF_FACTOR
- = "data.tx.client.retry.backoff.factor";
-
- /** to specify the sleep time limit for retry strategy backoff. */
- public static final String CFG_DATA_TX_CLIENT_BACKOFF_LIMIT
- = "data.tx.client.retry.backoff.limit";
-
- /** the default tx client socket timeout in milli seconds. */
- public static final int DEFAULT_DATA_TX_CLIENT_TIMEOUT_MS
- = 30 * 1000;
-
- /** default number of tx clients for client provider "pool". */
- public static final int DEFAULT_DATA_TX_CLIENT_COUNT
- = 50;
-
- /** default timeout (in milliseconds) for obtaining client from client provider "pool". */
- public static final long DEFAULT_DATA_TX_CLIENT_OBTAIN_TIMEOUT_MS
- = TimeUnit.SECONDS.toMillis(3);
-
- /** default tx client provider strategy. */
- public static final String DEFAULT_DATA_TX_CLIENT_PROVIDER
- = "pool";
-
- /** retry strategy for thrift clients, e.g. backoff, or n-times. */
- public static final String DEFAULT_DATA_TX_CLIENT_RETRY_STRATEGY
- = "backoff";
-
- /** default number of attempts for strategy n-times. */
- public static final int DEFAULT_DATA_TX_CLIENT_ATTEMPTS
- = 2;
-
- /** default initial sleep is 100ms. */
- public static final int DEFAULT_DATA_TX_CLIENT_BACKOFF_INITIAL
- = 100;
-
- /** default backoff factor is 4. */
- public static final int DEFAULT_DATA_TX_CLIENT_BACKOFF_FACTOR
- = 4;
-
- /** default sleep limit is 30 sec. */
- public static final int DEFAULT_DATA_TX_CLIENT_BACKOFF_LIMIT
- = 30 * 1000;
- }
-
- /**
- * Configuration properties for metrics reporting
- */
- public static final class Metrics {
- /**
- * Frequency at which metrics should be reported, in seconds.
- */
- public static final String REPORT_PERIOD_KEY = "data.tx.metrics.period";
- /**
- * Default report period for metrics, in seconds.
- */
- public static final int REPORT_PERIOD_DEFAULT = 60;
- }
-
- /**
- * Configuration properties used by HBase
- */
- public static final class HBase {
- public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
- public static final String ZK_SESSION_TIMEOUT = "zookeeper.session.timeout";
- public static final int DEFAULT_ZK_SESSION_TIMEOUT = 180 * 1000;
- }
-
- /**
- * Configuration for the TransactionDataJanitor coprocessor.
- */
- public static final class DataJanitor {
- /**
- * Whether or not the TransactionDataJanitor coprocessor should be enabled on tables.
- * Disable for testing.
- */
- public static final String CFG_TX_JANITOR_ENABLE = "data.tx.janitor.enable";
- public static final boolean DEFAULT_TX_JANITOR_ENABLE = true;
- }
-
- /**
- * Configuration for the transaction snapshot persistence.
- */
- public static final class Persist {
- /**
- * The class names of all known transaction snapshot codecs.
- */
- public static final String CFG_TX_SNAPHOT_CODEC_CLASSES = "data.tx.snapshot.codecs";
- public static final Class[] DEFAULT_TX_SNAPHOT_CODEC_CLASSES =
- { DefaultSnapshotCodec.class, SnapshotCodecV2.class, SnapshotCodecV3.class, SnapshotCodecV4.class };
- }
-
- /**
- * Configuration for transaction log edit entries
- */
- public static final class TransactionLog {
- /**
- * Key used to denote the number of entries appended.
- */
- public static final String NUM_ENTRIES_APPENDED = "count";
- public static final String VERSION_KEY = "version";
- public static final byte CURRENT_VERSION = 2;
- }
-
-}
[20/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java
new file mode 100644
index 0000000..6760ed7
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java
@@ -0,0 +1,1555 @@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase96;
+
+ import com.google.common.collect.ImmutableList;
+ import com.google.common.collect.Lists;
+ import com.google.common.primitives.Longs;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.hbase.Cell;
+ import org.apache.hadoop.hbase.CellUtil;
+ import org.apache.hadoop.hbase.Coprocessor;
+ import org.apache.hadoop.hbase.DoNotRetryIOException;
+ import org.apache.hadoop.hbase.HBaseTestingUtility;
+ import org.apache.hadoop.hbase.HColumnDescriptor;
+ import org.apache.hadoop.hbase.HConstants;
+ import org.apache.hadoop.hbase.HTableDescriptor;
+ import org.apache.hadoop.hbase.KeyValue;
+ import org.apache.hadoop.hbase.TableName;
+ import org.apache.hadoop.hbase.client.Delete;
+ import org.apache.hadoop.hbase.client.Durability;
+ import org.apache.hadoop.hbase.client.Get;
+ import org.apache.hadoop.hbase.client.HBaseAdmin;
+ import org.apache.hadoop.hbase.client.HTable;
+ import org.apache.hadoop.hbase.client.HTableInterface;
+ import org.apache.hadoop.hbase.client.Put;
+ import org.apache.hadoop.hbase.client.Result;
+ import org.apache.hadoop.hbase.client.ResultScanner;
+ import org.apache.hadoop.hbase.client.Scan;
+ import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+ import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+ import org.apache.hadoop.hbase.filter.BinaryComparator;
+ import org.apache.hadoop.hbase.filter.CompareFilter;
+ import org.apache.hadoop.hbase.filter.ValueFilter;
+ import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+ import org.apache.hadoop.hbase.util.Bytes;
+ import org.apache.tephra.Transaction;
+ import org.apache.tephra.TransactionConflictException;
+ import org.apache.tephra.TransactionContext;
+ import org.apache.tephra.TransactionManager;
+ import org.apache.tephra.TransactionSystemClient;
+ import org.apache.tephra.TxConstants;
+ import org.apache.tephra.hbase96.coprocessor.TransactionProcessor;
+ import org.apache.tephra.inmemory.InMemoryTxSystemClient;
+ import org.apache.tephra.metrics.TxMetricsCollector;
+ import org.apache.tephra.persist.InMemoryTransactionStateStorage;
+ import org.apache.tephra.persist.TransactionStateStorage;
+ import org.junit.After;
+ import org.junit.AfterClass;
+ import org.junit.Assert;
+ import org.junit.Before;
+ import org.junit.BeforeClass;
+ import org.junit.Test;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+
+ import static org.junit.Assert.assertArrayEquals;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertNotEquals;
+ import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.assertNull;
+ import static org.junit.Assert.assertTrue;
+ import static org.junit.Assert.fail;
+
+/**
+ * Tests for TransactionAwareHTables.
+ */
+public class TransactionAwareHTableTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
+
+ private static HBaseTestingUtility testUtil;
+ private static HBaseAdmin hBaseAdmin;
+ private static TransactionStateStorage txStateStorage;
+ private static TransactionManager txManager;
+ private static Configuration conf;
+ private TransactionContext transactionContext;
+ private TransactionAwareHTable transactionAwareHTable;
+ private HTable hTable;
+
+ private static final class TestBytes {
+ private static final byte[] table = Bytes.toBytes("testtable");
+ private static final byte[] family = Bytes.toBytes("f1");
+ private static final byte[] family2 = Bytes.toBytes("f2");
+ private static final byte[] qualifier = Bytes.toBytes("col1");
+ private static final byte[] qualifier2 = Bytes.toBytes("col2");
+ private static final byte[] row = Bytes.toBytes("row");
+ private static final byte[] row2 = Bytes.toBytes("row2");
+ private static final byte[] row3 = Bytes.toBytes("row3");
+ private static final byte[] row4 = Bytes.toBytes("row4");
+ private static final byte[] value = Bytes.toBytes("value");
+ private static final byte[] value2 = Bytes.toBytes("value2");
+ private static final byte[] value3 = Bytes.toBytes("value3");
+ }
+
+ private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
+
+ public static class TestRegionObserver extends BaseRegionObserver {
+ @Override
+ public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Put put, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (put.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Put should preserve attributes");
+ }
+ if (put.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+
+ @Override
+ public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Delete delete, final WALEdit edit,
+ final Durability durability) throws IOException {
+ if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
+ throw new DoNotRetryIOException("Delete should preserve attributes");
+ }
+ if (delete.getDurability() != Durability.USE_DEFAULT) {
+ throw new DoNotRetryIOException("Durability is not propagated correctly");
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ conf = testUtil.getConfiguration();
+
+ // Tune down the connection thread pool size
+ conf.setInt("hbase.hconnection.threads.core", 5);
+ conf.setInt("hbase.hconnection.threads.max", 10);
+ // Tunn down handler threads in regionserver
+ conf.setInt("hbase.regionserver.handler.count", 10);
+
+ // Set to random port
+ conf.setInt("hbase.master.port", 0);
+ conf.setInt("hbase.master.info.port", 0);
+ conf.setInt("hbase.regionserver.port", 0);
+ conf.setInt("hbase.regionserver.info.port", 0);
+
+ testUtil.startMiniCluster();
+ hBaseAdmin = testUtil.getHBaseAdmin();
+ txStateStorage = new InMemoryTransactionStateStorage();
+ txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
+ txManager.startAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ testUtil.shutdownMiniCluster();
+ hBaseAdmin.close();
+ }
+
+ @Before
+ public void setupBeforeTest() throws Exception {
+ hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
+ transactionAwareHTable = new TransactionAwareHTable(hTable);
+ transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
+ }
+
+ @After
+ public void shutdownAfterTest() throws IOException {
+ hBaseAdmin.disableTable(TestBytes.table);
+ hBaseAdmin.deleteTable(TestBytes.table);
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
+ return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
+ }
+
+ private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
+ List<String> coprocessors) throws Exception {
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+ for (byte[] family : columnFamilies) {
+ HColumnDescriptor columnDesc = new HColumnDescriptor(family);
+ columnDesc.setMaxVersions(Integer.MAX_VALUE);
+ columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
+ desc.addFamily(columnDesc);
+ }
+ if (existingData) {
+ desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
+ }
+ // Divide individually to prevent any overflow
+ int priority = Coprocessor.PRIORITY_USER;
+ desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
+ // order in list is the same order that coprocessors will be invoked
+ for (String coprocessor : coprocessors) {
+ desc.addCoprocessor(coprocessor, null, ++priority, null);
+ }
+ hBaseAdmin.createTable(desc);
+ testUtil.waitTableAvailable(tableName, 5000);
+ return new HTable(testUtil.getConfiguration(), tableName);
+ }
+
+ /**
+ * Test transactional put and get requests.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Test aborted put requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionPutAndGet() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(value, null);
+ }
+
+ /**
+ * Test transactional delete operations.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testValidTransactionalDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
+ new byte[][]{TestBytes.family, TestBytes.family2});
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+
+ // test column delete
+ // load 10 rows
+ txContext.start();
+ int rowCount = 10;
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
+ }
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertFalse(r.isEmpty());
+ for (int j = 0; j < 10; j++) {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
+ }
+ }
+ txContext.finish();
+
+ // delete odds columns from odd rows and even columns from even rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("row" + i));
+ for (int j = 0; j < 10; j++) {
+ if (i % 2 == j % 2) {
+ LOG.info("Deleting row={}, column={}", i, j);
+ d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
+ }
+ }
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted columns
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("row" + i));
+ Result r = txTable.get(g);
+ assertEquals(5, r.size());
+ for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
+ int col = Bytes.toInt(entry.getKey());
+ LOG.info("Got row={}, col={}", i, col);
+ // each row should only have the opposite mod (odd=even, even=odd)
+ assertNotEquals(i % 2, col % 2);
+ assertArrayEquals(TestBytes.value, entry.getValue());
+ }
+ }
+ txContext.finish();
+
+ // test family delete
+ // load 10 rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Put p = new Put(Bytes.toBytes("famrow" + i));
+ p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
+ txTable.put(p);
+ }
+ txContext.finish();
+
+ // verify all loaded rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(2, r.size());
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ txContext.finish();
+
+ // delete family1 for even rows, family2 for odd rows
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Delete d = new Delete(Bytes.toBytes("famrow" + i));
+ d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
+ txTable.delete(d);
+ }
+ txContext.finish();
+
+ // verify deleted families
+ txContext.start();
+ for (int i = 0; i < rowCount; i++) {
+ Get g = new Get(Bytes.toBytes("famrow" + i));
+ Result r = txTable.get(g);
+ assertEquals(1, r.size());
+ if (i % 2 == 0) {
+ assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ } else {
+ assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
+ }
+ }
+ txContext.finish();
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test that put and delete attributes are preserved
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAttributesPreserved() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, false,
+ Lists.newArrayList(TestRegionObserver.class.getName()));
+ try {
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ txContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
+ // set an attribute on the put, TestRegionObserver will verify it still exists
+ put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.put(put);
+ txContext.finish();
+
+ txContext.start();
+ Result result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ value = result.getValue(TestBytes.family2, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value2, value);
+
+ // test full row delete, TestRegionObserver will verify it still exists
+ txContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
+ txTable.delete(delete);
+ txContext.finish();
+
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ txContext.finish();
+ assertTrue(result.isEmpty());
+ } finally {
+ hTable.close();
+ }
+ }
+
+ /**
+ * Test aborted transactional delete requests, that must be rolled back.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAbortedTransactionalDelete() throws Exception {
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ Result result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row);
+ transactionAwareHTable.delete(delete);
+ transactionContext.abort();
+
+ transactionContext.start();
+ result = transactionAwareHTable.get(new Get(TestBytes.row));
+ transactionContext.finish();
+ value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
+ String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
+ HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ // Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ // Start a tx, delete a column family and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
+ txContext.abort();
+
+ // Above operations should have no effect on the row, since they were aborted
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testDeleteRollback() throws Exception {
+ testDeleteRollback(TxConstants.ConflictDetection.ROW);
+ testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
+ testDeleteRollback(TxConstants.ConflictDetection.NONE);
+ }
+
+ @Test
+ public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
+ try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext.finish();
+
+ txContext.start();
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ Result result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+
+ //Start a tx, delete the row and then abort the tx
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.abort();
+
+ //Start a tx and scan all the col families to make sure none of them have delete markers
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ result = txTable.get(new Get(TestBytes.row));
+ Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
+ Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
+ txContext.finish();
+ }
+ }
+
+ @Test
+ public void testRowDelete() throws Exception {
+ HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][] {TestBytes.family, TestBytes.family2});
+ TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
+ try {
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Test 1: full row delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ Get get = new Get(TestBytes.row);
+ Result result = txTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // delete entire row
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row));
+ txContext.finish();
+
+ // verify row is now empty
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for explicit column retrieval
+ result = txTable.get(new Get(TestBytes.row)
+ .addColumn(TestBytes.family, TestBytes.qualifier)
+ .addFamily(TestBytes.family2));
+ assertTrue(result.isEmpty());
+
+ // verify row is empty for scan
+ ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
+ assertNull(scanner.next());
+ scanner.close();
+
+ // verify row is empty for scan with explicit column
+ scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
+ assertNull(scanner.next());
+ scanner.close();
+ txContext.finish();
+
+ // write swapped values to one column per family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
+ txContext.finish();
+
+ // verify new values appear
+ txContext.start();
+ result = txTable.get(new Get(TestBytes.row));
+ assertFalse(result.isEmpty());
+ assertEquals(2, result.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+
+ scanner = txTable.getScanner(new Scan(TestBytes.row));
+ Result result1 = scanner.next();
+ assertNotNull(result1);
+ assertFalse(result1.isEmpty());
+ assertEquals(2, result1.size());
+ assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ scanner.close();
+ txContext.finish();
+
+ // Test 2: delete of first column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row2)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
+ txContext.finish();
+
+ txContext.start();
+ Result fam1Result = txTable.get(new Get(TestBytes.row2));
+ assertFalse(fam1Result.isEmpty());
+ assertEquals(2, fam1Result.size());
+ assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 3: delete of second column family
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
+ .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txContext.finish();
+
+ txContext.start();
+ txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
+ txContext.finish();
+
+ txContext.start();
+ Result fam2Result = txTable.get(new Get(TestBytes.row3));
+ assertFalse(fam2Result.isEmpty());
+ assertEquals(2, fam2Result.size());
+ assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // Test 4: delete specific rows in a range
+ txContext.start();
+ for (int i = 0; i < 10; i++) {
+ txTable.put(new Put(Bytes.toBytes("z" + i))
+ .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
+ .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ // delete odd rows
+ for (int i = 1; i < 10; i += 2) {
+ txTable.delete(new Delete(Bytes.toBytes("z" + i)));
+ }
+ txContext.finish();
+
+ txContext.start();
+ int cnt = 0;
+ ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
+ Result res;
+ while ((res = zScanner.next()) != null) {
+ assertFalse(res.isEmpty());
+ assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
+ cnt += 2;
+ }
+
+ // Test 5: delete prior writes in the same transaction
+ txContext.start();
+ txTable.put(new Put(TestBytes.row4)
+ .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
+ .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+ txTable.delete(new Delete(TestBytes.row4));
+ txContext.finish();
+
+ txContext.start();
+ Result row4Result = txTable.get(new Get(TestBytes.row4));
+ assertTrue(row4Result.isEmpty());
+ txContext.finish();
+ } finally {
+ txTable.close();
+ }
+ }
+
+ /**
+ * Expect an exception since a transaction hasn't been started.
+ *
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testTransactionlessFailure() throws Exception {
+ transactionAwareHTable.get(new Get(TestBytes.row));
+ }
+
+ /**
+ * Tests that each transaction can see its own persisted writes, while not seeing writes from other
+ * in-progress transactions.
+ */
+ @Test
+ public void testReadYourWrites() throws Exception {
+ // In-progress tx1: started before our main transaction
+ HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
+ TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
+
+ // In-progress tx2: started while our main transaction is running
+ HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
+ TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
+ TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
+
+ // create an in-progress write that should be ignored
+ byte[] col2 = Bytes.toBytes("col2");
+ inprogressTxContext1.start();
+ Put putCol2 = new Put(TestBytes.row);
+ byte[] valueCol2 = Bytes.toBytes("writing in progress");
+ putCol2.add(TestBytes.family, col2, valueCol2);
+ txHTable1.put(putCol2);
+
+ // start a tx and write a value to test reading in same tx
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] value = Bytes.toBytes("writing");
+ put.add(TestBytes.family, TestBytes.qualifier, value);
+ transactionAwareHTable.put(put);
+
+ // test that a write from a tx started after the first is not visible
+ inprogressTxContext2.start();
+ Put put2 = new Put(TestBytes.row);
+ byte[] value2 = Bytes.toBytes("writing2");
+ put2.add(TestBytes.family, TestBytes.qualifier, value2);
+ txHTable2.put(put2);
+
+ Get get = new Get(TestBytes.row);
+ Result row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
+ assertNotNull(col1Value);
+ assertArrayEquals(value, col1Value);
+ // write from in-progress transaction should not be visible
+ byte[] col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ // commit in-progress transaction, should still not be visible
+ inprogressTxContext1.finish();
+
+ get = new Get(TestBytes.row);
+ row = transactionAwareHTable.get(get);
+ assertFalse(row.isEmpty());
+ col2Value = row.getValue(TestBytes.family, col2);
+ assertNull(col2Value);
+
+ transactionContext.finish();
+
+ inprogressTxContext2.abort();
+ }
+
+ @Test
+ public void testRowLevelConflictDetection() throws Exception {
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.ROW);
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ byte[] row1 = Bytes.toBytes("row1");
+ byte[] row2 = Bytes.toBytes("row2");
+ byte[] col1 = Bytes.toBytes("c1");
+ byte[] col2 = Bytes.toBytes("c2");
+ byte[] val1 = Bytes.toBytes("val1");
+ byte[] val2 = Bytes.toBytes("val2");
+
+ // test that concurrent writing to different rows succeeds
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
+
+ // should be no conflicts
+ txContext1.finish();
+ txContext2.finish();
+
+ transactionContext.start();
+ Result res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // test that writing to different columns in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
+
+ txContext2.start();
+ txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row-level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row1));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col1);
+ assertNotNull(cell);
+ // should now be val2
+ assertArrayEquals(val2, CellUtil.cloneValue(cell));
+
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ // col2 should not be visible due to conflict
+ assertNull(cell);
+ transactionContext.finish();
+
+ // test that writing to the same column in the same row fails
+ txContext1.start();
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
+
+ txContext2.start();
+ txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ txContext1.finish();
+ try {
+ txContext2.finish();
+ fail("txContext2 should have encountered a row and column level conflict during commit");
+ } catch (TransactionConflictException tce) {
+ txContext2.abort();
+ }
+
+ transactionContext.start();
+ res = transactionAwareHTable.get(new Get(row2));
+ assertFalse(res.isEmpty());
+ cell = res.getColumnLatestCell(TestBytes.family, col2);
+ assertNotNull(cell);
+ // should now be val1
+ assertArrayEquals(val1, CellUtil.cloneValue(cell));
+ transactionContext.finish();
+
+ // verify change set that is being reported only on rows
+ txContext1.start();
+ txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
+ txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
+
+ Collection<byte[]> changeSet = txTable1.getTxChanges();
+ assertNotNull(changeSet);
+ assertEquals(2, changeSet.size());
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
+ assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
+ txContext1.finish();
+ }
+
+ @Test
+ public void testNoneLevelConflictDetection() throws Exception {
+ InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
+
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
+ TxConstants.ConflictDetection.NONE);
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+
+ // overlapping writes to the same row + column should not conflict
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // changes should not be visible yet
+ txContext2.start();
+ Result row = txTable2.get(new Get(TestBytes.row));
+ assertTrue(row.isEmpty());
+
+ txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // both commits should succeed
+ txContext1.finish();
+ txContext2.finish();
+
+ txContext1.start();
+ row = txTable1.get(new Get(TestBytes.row));
+ assertFalse(row.isEmpty());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+ txContext1.finish();
+
+ // transaction abort should still rollback changes
+
+ txContext1.start();
+ txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txContext1.abort();
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row2 = txTable2.get(new Get(TestBytes.row2));
+ assertTrue(row2.isEmpty());
+ txContext2.finish();
+
+ // transaction invalidate should still make changes invisible
+
+ txContext1.start();
+ Transaction tx1 = txContext1.getCurrentTransaction();
+ txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ assertNotNull(tx1);
+ txClient.invalidate(tx1.getWritePointer());
+
+ // changes to row2 should be rolled back
+ txContext2.start();
+ Result row3 = txTable2.get(new Get(TestBytes.row3));
+ assertTrue(row3.isEmpty());
+ txContext2.finish();
+ }
+
+ @Test
+ public void testCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
+ assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
+ long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
+ assertEquals(1, checkpointPtrs.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
+
+ assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
+ assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
+ long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
+ assertEquals(2, checkpointPtrs2.length);
+ assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
+ assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
+
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ // by default, all rows should be visible with Read-Your-Writes
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+
+ // when disabling current write pointer, only the previous checkpoints should be visible
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ Get get = new Get(TestBytes.row);
+ verifyRow(transactionAwareHTable, get, TestBytes.value);
+ get = new Get(TestBytes.row2);
+ verifyRow(transactionAwareHTable, get, TestBytes.value2);
+ get = new Get(TestBytes.row3);
+ verifyRow(transactionAwareHTable, get, null);
+
+ // test scan results excluding current write pointer
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+
+ Result row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNotNull(row);
+ assertArrayEquals(TestBytes.row2, row.getRow());
+ assertEquals(1, row.size());
+ assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
+
+ row = scanner.next();
+ assertNull(row);
+ scanner.close();
+ transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+
+ // commit transaction, verify writes are visible
+ transactionContext.finish();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testInProgressCheckpoint() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+
+ // check that writes are still not visible to other clients
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
+
+ txContext2.start();
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ txContext2.finish();
+ txTable2.close();
+
+ transactionContext.finish();
+
+ // verify writes are visible after commit
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
+ verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointRollback() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ transactionContext.abort();
+
+ transactionContext.start();
+ verifyRow(transactionAwareHTable, TestBytes.row, null);
+ verifyRow(transactionAwareHTable, TestBytes.row2, null);
+ verifyRow(transactionAwareHTable, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = transactionAwareHTable.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ transactionContext.finish();
+ }
+
+ @Test
+ public void testCheckpointInvalidate() throws Exception {
+ // start a transaction, using checkpoints between writes
+ transactionContext.start();
+ Transaction origTx = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ transactionContext.checkpoint();
+ Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
+ transactionContext.checkpoint();
+ Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
+ transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
+
+ // check that writes are not visible
+ TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
+ TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
+ txContext2.start();
+ Transaction newTx = txContext2.getCurrentTransaction();
+
+ // all 3 writes pointers from the previous transaction should now be excluded
+ assertTrue(newTx.isExcluded(origTx.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
+ assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
+
+ verifyRow(txTable2, TestBytes.row, null);
+ verifyRow(txTable2, TestBytes.row2, null);
+ verifyRow(txTable2, TestBytes.row3, null);
+
+ Scan scan = new Scan();
+ ResultScanner scanner = txTable2.getScanner(scan);
+ assertNull(scanner.next());
+ scanner.close();
+ txContext2.finish();
+ }
+
+ @Test
+ public void testExistingData() throws Exception {
+ byte[] val11 = Bytes.toBytes("val11");
+ byte[] val12 = Bytes.toBytes("val12");
+ byte[] val21 = Bytes.toBytes("val21");
+ byte[] val22 = Bytes.toBytes("val22");
+ byte[] val31 = Bytes.toBytes("val31");
+ byte[] val111 = Bytes.toBytes("val111");
+
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
+ Collections.<String>emptyList()));
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // Add some pre-existing, non-transactional data
+ HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
+ nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
+ nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
+ HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
+ nonTxTable.flushCommits();
+
+ // Add transactional data
+ txContext.start();
+ txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
+ txContext.finish();
+
+ txContext.start();
+ // test get
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+
+ // test scan
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+
+ // test update and delete
+ txContext.start();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
+ txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
+ txContext.finish();
+
+ txContext.start();
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
+ verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
+ HConstants.EMPTY_BYTE_ARRAY);
+ verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
+ HConstants.EMPTY_BYTE_ARRAY);
+ txContext.finish();
+
+ // test scan
+ txContext.start();
+ try (ResultScanner scanner = txTable.getScanner(new Scan())) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row2, result.getRow());
+ assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row4, result.getRow());
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
+ TxConstants.FAMILY_DELETE_QUALIFIER));
+ assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
+ assertNull(scanner.next());
+ }
+ txContext.finish();
+ }
+
+ private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
+ verifyRow(table, new Get(rowkey), expectedValue);
+ }
+
+ private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
+ verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
+ }
+
+ private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
+ Result result = table.get(get);
+ if (expectedValues == null) {
+ assertTrue(result.isEmpty());
+ } else {
+ assertFalse(result.isEmpty());
+ byte[] family = TestBytes.family;
+ byte[] col = TestBytes.qualifier;
+ if (get.hasFamilies()) {
+ family = get.getFamilyMap().keySet().iterator().next();
+ col = get.getFamilyMap().get(family).first();
+ }
+ Iterator<Cell> it = result.getColumnCells(family, col).iterator();
+ for (byte[] expectedValue : expectedValues) {
+ Assert.assertTrue(it.hasNext());
+ assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
+ }
+ }
+ }
+
+ private Cell[] getRow(HTableInterface table, Get get) throws Exception {
+ Result result = table.get(get);
+ return result.rawCells();
+ }
+
+ private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
+ List<Cell> actualCells = new ArrayList<>();
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ Result[] results = scanner.next(expectedCells.size() + 1);
+ for (Result result : results) {
+ actualCells.addAll(Lists.newArrayList(result.rawCells()));
+ }
+ Assert.assertEquals(expectedCells, actualCells);
+ }
+ }
+
+ @Test
+ public void testVisibilityAll() throws Exception {
+ HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
+ new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
+ TransactionAwareHTable txTable =
+ new TransactionAwareHTable(nonTxTable,
+ TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
+ TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
+
+ // start a transaction and create a delete marker
+ txContext.start();
+ //noinspection ConstantConditions
+ long txWp0 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
+ txContext.finish();
+
+ // start a new transaction and write some values
+ txContext.start();
+ @SuppressWarnings("ConstantConditions")
+ long txWp1 = txContext.getCurrentTransaction().getWritePointer();
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
+
+ // verify written data
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ TestBytes.value);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ TestBytes.value2);
+
+ // checkpoint and make changes to written data now
+ txContext.checkpoint();
+ long txWp2 = txContext.getCurrentTransaction().getWritePointer();
+ // delete a column
+ txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
+ // no change to a column
+ txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
+ // update a column
+ txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
+ // delete column family
+ txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
+
+ // verify changed values
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
+ null);
+
+ // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
+ // and the raw change by prior transaction
+ //noinspection ConstantConditions
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ List<KeyValue> expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // verify a Get is also able to return all snapshot versions
+ Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
+ Cell[] cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(3, cells.length);
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+ Assert.assertTrue(CellUtil.isDelete(cells[2]));
+
+ verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ ImmutableList.of(TestBytes.value3, TestBytes.value));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
+
+ get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
+ cells = getRow(txTable, get);
+ Assert.assertEquals(2, cells.length);
+ Assert.assertTrue(CellUtil.isDelete(cells[0]));
+ Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
+
+ // Verify VisibilityLevel.SNAPSHOT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
+ new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
+ );
+ verifyScan(txTable, new Scan(), expected);
+ txContext.finish();
+
+ // finally verify values once more after commit, this time we should get only committed raw values for
+ // all visibility levels
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
+ );
+ verifyScan(txTable, new Scan(), expected);
+
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
+ null);
+ verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
+ TestBytes.value2);
+ verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
+ TestBytes.value3);
+ txContext.finish();
+
+ // Test with regular HBase deletes in pre-existing data
+ long now = System.currentTimeMillis();
+ Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteColumn);
+ Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
+ // to prevent Tephra from replacing delete with delete marker
+ deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ nonTxTable.delete(deleteFamily);
+ nonTxTable.flushCommits();
+
+ txContext.start();
+ txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ expected = ImmutableList.of(
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
+ new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
+ new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
+ new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
+ new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
+ new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
+ );
+ // test scan
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ verifyScan(txTable, scan, expected);
+ txContext.finish();
+ }
+
+ @Test
+ public void testFilters() throws Exception {
+ // Add some values to table
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ byte[] val1 = Bytes.toBytes(1L);
+ put.add(TestBytes.family, TestBytes.qualifier, val1);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row2);
+ byte[] val2 = Bytes.toBytes(2L);
+ put.add(TestBytes.family, TestBytes.qualifier, val2);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row3);
+ byte[] val3 = Bytes.toBytes(3L);
+ put.add(TestBytes.family, TestBytes.qualifier, val3);
+ transactionAwareHTable.put(put);
+ put = new Put(TestBytes.row4);
+ byte[] val4 = Bytes.toBytes(4L);
+ put.add(TestBytes.family, TestBytes.qualifier, val4);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Delete cell with value 2
+ transactionContext.start();
+ Delete delete = new Delete(TestBytes.row2);
+ delete.deleteColumn(TestBytes.family, TestBytes.qualifier);
+ transactionAwareHTable.delete(delete);
+ transactionContext.finish();
+
+ // Scan for values less than 4, should get only values 1 and 3
+ transactionContext.start();
+ Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS,
+ new BinaryComparator(Longs.toByteArray(4))));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ Result result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run a Get with a filter for less than 10 on row4, should get value 4
+ transactionContext.start();
+ Get get = new Get(TestBytes.row4);
+ get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(Longs.toByteArray(10))));
+ Result result = transactionAwareHTable.get(get);
+ assertFalse(result.isEmpty());
+ assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
+ transactionContext.finish();
+
+ // Change value of row4 to 40
+ transactionContext.start();
+ put = new Put(TestBytes.row4);
+ byte[] val40 = Bytes.toBytes(40L);
+ put.add(TestBytes.family, TestBytes.qualifier, val40);
+ transactionAwareHTable.put(put);
+ transactionContext.finish();
+
+ // Scan for values less than 10, should get only values 1 and 3
+ transactionContext.start();
+ scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS,
+ new BinaryComparator(Longs.toByteArray(10))));
+ try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row, result.getRow());
+ assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNotNull(result);
+ assertArrayEquals(TestBytes.row3, result.getRow());
+ assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
+ result = scanner.next();
+ assertNull(result);
+ }
+ transactionContext.finish();
+
+ // Run the Get again with a filter for less than 10 on row4, this time should not get any results
+ transactionContext.start();
+ result = transactionAwareHTable.get(get);
+ assertTrue(result.isEmpty());
+ transactionContext.finish();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilterTest.java
new file mode 100644
index 0000000..a1fc0c7
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilterTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * HBase 0.96 specific test for testing {@link CellSkipFilter}.
+ */
+public class CellSkipFilterTest {
+
+ private static final String ROW1KEY = "row1";
+ private static final String ROW2KEY = "row2";
+ private static final String FAM1KEY = "fam1";
+ private static final String COL1KEY = "col1";
+ private static final String FAM2KEY = "fam2";
+ private static final String COL2KEY = "col2";
+ private static final String VALUE = "value";
+
+ @Test
+ public void testSkipFiltering() throws Exception {
+ long timestamp = System.currentTimeMillis();
+ // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
+ Filter filter = new CellSkipFilter(new MyFilter(0));
+ assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give SKIP from MyFilter
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
+ timestamp - 4)));
+
+ // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
+ filter = new CellSkipFilter(new MyFilter(2));
+ assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp)));
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 1)));
+
+ // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 2)));
+
+ // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
+ assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
+ timestamp - 3)));
+
+ // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
+ // give NEXT_ROW from MyFilter
+ assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
+ timestamp - 4)));
+
+ // Next call with the new key should returned the SEEK_NEXT_USING_HINT
+ assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
+ VALUE, timestamp - 5)));
+ }
+
+ private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
+ timestamp, Bytes.toBytes(value));
+ }
+
+ /**
+ * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
+ * start index in the list and start serving the return codes corresponding that that index. Every time the
+ * return code is served, index is incremented.
+ */
+ class MyFilter extends FilterBase {
+
+ private final List<ReturnCode> returnCodes;
+ private int counter;
+
+ public MyFilter(int startIndex) {
+ returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
+ ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
+ counter = startIndex;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ ReturnCode code = returnCodes.get(counter % returnCodes.size());
+ counter++;
+ return code;
+ }
+ }
+}
[07/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java
new file mode 100644
index 0000000..c4efbae
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessorTest.java
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh.coprocessor;
+
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MockRegionServerServices;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.tephra.ChangeId;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionType;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.metrics.TxMetricsCollector;
+import org.apache.tephra.persist.HDFSTransactionStateStorage;
+import org.apache.tephra.persist.TransactionSnapshot;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.snapshot.DefaultSnapshotCodec;
+import org.apache.tephra.snapshot.SnapshotCodecProvider;
+import org.apache.tephra.util.TxUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
+ */
+public class TransactionProcessorTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
+
+ // 8 versions, 1 hour apart, latest is current ts.
+ private static final long[] V;
+
+ static {
+ long now = System.currentTimeMillis();
+ V = new long[9];
+ for (int i = 0; i < V.length; i++) {
+ V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
+ }
+ }
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+ private static MiniDFSCluster dfsCluster;
+ private static Configuration conf;
+ private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
+ private static TransactionVisibilityState txVisibilityState;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration hConf = new Configuration();
+ String rootDir = tmpFolder.newFolder().getAbsolutePath();
+ hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
+ hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
+
+ dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
+ dfsCluster.waitActive();
+ conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
+
+ conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
+ conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
+ String localTestDir = tmpFolder.newFolder().getAbsolutePath();
+ conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
+ conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
+
+ // write an initial transaction snapshot
+ TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(
+ System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
+ // this will set visibility upper bound to V[6]
+ Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
+ TransactionType.SHORT))),
+ new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
+ txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
+ txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
+ txSnapshot.getInProgress());
+ HDFSTransactionStateStorage tmpStorage =
+ new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
+ tmpStorage.startAndWait();
+ tmpStorage.writeSnapshot(txSnapshot);
+ tmpStorage.stopAndWait();
+ }
+
+ @AfterClass
+ public static void shutdownAfterClass() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ @Test
+ public void testDataJanitorRegionScanner() throws Exception {
+ String tableName = "TestRegionScanner";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ for (int i = 1; i <= 8; i++) {
+ for (int k = 1; k <= i; k++) {
+ Put p = new Put(Bytes.toBytes(i));
+ p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
+ region.put(p);
+ }
+ }
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should only return the valid rows
+ // do not use a filter here to test that cleanup works on flush
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+
+ // first returned value should be "4" with version "4"
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 4, new long[] {V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 5, new long[] {V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 6, new long[] {V[6], V[4]});
+
+ results.clear();
+ assertTrue(regionScanner.next(results));
+ assertKeyValueMatches(results, 7, new long[] {V[6], V[4]});
+
+ results.clear();
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteFiltering() throws Exception {
+ String tableName = "TestDeleteFiltering";
+ byte[] familyBytes = Bytes.toBytes("f");
+ byte[] columnBytes = Bytes.toBytes("c");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+ TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
+ LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
+
+ byte[] row = Bytes.toBytes(1);
+ for (int i = 4; i < V.length; i++) {
+ Put p = new Put(row);
+ p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
+ region.put(p);
+ }
+
+ // delete from the third entry back
+ // take that cell's timestamp + 1 to simulate a delete in a new tx
+ long deleteTs = V[5] + 1;
+ Delete d = new Delete(row, deleteTs);
+ LOG.info("Issuing delete at timestamp " + deleteTs);
+ // row deletes are not yet supported (TransactionAwareHTable normally handles this)
+ d.deleteColumns(familyBytes, columnBytes);
+ region.delete(d);
+
+ List<Cell> results = Lists.newArrayList();
+
+ // force a flush to clear the data
+ // during flush, we should drop the deleted version, but not the others
+ LOG.info("Flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // now a normal scan should return row with versions at: V[8], V[6].
+ // V[7] is invalid and V[5] and prior are deleted.
+ Scan scan = new Scan();
+ scan.setMaxVersions(10);
+ RegionScanner regionScanner = region.getScanner(scan);
+ // should be only one row
+ assertFalse(regionScanner.next(results));
+ assertKeyValueMatches(results, 1,
+ new long[]{V[8], V[6], deleteTs},
+ new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testDeleteMarkerCleanup() throws Exception {
+ String tableName = "TestDeleteMarkerCleanup";
+ byte[] familyBytes = Bytes.toBytes("f");
+ HRegion region = createRegion(tableName, familyBytes, 0);
+ try {
+ region.initialize();
+
+ // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
+ long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
+ // deletes are performed after the writes, but still before the visibility upper bound
+ long deleteTs = writeTs + 1;
+ // write separate columns to confirm that delete markers survive across flushes
+ byte[] row = Bytes.toBytes(100);
+ Put p = new Put(row);
+
+ LOG.info("Writing columns at timestamp " + writeTs);
+ for (int i = 0; i < 5; i++) {
+ byte[] iBytes = Bytes.toBytes(i);
+ p.add(familyBytes, iBytes, writeTs, iBytes);
+ }
+ region.put(p);
+ // read all back
+ Scan scan = new Scan(row);
+ RegionScanner regionScanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ for (int i = 0; i < 5; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a flush to clear the memstore
+ LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ // delete the odd entries
+ for (int i = 0; i < 5; i++) {
+ if (i % 2 == 1) {
+ // deletes are performed as puts with empty values
+ Put deletePut = new Put(row);
+ deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
+ region.put(deletePut);
+ }
+ }
+
+ // read all back
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ LOG.info("Got cell " + cell);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force another flush on the delete markers
+ // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
+ LOG.info("After delete, flushing region " + region.getRegionNameAsString());
+ region.flushcache();
+
+ scan = new Scan(row);
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+
+ // force a major compaction
+ LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
+ region.compactStores(true);
+
+ // perform a raw scan (no filter) to confirm that the delete markers are now gone
+ scan = new Scan(row);
+ regionScanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ assertFalse(regionScanner.next(results));
+ assertEquals(3, results.size());
+ // only even columns should exist
+ for (int i = 0; i < 3; i++) {
+ Cell cell = results.get(i);
+ assertArrayEquals(row, cell.getRow());
+ byte[] idxBytes = Bytes.toBytes(i * 2);
+ assertArrayEquals(idxBytes, cell.getQualifier());
+ assertArrayEquals(idxBytes, cell.getValue());
+ }
+ } finally {
+ region.close();
+ }
+ }
+
+ /**
+ * Test that we correctly preserve the timestamp set for column family delete markers. This is not
+ * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
+ * that we make it easy to interoperate with other systems.
+ */
+ @Test
+ public void testFamilyDeleteTimestamp() throws Exception {
+ String tableName = "TestFamilyDeleteTimestamp";
+ byte[] family1Bytes = Bytes.toBytes("f1");
+ byte[] columnBytes = Bytes.toBytes("c");
+ byte[] rowBytes = Bytes.toBytes("row");
+ byte[] valBytes = Bytes.toBytes("val");
+ HRegion region = createRegion(tableName, family1Bytes, 0);
+ try {
+ region.initialize();
+
+ long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+ Put p = new Put(rowBytes);
+ p.add(family1Bytes, columnBytes, now - 10, valBytes);
+ region.put(p);
+
+ // issue a family delete with an explicit timestamp
+ Delete delete = new Delete(rowBytes, now);
+ delete.deleteFamily(family1Bytes, now - 5);
+ region.delete(delete);
+
+ // test that the delete marker preserved the timestamp
+ Scan scan = new Scan();
+ scan.setMaxVersions();
+ RegionScanner scanner = region.getScanner(scan);
+ List<Cell> results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(2, results.size());
+ // delete marker should appear first
+ Cell cell = results.get(0);
+ assertArrayEquals(new byte[0], cell.getQualifier());
+ assertArrayEquals(new byte[0], cell.getValue());
+ assertEquals(now - 5, cell.getTimestamp());
+ // since this is an unfiltered scan against the region, the original put should be next
+ cell = results.get(1);
+ assertArrayEquals(valBytes, cell.getValue());
+ assertEquals(now - 10, cell.getTimestamp());
+ scanner.close();
+
+
+ // with a filtered scan the original put should disappear
+ scan = new Scan();
+ scan.setMaxVersions();
+ scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
+ new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
+ scanner = region.getScanner(scan);
+ results = Lists.newArrayList();
+ scanner.next(results);
+ assertEquals(0, results.size());
+ scanner.close();
+ } finally {
+ region.close();
+ }
+ }
+
+ @Test
+ public void testPreExistingData() throws Exception {
+ String tableName = "TestPreExistingData";
+ byte[] familyBytes = Bytes.toBytes("f");
+ long ttlMillis = TimeUnit.DAYS.toMillis(14);
+ HRegion region = createRegion(tableName, familyBytes, ttlMillis);
+ try {
+ region.initialize();
+
+ // timestamps for pre-existing, non-transactional data
+ long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
+ long older = now - ttlMillis / 2;
+ long newer = now - ttlMillis / 3;
+ // timestamps for transactional data
+ long nowTx = txVisibilityState.getVisibilityUpperBound();
+ long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
+ long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
+
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ ttls.put(familyBytes, ttlMillis);
+
+ List<Cell> cells = new ArrayList<>();
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
+ cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
+ cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
+ cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
+
+ // Write non-transactional and transactional data
+ for (Cell c : cells) {
+ region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
+ }
+
+ Scan rawScan = new Scan();
+ rawScan.setMaxVersions();
+
+ Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
+ Scan txScan = new Scan();
+ txScan.setMaxVersions();
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // read all back with raw scanner
+ scanAndAssert(region, cells, rawScan);
+
+ // read all back with transaction filter
+ scanAndAssert(region, cells, txScan);
+
+ // force a flush to clear the memstore
+ region.flushcache();
+ scanAndAssert(region, cells, txScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ scanAndAssert(region, cells, txScan);
+
+ // Reduce TTL, this should make cells with timestamps older and olderTx expire
+ long newTtl = ttlMillis / 2 - 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // Raw scan should still give all cells
+ scanAndAssert(region, cells, rawScan);
+ // However, tx scan should not return expired cells
+ scanAndAssert(region, select(cells, 1, 3, 5), txScan);
+
+ region.flushcache();
+ scanAndAssert(region, cells, rawScan);
+
+ // force a major compaction to remove any expired cells
+ region.compactStores(true);
+ // This time raw scan too should not return expired cells, as they would be dropped during major compaction
+ scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
+
+ // Reduce TTL again to 1 ms, this should expire all cells
+ newTtl = 1;
+ region = updateTtl(region, familyBytes, newTtl);
+ ttls.put(familyBytes, newTtl);
+ txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
+ TxUtils.getMaxVisibleTimestamp(dummyTransaction));
+ txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
+
+ // force a major compaction to remove expired cells
+ region.compactStores(true);
+ // This time raw scan should not return any cells, as all cells have expired.
+ scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
+ } finally {
+ region.close();
+ }
+ }
+
+ private List<Cell> select(List<Cell> cells, int... indexes) {
+ List<Cell> newCells = new ArrayList<>();
+ for (int i : indexes) {
+ newCells.add(cells.get(i));
+ }
+ return newCells;
+ }
+
+ @SuppressWarnings("StatementWithEmptyBody")
+ private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
+ try (RegionScanner regionScanner = region.getScanner(scan)) {
+ List<Cell> results = Lists.newArrayList();
+ while (regionScanner.next(results)) { }
+ assertEquals(expected, results);
+ }
+ }
+
+ private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
+ region.close();
+ HTableDescriptor htd = region.getTableDesc();
+ HColumnDescriptor cfd = htd.getFamily(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
+ new LocalRegionServerServices(conf, ServerName.valueOf(
+ InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
+ }
+
+ private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
+ HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
+ HColumnDescriptor cfd = new HColumnDescriptor(family);
+ if (ttl > 0) {
+ cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
+ }
+ cfd.setMaxVersions(10);
+ htd.addFamily(cfd);
+ htd.addCoprocessor(TransactionProcessor.class.getName());
+ Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
+ FileSystem fs = FileSystem.get(conf);
+ assertTrue(fs.mkdirs(tablePath));
+ WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
+ WAL hLog = walFactory.getWAL(new byte[]{1});
+ HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
+ HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
+ return new HRegion(regionFS, hLog, conf, htd,
+ new LocalRegionServerServices(conf, ServerName.valueOf(
+ InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
+ byte[][] values = new byte[versions.length][];
+ for (int i = 0; i < versions.length; i++) {
+ values[i] = Bytes.toBytes(versions[i]);
+ }
+ assertKeyValueMatches(results, index, versions, values);
+ }
+
+ private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
+ assertEquals(versions.length, results.size());
+ assertEquals(values.length, results.size());
+ for (int i = 0; i < versions.length; i++) {
+ Cell kv = results.get(i);
+ assertArrayEquals(Bytes.toBytes(index), kv.getRow());
+ assertEquals(versions[i], kv.getTimestamp());
+ assertArrayEquals(values[i], kv.getValue());
+ }
+ }
+
+ @Test
+ public void testTransactionStateCache() throws Exception {
+ TransactionStateCache cache = new TransactionStateCache();
+ cache.setConf(conf);
+ cache.startAndWait();
+ // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
+ TransactionVisibilityState cachedSnapshot = cache.getLatestState();
+ assertNotNull(cachedSnapshot);
+ assertEquals(invalidSet, cachedSnapshot.getInvalid());
+ cache.stopAndWait();
+ }
+
+ private static class LocalRegionServerServices extends MockRegionServerServices {
+ private final ServerName serverName;
+
+ public LocalRegionServerServices(Configuration conf, ServerName serverName) {
+ super(conf);
+ this.serverName = serverName;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return serverName;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java
new file mode 100644
index 0000000..562a8f9
--- /dev/null
+++ b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionVisibilityFilterTest.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10cdh.coprocessor;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.hbase.AbstractTransactionVisibilityFilterTest;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * HBase 1.0 (CDH) specific test for filtering logic applied when reading data transactionally.
+ */
+public class TransactionVisibilityFilterTest extends AbstractTransactionVisibilityFilterTest {
+ /**
+ * Test filtering of KeyValues for in-progress and invalid transactions.
+ * @throws Exception
+ */
+ @Test
+ public void testFiltering() throws Exception {
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+ }
+
+ @Test
+ public void testSubFilter() throws Exception {
+ final FilterBase includeFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE;
+ }
+ };
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter skipFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.SKIP;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ final Filter includeNextFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter nextColFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ }
+
+ @Test
+ public void testSubFilterOverride() throws Exception {
+ final FilterBase includeFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE;
+ }
+ };
+ TxFilterFactory txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE,
+ Filter.ReturnCode.INCLUDE));
+
+ final Filter skipFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.SKIP;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ final Filter includeNextFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
+
+ final Filter nextColFilter = new FilterBase() {
+ @Override
+ public ReturnCode filterKeyValue(Cell ignored) throws IOException {
+ return ReturnCode.NEXT_COL;
+ }
+ };
+ txFilterFactory = new TxFilterFactory() {
+ @Override
+ public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
+ return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+ }
+ };
+ runFilteringTest(txFilterFactory,
+ ImmutableList.of(Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.SKIP,
+ Filter.ReturnCode.NEXT_COL,
+ Filter.ReturnCode.NEXT_COL));
+
+ }
+
+ private void runFilteringTest(TxFilterFactory txFilterFactory,
+ List<Filter.ReturnCode> assertCodes) throws Exception {
+ /*
+ * Start and stop some transactions. This will give us a transaction state something like the following
+ * (numbers only reflect ordering, not actual transaction IDs):
+ * 6 - in progress
+ * 5 - committed
+ * 4 - invalid
+ * 3 - in-progress
+ * 2 - committed
+ * 1 - committed
+ *
+ * read ptr = 5
+ * write ptr = 6
+ */
+
+ Transaction tx1 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx1, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx1));
+
+ Transaction tx2 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx2, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx2));
+
+ Transaction tx3 = txManager.startShort();
+ Transaction tx4 = txManager.startShort();
+ txManager.invalidate(tx4.getTransactionId());
+
+ Transaction tx5 = txManager.startShort();
+ assertTrue(txManager.canCommit(tx5, EMPTY_CHANGESET));
+ assertTrue(txManager.commit(tx5));
+
+ Transaction tx6 = txManager.startShort();
+
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ Filter filter = txFilterFactory.getTxFilter(tx6, ttls);
+
+ assertEquals(assertCodes.get(5),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx6.getTransactionId())));
+ assertEquals(assertCodes.get(4),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx5.getTransactionId())));
+ assertEquals(assertCodes.get(3),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx4.getTransactionId())));
+ assertEquals(assertCodes.get(2),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx3.getTransactionId())));
+ assertEquals(assertCodes.get(1),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx2.getTransactionId())));
+ assertEquals(assertCodes.get(0),
+ filter.filterKeyValue(newKeyValue("row1", "val1", tx1.getTransactionId())));
+ }
+
+ /**
+ * Test filtering for TTL settings.
+ * @throws Exception
+ */
+ @Test
+ public void testTTLFiltering() throws Exception {
+ Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ ttls.put(FAM, 10L);
+ ttls.put(FAM2, 30L);
+ ttls.put(FAM3, 0L);
+
+ Transaction tx = txManager.startShort();
+ long now = tx.getVisibilityUpperBound();
+ Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 21 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 1001 * TxConstants.MAX_TX_PER_MS)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
+
+ // Verify ttl for pre-existing, non-transactional data
+ long preNow = now / TxConstants.MAX_TX_PER_MS;
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 9L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 10L)));
+ assertEquals(Filter.ReturnCode.NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 11L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 9L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 10L)));
+ assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
+ filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 1001L)));
+ }
+
+ protected KeyValue newKeyValue(String rowkey, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), FAM, COL, timestamp, Bytes.toBytes(value));
+ }
+
+ protected KeyValue newKeyValue(String rowkey, byte[] family, String value, long timestamp) {
+ return new KeyValue(Bytes.toBytes(rowkey), family, COL, timestamp, Bytes.toBytes(value));
+ }
+
+ private interface TxFilterFactory {
+ Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs);
+ }
+
+ private class CustomTxFilter extends TransactionVisibilityFilter {
+ public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
+ @Nullable Filter cellFilter) {
+ super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+ }
+
+ @Override
+ protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
+ switch (subFilterCode) {
+ case INCLUDE:
+ return ReturnCode.INCLUDE;
+ case INCLUDE_AND_NEXT_COL:
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ case SKIP:
+ return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
+ default:
+ return subFilterCode;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/HBase10ConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/HBase10ConfigurationProvider.java b/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/HBase10ConfigurationProvider.java
deleted file mode 100644
index a80e572..0000000
--- a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/HBase10ConfigurationProvider.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10;
-
-import co.cask.tephra.util.ConfigurationProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-/**
- * HBase 1.0 version of {@link co.cask.tephra.util.ConfigurationProvider}.
- */
-public class HBase10ConfigurationProvider extends ConfigurationProvider {
- @Override
- public Configuration get() {
- return HBaseConfiguration.create();
- }
-
- @Override
- public Configuration get(Configuration baseConf) {
- return HBaseConfiguration.create(baseConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/SecondaryIndexTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/SecondaryIndexTable.java b/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/SecondaryIndexTable.java
deleted file mode 100644
index 1c9de95..0000000
--- a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/SecondaryIndexTable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase10;
-
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * A Transactional SecondaryIndexTable.
- */
-public class SecondaryIndexTable {
- private byte[] secondaryIndex;
- private TransactionAwareHTable transactionAwareHTable;
- private TransactionAwareHTable secondaryIndexTable;
- private TransactionContext transactionContext;
- private final TableName secondaryIndexTableName;
- private static final byte[] secondaryIndexFamily = Bytes.toBytes("secondaryIndexFamily");
- private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r');
- private static final byte[] DELIMITER = new byte[] {0};
-
- public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, HTableInterface hTable,
- byte[] secondaryIndex) {
- secondaryIndexTableName = TableName.valueOf(hTable.getName().getNameAsString() + ".idx");
- HTable secondaryIndexHTable = null;
- HBaseAdmin hBaseAdmin = null;
- try {
- hBaseAdmin = new HBaseAdmin(hTable.getConfiguration());
- if (!hBaseAdmin.tableExists(secondaryIndexTableName)) {
- hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName));
- }
- secondaryIndexHTable = new HTable(hTable.getConfiguration(), secondaryIndexTableName);
- } catch (Exception e) {
- Throwables.propagate(e);
- } finally {
- try {
- hBaseAdmin.close();
- } catch (Exception e) {
- Throwables.propagate(e);
- }
- }
-
- this.secondaryIndex = secondaryIndex;
- this.transactionAwareHTable = new TransactionAwareHTable(hTable);
- this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable);
- this.transactionContext = new TransactionContext(transactionServiceClient, transactionAwareHTable,
- secondaryIndexTable);
- }
-
- public Result get(Get get) throws IOException {
- return get(Collections.singletonList(get))[0];
- }
-
- public Result[] get(List<Get> gets) throws IOException {
- try {
- transactionContext.start();
- Result[] result = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return result;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public Result[] getByIndex(byte[] value) throws IOException {
- try {
- transactionContext.start();
- Scan scan = new Scan(value, Bytes.add(value, new byte[0]));
- scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier);
- ResultScanner indexScanner = secondaryIndexTable.getScanner(scan);
-
- ArrayList<Get> gets = new ArrayList<Get>();
- for (Result result : indexScanner) {
- for (Cell cell : result.listCells()) {
- gets.add(new Get(cell.getValue()));
- }
- }
- Result[] results = transactionAwareHTable.get(gets);
- transactionContext.finish();
- return results;
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- return null;
- }
-
- public void put(Put put) throws IOException {
- put(Collections.singletonList(put));
- }
-
-
- public void put(List<Put> puts) throws IOException {
- try {
- transactionContext.start();
- ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>();
- for (Put put : puts) {
- List<Put> indexPuts = new ArrayList<Put>();
- Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet();
- for (Map.Entry<byte [], List<KeyValue>> family : familyMap) {
- for (KeyValue value : family.getValue()) {
- if (Bytes.equals(value.getQualifierArray(), value.getQualifierOffset(), value.getQualifierLength(),
- secondaryIndex, 0, secondaryIndex.length)) {
- byte[] secondaryRow = Bytes.add(value.getQualifier(), DELIMITER,
- Bytes.add(value.getValue(), DELIMITER,
- value.getRow()));
- Put indexPut = new Put(secondaryRow);
- indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow());
- indexPuts.add(indexPut);
- }
- }
- }
- secondaryIndexPuts.addAll(indexPuts);
- }
- transactionAwareHTable.put(puts);
- secondaryIndexTable.put(secondaryIndexPuts);
- transactionContext.finish();
- } catch (Exception e) {
- try {
- transactionContext.abort();
- } catch (TransactionFailureException e1) {
- throw new IOException("Could not rollback transaction", e1);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/TransactionAwareHTable.java b/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/TransactionAwareHTable.java
deleted file mode 100644
index cf308a6..0000000
--- a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/TransactionAwareHTable.java
+++ /dev/null
@@ -1,674 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase10;
-
-import co.cask.tephra.AbstractTransactionAwareTable;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TxConstants;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.OperationWithAttributes;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.Batch;
-import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-
-/**
- * A Transaction Aware HTable implementation for HBase 1.0. Operations are committed as usual,
- * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
- * was started.
- */
-public class TransactionAwareHTable extends AbstractTransactionAwareTable
- implements HTableInterface, TransactionAware {
-
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
- private final HTableInterface hTable;
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- */
- public TransactionAwareHTable(HTableInterface hTable) {
- this(hTable, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable
- *
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
- this(hTable, conflictLevel, false);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
- this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
- }
-
- /**
- * Create a transactional aware instance of the passed HTable, with the option
- * of allowing non-transactional operations.
- * @param hTable underlying HBase table to use
- * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
- * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
- * will be available, though non-transactional
- */
- public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
- boolean allowNonTransactional) {
- super(conflictLevel, allowNonTransactional);
- this.hTable = hTable;
- }
-
- /* AbstractTransactionAwareTable implementation */
-
- @Override
- protected byte[] getTableKey() {
- return getTableName();
- }
-
- @Override
- protected boolean doCommit() throws IOException {
- hTable.flushCommits();
- return true;
- }
-
- @Override
- protected boolean doRollback() throws Exception {
- try {
- // pre-size arraylist of deletes
- int size = 0;
- for (Set<ActionChange> cs : changeSets.values()) {
- size += cs.size();
- }
- List<Delete> rollbackDeletes = new ArrayList<>(size);
- for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
- long transactionTimestamp = entry.getKey();
- for (ActionChange change : entry.getValue()) {
- byte[] row = change.getRow();
- byte[] family = change.getFamily();
- byte[] qualifier = change.getQualifier();
- Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- switch (conflictLevel) {
- case ROW:
- case NONE:
- // issue family delete for the tx write pointer
- rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
- break;
- case COLUMN:
- if (family != null && qualifier == null) {
- rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
- } else if (family != null && qualifier != null) {
- rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
- }
- break;
- default:
- throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
- }
- rollbackDeletes.add(rollbackDelete);
- }
- }
- hTable.delete(rollbackDeletes);
- return true;
- } finally {
- try {
- hTable.flushCommits();
- } catch (Exception e) {
- LOG.error("Could not flush HTable commits", e);
- }
- tx = null;
- changeSets.clear();
- }
- }
-
- /* HTableInterface implementation */
-
- @Override
- public byte[] getTableName() {
- return hTable.getTableName();
- }
-
- @Override
- public TableName getName() {
- return hTable.getName();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hTable.getConfiguration();
- }
-
- @Override
- public HTableDescriptor getTableDescriptor() throws IOException {
- return hTable.getTableDescriptor();
- }
-
- @Override
- public boolean exists(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.exists(transactionalizeAction(get));
- }
-
- @Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.exists(transactionalizedGets);
- }
-
- @Override
- public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batch(transactionalizeActions(actions), results);
- }
-
- @Override
- public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batch(transactionalizeActions(actions));
- }
-
- @Override
- public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
- IOException, InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.batchCallback(transactionalizeActions(actions), results, callback);
- }
-
- @Override
- public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
- InterruptedException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.batchCallback(transactionalizeActions(actions), callback);
- }
-
- @Override
- public Result get(Get get) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.get(transactionalizeAction(get));
- }
-
- @Override
- public Result[] get(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.get(transactionalizedGets);
- }
-
- @Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- if (allowNonTransactional) {
- return hTable.getRowOrBefore(row, family);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public ResultScanner getScanner(Scan scan) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addFamily(family);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Scan scan = new Scan();
- scan.addColumn(family, qualifier);
- return hTable.getScanner(transactionalizeAction(scan));
- }
-
- @Override
- public void put(Put put) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- Put txPut = transactionalizeAction(put);
- hTable.put(txPut);
- }
-
- @Override
- public void put(List<Put> puts) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
- for (Put put : puts) {
- Put txPut = transactionalizeAction(put);
- transactionalizedPuts.add(txPut);
- }
- hTable.put(transactionalizedPuts);
- }
-
- @Override
- public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndPut(row, family, qualifier, value, put);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public void delete(Delete delete) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- hTable.delete(transactionalizeAction(delete));
- }
-
- @Override
- public void delete(List<Delete> deletes) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
- for (Delete delete : deletes) {
- Delete txDelete = transactionalizeAction(delete);
- transactionalizedDeletes.add(txDelete);
- }
- hTable.delete(transactionalizedDeletes);
- }
-
- @Override
- public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndDelete(row, family, qualifier, value, delete);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
- byte[] bytes3, Delete delete) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndDelete(bytes, bytes1, bytes2, compareOp, bytes3, delete);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp,
- byte[] bytes3, Put put) throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndPut(bytes, bytes1, bytes2, compareOp, bytes3, put);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean[] existsAll(List<Get> gets) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
- for (Get get : gets) {
- transactionalizedGets.add(transactionalizeAction(get));
- }
- return hTable.existsAll(transactionalizedGets);
- }
-
- @Override
- public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier,
- CompareFilter.CompareOp compareOp, byte[] value, RowMutations rowMutations)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.checkAndMutate(row, family, qualifier, compareOp, value, rowMutations);
- }
-
- throw new UnsupportedOperationException("checkAndMutate operation is not supported transactionally");
- }
-
- @Override
- public void mutateRow(RowMutations rm) throws IOException {
- if (tx == null) {
- throw new IOException("Transaction not started");
- }
- RowMutations transactionalMutations = new RowMutations();
- for (Mutation mutation : rm.getMutations()) {
- if (mutation instanceof Put) {
- transactionalMutations.add(transactionalizeAction((Put) mutation));
- } else if (mutation instanceof Delete) {
- transactionalMutations.add(transactionalizeAction((Delete) mutation));
- }
- }
- hTable.mutateRow(transactionalMutations);
- }
-
- @Override
- public Result append(Append append) throws IOException {
- if (allowNonTransactional) {
- return hTable.append(append);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public Result increment(Increment increment) throws IOException {
- if (allowNonTransactional) {
- return hTable.increment(increment);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
- throws IOException {
- if (allowNonTransactional) {
- return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
- } else {
- throw new UnsupportedOperationException("Operation is not supported transactionally");
- }
- }
-
- @Override
- public boolean isAutoFlush() {
- return hTable.isAutoFlush();
- }
-
- @Override
- public void flushCommits() throws IOException {
- hTable.flushCommits();
- }
-
- @Override
- public void close() throws IOException {
- hTable.close();
- }
-
- @Override
- public CoprocessorRpcChannel coprocessorService(byte[] row) {
- return hTable.coprocessorService(row);
- }
-
- @Override
- public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable)
- throws ServiceException, Throwable {
- return hTable.coprocessorService(service, startKey, endKey, callable);
- }
-
- @Override
- public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
- Batch.Call<T, R> callable, Batch.Callback<R> callback)
- throws ServiceException, Throwable {
- hTable.coprocessorService(service, startKey, endKey, callable, callback);
- }
-
- @Override
- public <R extends Message> Map<byte[], R> batchCoprocessorService(
- MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey,
- R responsePrototype) throws ServiceException, Throwable {
- return hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype);
- }
-
- @Override
- public <R extends Message> void batchCoprocessorService(MethodDescriptor methodDescriptor,
- Message request, byte[] startKey, byte[] endKey, R responsePrototype, Callback<R> callback)
- throws ServiceException, Throwable {
- hTable.batchCoprocessorService(methodDescriptor, request, startKey, endKey, responsePrototype, callback);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush) {
- setAutoFlushTo(autoFlush);
- }
-
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- hTable.setAutoFlush(autoFlush, clearBufferOnFail);
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- hTable.setAutoFlushTo(autoFlush);
- }
-
- @Override
- public long getWriteBufferSize() {
- return hTable.getWriteBufferSize();
- }
-
- @Override
- public void setWriteBufferSize(long writeBufferSize) throws IOException {
- hTable.setWriteBufferSize(writeBufferSize);
- }
-
- // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
-
- private Get transactionalizeAction(Get get) throws IOException {
- addToOperation(get, tx);
- return get;
- }
-
- private Scan transactionalizeAction(Scan scan) throws IOException {
- addToOperation(scan, tx);
- return scan;
- }
-
- private Put transactionalizeAction(Put put) throws IOException {
- Put txPut = new Put(put.getRow(), tx.getWritePointer());
- Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
- if (!familyMap.isEmpty()) {
- for (Map.Entry<byte[], List<Cell>> family : familyMap) {
- List<Cell> familyValues = family.getValue();
- if (!familyValues.isEmpty()) {
- for (Cell value : familyValues) {
- txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
- addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
- txPut.setAttribute(entry.getKey(), entry.getValue());
- }
- txPut.setDurability(put.getDurability());
- addToOperation(txPut, tx);
- return txPut;
- }
-
- private Delete transactionalizeAction(Delete delete) throws IOException {
- long transactionTimestamp = tx.getWritePointer();
-
- byte[] deleteRow = delete.getRow();
- Delete txDelete = new Delete(deleteRow, transactionTimestamp);
-
- Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
- if (familyToDelete.isEmpty()) {
- // perform a row delete if we are using row-level conflict detection
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // Row delete leaves delete markers in all column families of the table
- // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
- for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
- // no need to identify individual columns deleted
- addToChangeSet(deleteRow, columnDescriptor.getName(), null);
- }
- } else {
- Result result = get(new Get(delete.getRow()));
- // Delete everything
- NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
- for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
- }
- }
- }
- } else {
- for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
- byte[] family = familyEntry.getKey();
- List<Cell> entries = familyEntry.getValue();
- boolean isFamilyDelete = false;
- if (entries.size() == 1) {
- Cell cell = entries.get(0);
- isFamilyDelete = CellUtil.isDeleteFamily(cell);
- }
- if (isFamilyDelete) {
- if (conflictLevel == TxConstants.ConflictDetection.ROW ||
- conflictLevel == TxConstants.ConflictDetection.NONE) {
- // no need to identify individual columns deleted
- txDelete.deleteFamily(family);
- addToChangeSet(deleteRow, family, null);
- } else {
- Result result = get(new Get(delete.getRow()).addFamily(family));
- // Delete entire family
- NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
- for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
- txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
- addToChangeSet(deleteRow, family, column.getKey());
- }
- }
- } else {
- for (Cell value : entries) {
- txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
- addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
- }
- }
- }
- }
- for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
- txDelete.setAttribute(entry.getKey(), entry.getValue());
- }
- txDelete.setDurability(delete.getDurability());
- return txDelete;
- }
-
- private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
- List<Row> transactionalizedActions = new ArrayList<>(actions.size());
- for (Row action : actions) {
- if (action instanceof Get) {
- transactionalizedActions.add(transactionalizeAction((Get) action));
- } else if (action instanceof Put) {
- transactionalizedActions.add(transactionalizeAction((Put) action));
- } else if (action instanceof Delete) {
- transactionalizedActions.add(transactionalizeAction((Delete) action));
- } else {
- transactionalizedActions.add(action);
- }
- }
- return transactionalizedActions;
- }
-
- public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
- op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilter.java
deleted file mode 100644
index ab3418a..0000000
--- a/tephra-hbase-compat-1.0/src/main/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilter.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
- * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
- * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
- * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
- * Please see TEPHRA-169 for more details.
- */
-public class CellSkipFilter extends FilterBase {
- private final Filter filter;
- // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
- private KeyValue skipColumn = null;
-
- public CellSkipFilter(Filter filter) {
- this.filter = filter;
- }
-
- /**
- * Determines whether the current cell should be skipped. The cell will be skipped
- * if the previous keyvalue had the same key as the current cell. This means filter already responded
- * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
- * @param cell the {@link Cell} to be tested for skipping
- * @return true is current cell should be skipped, false otherwise
- */
- private boolean skipCellVersion(Cell cell) {
- return skipColumn != null
- && CellUtil.matchingRow(cell, skipColumn.getRowArray(), skipColumn.getRowOffset(),
- skipColumn.getRowLength())
- && CellUtil.matchingFamily(cell, skipColumn.getFamilyArray(), skipColumn.getFamilyOffset(),
- skipColumn.getFamilyLength())
- && CellUtil.matchingQualifier(cell, skipColumn.getQualifierArray(), skipColumn.getQualifierOffset(),
- skipColumn.getQualifierLength());
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- if (skipCellVersion(cell)) {
- return ReturnCode.NEXT_COL;
- }
-
- ReturnCode code = filter.filterKeyValue(cell);
- if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
- // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
- skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
- cell.getFamilyArray(), cell.getFamilyOffset(),
- cell.getFamilyLength(), cell.getQualifierArray(),
- cell.getQualifierOffset(), cell.getQualifierLength());
- } else {
- skipColumn = null;
- }
- return code;
- }
-
- @Override
- public boolean filterRow() throws IOException {
- return filter.filterRow();
- }
-
- @Override
- public Cell transformCell(Cell cell) throws IOException {
- return filter.transformCell(cell);
- }
-
- @Override
- public void reset() throws IOException {
- filter.reset();
- }
-
- @Override
- public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
- return filter.filterRowKey(buffer, offset, length);
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- return filter.filterAllRemaining();
- }
-
- @Override
- public void filterRowCells(List<Cell> kvs) throws IOException {
- filter.filterRowCells(kvs);
- }
-
- @Override
- public boolean hasFilterRow() {
- return filter.hasFilterRow();
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
- return filter.getNextKeyHint(currentKV);
- }
-
- @Override
- public Cell getNextCellHint(Cell currentKV) throws IOException {
- return filter.getNextCellHint(currentKV);
- }
-
- @Override
- public boolean isFamilyEssential(byte[] name) throws IOException {
- return filter.isFamilyEssential(name);
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return filter.toByteArray();
- }
-}
[03/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessorTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessorTest.java b/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessorTest.java
deleted file mode 100644
index 226cc41..0000000
--- a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionProcessorTest.java
+++ /dev/null
@@ -1,620 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10.coprocessor;
-
-import co.cask.tephra.ChangeId;
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionType;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.coprocessor.TransactionStateCache;
-import co.cask.tephra.coprocessor.TransactionStateCacheSupplier;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.HDFSTransactionStateStorage;
-import co.cask.tephra.persist.TransactionSnapshot;
-import co.cask.tephra.persist.TransactionVisibilityState;
-import co.cask.tephra.snapshot.DefaultSnapshotCodec;
-import co.cask.tephra.snapshot.SnapshotCodecProvider;
-import co.cask.tephra.util.TxUtils;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.MockRegionServerServices;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALFactory;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests filtering of invalid transaction data by the {@link TransactionProcessor} coprocessor.
- */
-public class TransactionProcessorTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionProcessorTest.class);
-
- // 8 versions, 1 hour apart, latest is current ts.
- private static final long[] V;
-
- static {
- long now = System.currentTimeMillis();
- V = new long[9];
- for (int i = 0; i < V.length; i++) {
- V[i] = (now - TimeUnit.HOURS.toMillis(8 - i)) * TxConstants.MAX_TX_PER_MS;
- }
- }
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
- private static MiniDFSCluster dfsCluster;
- private static Configuration conf;
- private static LongArrayList invalidSet = new LongArrayList(new long[]{V[3], V[5], V[7]});
- private static TransactionVisibilityState txVisibilityState;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- Configuration hConf = new Configuration();
- String rootDir = tmpFolder.newFolder().getAbsolutePath();
- hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, rootDir);
- hConf.set(HConstants.HBASE_DIR, rootDir + "/hbase");
-
- dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
- dfsCluster.waitActive();
- conf = HBaseConfiguration.create(dfsCluster.getFileSystem().getConf());
-
- conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
- conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
- String localTestDir = tmpFolder.newFolder().getAbsolutePath();
- conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, localTestDir);
- conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());
-
- // write an initial transaction snapshot
- TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(
- System.currentTimeMillis(), V[6] - 1, V[7], invalidSet,
- // this will set visibility upper bound to V[6]
- Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE,
- TransactionType.SHORT))),
- new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
- txVisibilityState = new TransactionSnapshot(txSnapshot.getTimestamp(), txSnapshot.getReadPointer(),
- txSnapshot.getWritePointer(), txSnapshot.getInvalid(),
- txSnapshot.getInProgress());
- HDFSTransactionStateStorage tmpStorage =
- new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
- tmpStorage.startAndWait();
- tmpStorage.writeSnapshot(txSnapshot);
- tmpStorage.stopAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- dfsCluster.shutdown();
- }
-
- @Test
- public void testDataJanitorRegionScanner() throws Exception {
- String tableName = "TestRegionScanner";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3));
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- for (int i = 1; i <= 8; i++) {
- for (int k = 1; k <= i; k++) {
- Put p = new Put(Bytes.toBytes(i));
- p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k]));
- region.put(p);
- }
- }
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should only return the valid rows
- // do not use a filter here to test that cleanup works on flush
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
-
- // first returned value should be "4" with version "4"
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 4, new long[]{V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 5, new long[] {V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 6, new long[]{V[6], V[4]});
-
- results.clear();
- assertTrue(regionScanner.next(results));
- assertKeyValueMatches(results, 7, new long[]{V[6], V[4]});
-
- results.clear();
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteFiltering() throws Exception {
- String tableName = "TestDeleteFiltering";
- byte[] familyBytes = Bytes.toBytes("f");
- byte[] columnBytes = Bytes.toBytes("c");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
- TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get();
- LOG.info("Coprocessor is using transaction state: " + cache.getLatestState());
-
- byte[] row = Bytes.toBytes(1);
- for (int i = 4; i < V.length; i++) {
- Put p = new Put(row);
- p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i]));
- region.put(p);
- }
-
- // delete from the third entry back
- // take that cell's timestamp + 1 to simulate a delete in a new tx
- long deleteTs = V[5] + 1;
- Delete d = new Delete(row, deleteTs);
- LOG.info("Issuing delete at timestamp " + deleteTs);
- // row deletes are not yet supported (TransactionAwareHTable normally handles this)
- d.deleteColumns(familyBytes, columnBytes);
- region.delete(d);
-
- List<Cell> results = Lists.newArrayList();
-
- // force a flush to clear the data
- // during flush, we should drop the deleted version, but not the others
- LOG.info("Flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // now a normal scan should return row with versions at: V[8], V[6].
- // V[7] is invalid and V[5] and prior are deleted.
- Scan scan = new Scan();
- scan.setMaxVersions(10);
- RegionScanner regionScanner = region.getScanner(scan);
- // should be only one row
- assertFalse(regionScanner.next(results));
- assertKeyValueMatches(results, 1,
- new long[]{V[8], V[6], deleteTs},
- new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]});
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testDeleteMarkerCleanup() throws Exception {
- String tableName = "TestDeleteMarkerCleanup";
- byte[] familyBytes = Bytes.toBytes("f");
- HRegion region = createRegion(tableName, familyBytes, 0);
- try {
- region.initialize();
-
- // all puts use a timestamp before the tx snapshot's visibility upper bound, making them eligible for removal
- long writeTs = txVisibilityState.getVisibilityUpperBound() - 10;
- // deletes are performed after the writes, but still before the visibility upper bound
- long deleteTs = writeTs + 1;
- // write separate columns to confirm that delete markers survive across flushes
- byte[] row = Bytes.toBytes(100);
- Put p = new Put(row);
-
- LOG.info("Writing columns at timestamp " + writeTs);
- for (int i = 0; i < 5; i++) {
- byte[] iBytes = Bytes.toBytes(i);
- p.add(familyBytes, iBytes, writeTs, iBytes);
- }
- region.put(p);
- // read all back
- Scan scan = new Scan(row);
- RegionScanner regionScanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- for (int i = 0; i < 5; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a flush to clear the memstore
- LOG.info("Before delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- // delete the odd entries
- for (int i = 0; i < 5; i++) {
- if (i % 2 == 1) {
- // deletes are performed as puts with empty values
- Put deletePut = new Put(row);
- deletePut.add(familyBytes, Bytes.toBytes(i), deleteTs, new byte[0]);
- region.put(deletePut);
- }
- }
-
- // read all back
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- LOG.info("Got cell " + cell);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force another flush on the delete markers
- // during flush, we should retain the delete markers, since they can only safely be dropped by a major compaction
- LOG.info("After delete, flushing region " + region.getRegionNameAsString());
- region.flushcache();
-
- scan = new Scan(row);
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
-
- // force a major compaction
- LOG.info("Forcing major compaction of region " + region.getRegionNameAsString());
- region.compactStores(true);
-
- // perform a raw scan (no filter) to confirm that the delete markers are now gone
- scan = new Scan(row);
- regionScanner = region.getScanner(scan);
- results = Lists.newArrayList();
- assertFalse(regionScanner.next(results));
- assertEquals(3, results.size());
- // only even columns should exist
- for (int i = 0; i < 3; i++) {
- Cell cell = results.get(i);
- assertArrayEquals(row, cell.getRow());
- byte[] idxBytes = Bytes.toBytes(i * 2);
- assertArrayEquals(idxBytes, cell.getQualifier());
- assertArrayEquals(idxBytes, cell.getValue());
- }
- } finally {
- region.close();
- }
- }
-
- /**
- * Test that we correctly preserve the timestamp set for column family delete markers. This is not
- * directly required for the TransactionAwareHTable usage, but is the right thing to do and ensures
- * that we make it easy to interoperate with other systems.
- */
- @Test
- public void testFamilyDeleteTimestamp() throws Exception {
- String tableName = "TestFamilyDeleteTimestamp";
- byte[] family1Bytes = Bytes.toBytes("f1");
- byte[] columnBytes = Bytes.toBytes("c");
- byte[] rowBytes = Bytes.toBytes("row");
- byte[] valBytes = Bytes.toBytes("val");
- HRegion region = createRegion(tableName, family1Bytes, 0);
- try {
- region.initialize();
-
- long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
- Put p = new Put(rowBytes);
- p.add(family1Bytes, columnBytes, now - 10, valBytes);
- region.put(p);
-
- // issue a family delete with an explicit timestamp
- Delete delete = new Delete(rowBytes, now);
- delete.deleteFamily(family1Bytes, now - 5);
- region.delete(delete);
-
- // test that the delete marker preserved the timestamp
- Scan scan = new Scan();
- scan.setMaxVersions();
- RegionScanner scanner = region.getScanner(scan);
- List<Cell> results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(2, results.size());
- // delete marker should appear first
- Cell cell = results.get(0);
- assertArrayEquals(new byte[0], cell.getQualifier());
- assertArrayEquals(new byte[0], cell.getValue());
- assertEquals(now - 5, cell.getTimestamp());
- // since this is an unfiltered scan against the region, the original put should be next
- cell = results.get(1);
- assertArrayEquals(valBytes, cell.getValue());
- assertEquals(now - 10, cell.getTimestamp());
- scanner.close();
-
-
- // with a filtered scan the original put should disappear
- scan = new Scan();
- scan.setMaxVersions();
- scan.setFilter(TransactionFilters.getVisibilityFilter(TxUtils.createDummyTransaction(txVisibilityState),
- new TreeMap<byte[], Long>(), false, ScanType.USER_SCAN));
- scanner = region.getScanner(scan);
- results = Lists.newArrayList();
- scanner.next(results);
- assertEquals(0, results.size());
- scanner.close();
- } finally {
- region.close();
- }
- }
-
- @Test
- public void testPreExistingData() throws Exception {
- String tableName = "TestPreExistingData";
- byte[] familyBytes = Bytes.toBytes("f");
- long ttlMillis = TimeUnit.DAYS.toMillis(14);
- HRegion region = createRegion(tableName, familyBytes, ttlMillis);
- try {
- region.initialize();
-
- // timestamps for pre-existing, non-transactional data
- long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS;
- long older = now - ttlMillis / 2;
- long newer = now - ttlMillis / 3;
- // timestamps for transactional data
- long nowTx = txVisibilityState.getVisibilityUpperBound();
- long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS;
- long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS;
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(familyBytes, ttlMillis);
-
- List<Cell> cells = new ArrayList<>();
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11")));
- cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21")));
- cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31")));
- cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32")));
-
- // Write non-transactional and transactional data
- for (Cell c : cells) {
- region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue()));
- }
-
- Scan rawScan = new Scan();
- rawScan.setMaxVersions();
-
- Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState);
- Scan txScan = new Scan();
- txScan.setMaxVersions();
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // read all back with raw scanner
- scanAndAssert(region, cells, rawScan);
-
- // read all back with transaction filter
- scanAndAssert(region, cells, txScan);
-
- // force a flush to clear the memstore
- region.flushcache();
- scanAndAssert(region, cells, txScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- scanAndAssert(region, cells, txScan);
-
- // Reduce TTL, this should make cells with timestamps older and olderTx expire
- long newTtl = ttlMillis / 2 - 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // Raw scan should still give all cells
- scanAndAssert(region, cells, rawScan);
- // However, tx scan should not return expired cells
- scanAndAssert(region, select(cells, 1, 3, 5), txScan);
-
- region.flushcache();
- scanAndAssert(region, cells, rawScan);
-
- // force a major compaction to remove any expired cells
- region.compactStores(true);
- // This time raw scan too should not return expired cells, as they would be dropped during major compaction
- scanAndAssert(region, select(cells, 1, 3, 5), rawScan);
-
- // Reduce TTL again to 1 ms, this should expire all cells
- newTtl = 1;
- region = updateTtl(region, familyBytes, newTtl);
- ttls.put(familyBytes, newTtl);
- txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true),
- TxUtils.getMaxVisibleTimestamp(dummyTransaction));
- txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN));
-
- // force a major compaction to remove expired cells
- region.compactStores(true);
- // This time raw scan should not return any cells, as all cells have expired.
- scanAndAssert(region, Collections.<Cell>emptyList(), rawScan);
- } finally {
- region.close();
- }
- }
-
- private List<Cell> select(List<Cell> cells, int... indexes) {
- List<Cell> newCells = new ArrayList<>();
- for (int i : indexes) {
- newCells.add(cells.get(i));
- }
- return newCells;
- }
-
- @SuppressWarnings("StatementWithEmptyBody")
- private void scanAndAssert(HRegion region, List<Cell> expected, Scan scan) throws Exception {
- try (RegionScanner regionScanner = region.getScanner(scan)) {
- List<Cell> results = Lists.newArrayList();
- while (regionScanner.next(results)) { }
- assertEquals(expected, results);
- }
- }
-
- private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
- region.close();
- HTableDescriptor htd = region.getTableDesc();
- HColumnDescriptor cfd = htd.getFamily(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
- new LocalRegionServerServices(conf, ServerName.valueOf(
- InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
- }
-
- private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
- HColumnDescriptor cfd = new HColumnDescriptor(family);
- if (ttl > 0) {
- cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
- }
- cfd.setMaxVersions(10);
- htd.addFamily(cfd);
- htd.addCoprocessor(TransactionProcessor.class.getName());
- Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
- FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.mkdirs(tablePath));
- WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
- WAL hLog = walFactory.getWAL(new byte[]{1});
- HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
- HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
- return new HRegion(regionFS, hLog, conf, htd,
- new LocalRegionServerServices(conf, ServerName.valueOf(
- InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions) {
- byte[][] values = new byte[versions.length][];
- for (int i = 0; i < versions.length; i++) {
- values[i] = Bytes.toBytes(versions[i]);
- }
- assertKeyValueMatches(results, index, versions, values);
- }
-
- private void assertKeyValueMatches(List<Cell> results, int index, long[] versions, byte[][] values) {
- assertEquals(versions.length, results.size());
- assertEquals(values.length, results.size());
- for (int i = 0; i < versions.length; i++) {
- Cell kv = results.get(i);
- assertArrayEquals(Bytes.toBytes(index), kv.getRow());
- assertEquals(versions[i], kv.getTimestamp());
- assertArrayEquals(values[i], kv.getValue());
- }
- }
-
- @Test
- public void testTransactionStateCache() throws Exception {
- TransactionStateCache cache = new TransactionStateCache();
- cache.setConf(conf);
- cache.startAndWait();
- // verify that the transaction snapshot read matches what we wrote in setupBeforeClass()
- TransactionVisibilityState cachedSnapshot = cache.getLatestState();
- assertNotNull(cachedSnapshot);
- assertEquals(invalidSet, cachedSnapshot.getInvalid());
- cache.stopAndWait();
- }
-
- private static class LocalRegionServerServices extends MockRegionServerServices {
- private final ServerName serverName;
-
- public LocalRegionServerServices(Configuration conf, ServerName serverName) {
- super(conf);
- this.serverName = serverName;
- }
-
- @Override
- public ServerName getServerName() {
- return serverName;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilterTest.java
deleted file mode 100644
index 87fbeeb..0000000
--- a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/TransactionVisibilityFilterTest.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10.coprocessor;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase.AbstractTransactionVisibilityFilterTest;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * HBase 1.0 specific test for filtering logic applied when reading data transactionally.
- */
-public class TransactionVisibilityFilterTest extends AbstractTransactionVisibilityFilterTest {
- /**
- * Test filtering of KeyValues for in-progress and invalid transactions.
- * @throws Exception
- */
- @Test
- public void testFiltering() throws Exception {
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
- }
-
- @Test
- public void testSubFilter() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- @Test
- public void testSubFilterOverride() throws Exception {
- final FilterBase includeFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE;
- }
- };
- TxFilterFactory txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE,
- Filter.ReturnCode.INCLUDE));
-
- final Filter skipFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.SKIP;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- final Filter includeNextFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- Filter.ReturnCode.INCLUDE_AND_NEXT_COL));
-
- final Filter nextColFilter = new FilterBase() {
- @Override
- public ReturnCode filterKeyValue(Cell ignored) throws IOException {
- return ReturnCode.NEXT_COL;
- }
- };
- txFilterFactory = new TxFilterFactory() {
- @Override
- public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
- return new CustomTxFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
- }
- };
- runFilteringTest(txFilterFactory,
- ImmutableList.of(Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.SKIP,
- Filter.ReturnCode.NEXT_COL,
- Filter.ReturnCode.NEXT_COL));
-
- }
-
- private void runFilteringTest(TxFilterFactory txFilterFactory,
- List<Filter.ReturnCode> assertCodes) throws Exception {
- /*
- * Start and stop some transactions. This will give us a transaction state something like the following
- * (numbers only reflect ordering, not actual transaction IDs):
- * 6 - in progress
- * 5 - committed
- * 4 - invalid
- * 3 - in-progress
- * 2 - committed
- * 1 - committed
- *
- * read ptr = 5
- * write ptr = 6
- */
-
- Transaction tx1 = txManager.startShort();
- assertTrue(txManager.canCommit(tx1, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx1));
-
- Transaction tx2 = txManager.startShort();
- assertTrue(txManager.canCommit(tx2, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx2));
-
- Transaction tx3 = txManager.startShort();
- Transaction tx4 = txManager.startShort();
- txManager.invalidate(tx4.getTransactionId());
-
- Transaction tx5 = txManager.startShort();
- assertTrue(txManager.canCommit(tx5, EMPTY_CHANGESET));
- assertTrue(txManager.commit(tx5));
-
- Transaction tx6 = txManager.startShort();
-
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- Filter filter = txFilterFactory.getTxFilter(tx6, ttls);
-
- assertEquals(assertCodes.get(5),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx6.getTransactionId())));
- assertEquals(assertCodes.get(4),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx5.getTransactionId())));
- assertEquals(assertCodes.get(3),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx4.getTransactionId())));
- assertEquals(assertCodes.get(2),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx3.getTransactionId())));
- assertEquals(assertCodes.get(1),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx2.getTransactionId())));
- assertEquals(assertCodes.get(0),
- filter.filterKeyValue(newKeyValue("row1", "val1", tx1.getTransactionId())));
- }
-
- /**
- * Test filtering for TTL settings.
- * @throws Exception
- */
- @Test
- public void testTTLFiltering() throws Exception {
- Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- ttls.put(FAM, 10L);
- ttls.put(FAM2, 30L);
- ttls.put(FAM3, 0L);
-
- Transaction tx = txManager.startShort();
- long now = tx.getVisibilityUpperBound();
- Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 11 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 21 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM2, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 31 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", now - 1001 * TxConstants.MAX_TX_PER_MS)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row2", FAM, "val1", now - 1 * TxConstants.MAX_TX_PER_MS)));
-
- // Verify ttl for pre-existing, non-transactional data
- long preNow = now / TxConstants.MAX_TX_PER_MS;
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM, "val1", preNow - 11L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 9L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 10L)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
- filter.filterKeyValue(newKeyValue("row1", FAM3, "val1", preNow - 1001L)));
- }
-
- protected KeyValue newKeyValue(String rowkey, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), FAM, COL, timestamp, Bytes.toBytes(value));
- }
-
- protected KeyValue newKeyValue(String rowkey, byte[] family, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), family, COL, timestamp, Bytes.toBytes(value));
- }
-
- private interface TxFilterFactory {
- Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs);
- }
-
- private class CustomTxFilter extends TransactionVisibilityFilter {
- public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
- @Nullable Filter cellFilter) {
- super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
- }
-
- @Override
- protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
- switch (subFilterCode) {
- case INCLUDE:
- return ReturnCode.INCLUDE;
- case INCLUDE_AND_NEXT_COL:
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- case SKIP:
- return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
- default:
- return subFilterCode;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/HBase10ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/HBase10ConfigurationProviderTest.java b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/HBase10ConfigurationProviderTest.java
new file mode 100644
index 0000000..338c393
--- /dev/null
+++ b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/HBase10ConfigurationProviderTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase10;
+
+import org.apache.tephra.util.AbstractConfigurationProviderTest;
+import org.apache.tephra.util.HBaseVersion;
+
+/**
+ * Test for HBase 1.0 version specific behavior.
+ */
+public class HBase10ConfigurationProviderTest extends AbstractConfigurationProviderTest {
+ @Override
+ protected HBaseVersion.Version getExpectedVersion() {
+ return HBaseVersion.Version.HBASE_10;
+ }
+}
[04/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/TransactionAwareHTableTest.java b/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/TransactionAwareHTableTest.java
deleted file mode 100644
index 44b3a75..0000000
--- a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/TransactionAwareHTableTest.java
+++ /dev/null
@@ -1,1552 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package co.cask.tephra.hbase10;
-
-import co.cask.tephra.Transaction;
-import co.cask.tephra.TransactionConflictException;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.TxConstants;
-import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import co.cask.tephra.metrics.TxMetricsCollector;
-import co.cask.tephra.persist.InMemoryTransactionStateStorage;
-import co.cask.tephra.persist.TransactionStateStorage;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.LongComparator;
-import org.apache.hadoop.hbase.filter.ValueFilter;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Tests for TransactionAwareHTables.
- */
-public class TransactionAwareHTableTest {
- private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTableTest.class);
-
- private static HBaseTestingUtility testUtil;
- private static HBaseAdmin hBaseAdmin;
- private static TransactionStateStorage txStateStorage;
- private static TransactionManager txManager;
- private static Configuration conf;
- private TransactionContext transactionContext;
- private TransactionAwareHTable transactionAwareHTable;
- private HTable hTable;
-
- private static final class TestBytes {
- private static final byte[] table = Bytes.toBytes("testtable");
- private static final byte[] family = Bytes.toBytes("f1");
- private static final byte[] family2 = Bytes.toBytes("f2");
- private static final byte[] qualifier = Bytes.toBytes("col1");
- private static final byte[] qualifier2 = Bytes.toBytes("col2");
- private static final byte[] row = Bytes.toBytes("row");
- private static final byte[] row2 = Bytes.toBytes("row2");
- private static final byte[] row3 = Bytes.toBytes("row3");
- private static final byte[] row4 = Bytes.toBytes("row4");
- private static final byte[] value = Bytes.toBytes("value");
- private static final byte[] value2 = Bytes.toBytes("value2");
- private static final byte[] value3 = Bytes.toBytes("value3");
- }
-
- private static final String TEST_ATTRIBUTE = "TEST_ATTRIBUTE";
-
- public static class TestRegionObserver extends BaseRegionObserver {
- @Override
- public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Put put, final WALEdit edit,
- final Durability durability) throws IOException {
- if (put.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Put should preserve attributes");
- }
- if (put.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
-
- @Override
- public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
- final Delete delete, final WALEdit edit,
- final Durability durability) throws IOException {
- if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
- throw new DoNotRetryIOException("Delete should preserve attributes");
- }
- if (delete.getDurability() != Durability.USE_DEFAULT) {
- throw new DoNotRetryIOException("Durability is not propagated correctly");
- }
- }
- }
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- testUtil = new HBaseTestingUtility();
- conf = testUtil.getConfiguration();
-
- // Tune down the connection thread pool size
- conf.setInt("hbase.hconnection.threads.core", 5);
- conf.setInt("hbase.hconnection.threads.max", 10);
- // Tunn down handler threads in regionserver
- conf.setInt("hbase.regionserver.handler.count", 10);
-
- // Set to random port
- conf.setInt("hbase.master.port", 0);
- conf.setInt("hbase.master.info.port", 0);
- conf.setInt("hbase.regionserver.port", 0);
- conf.setInt("hbase.regionserver.info.port", 0);
-
- testUtil.startMiniCluster();
- hBaseAdmin = testUtil.getHBaseAdmin();
- txStateStorage = new InMemoryTransactionStateStorage();
- txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
- txManager.startAndWait();
- }
-
- @AfterClass
- public static void shutdownAfterClass() throws Exception {
- testUtil.shutdownMiniCluster();
- hBaseAdmin.close();
- }
-
- @Before
- public void setupBeforeTest() throws Exception {
- hTable = createTable(TestBytes.table, new byte[][]{TestBytes.family});
- transactionAwareHTable = new TransactionAwareHTable(hTable);
- transactionContext = new TransactionContext(new InMemoryTxSystemClient(txManager), transactionAwareHTable);
- }
-
- @After
- public void shutdownAfterTest() throws IOException {
- hBaseAdmin.disableTable(TestBytes.table);
- hBaseAdmin.deleteTable(TestBytes.table);
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
- return createTable(tableName, columnFamilies, false, Collections.<String>emptyList());
- }
-
- private HTable createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
- List<String> coprocessors) throws Exception {
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
- for (byte[] family : columnFamilies) {
- HColumnDescriptor columnDesc = new HColumnDescriptor(family);
- columnDesc.setMaxVersions(Integer.MAX_VALUE);
- columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
- desc.addFamily(columnDesc);
- }
- if (existingData) {
- desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
- }
- // Divide individually to prevent any overflow
- int priority = Coprocessor.PRIORITY_USER;
- desc.addCoprocessor(TransactionProcessor.class.getName(), null, priority, null);
- // order in list is the same order that coprocessors will be invoked
- for (String coprocessor : coprocessors) {
- desc.addCoprocessor(coprocessor, null, ++priority, null);
- }
- hBaseAdmin.createTable(desc);
- testUtil.waitTableAvailable(tableName, 5000);
- return new HTable(testUtil.getConfiguration(), tableName);
- }
-
- /**
- * Test transactional put and get requests.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
-
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- /**
- * Test aborted put requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionPutAndGet() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
-
- transactionContext.abort();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(value, null);
- }
-
- /**
- * Test transactional delete operations.
- *
- * @throws Exception
- */
- @Test
- public void testValidTransactionalDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestValidTransactionalDelete"),
- new byte[][]{TestBytes.family, TestBytes.family2});
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
-
- // test column delete
- // load 10 rows
- txContext.start();
- int rowCount = 10;
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
- }
- txTable.put(p);
- }
- txContext.finish();
-
- // verify loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertFalse(r.isEmpty());
- for (int j = 0; j < 10; j++) {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
- }
- }
- txContext.finish();
-
- // delete odds columns from odd rows and even columns from even rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("row" + i));
- for (int j = 0; j < 10; j++) {
- if (i % 2 == j % 2) {
- LOG.info("Deleting row={}, column={}", i, j);
- d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
- }
- }
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted columns
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("row" + i));
- Result r = txTable.get(g);
- assertEquals(5, r.size());
- for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
- int col = Bytes.toInt(entry.getKey());
- LOG.info("Got row={}, col={}", i, col);
- // each row should only have the opposite mod (odd=even, even=odd)
- assertNotEquals(i % 2, col % 2);
- assertArrayEquals(TestBytes.value, entry.getValue());
- }
- }
- txContext.finish();
-
- // test family delete
- // load 10 rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Put p = new Put(Bytes.toBytes("famrow" + i));
- p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
- txTable.put(p);
- }
- txContext.finish();
-
- // verify all loaded rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(2, r.size());
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- txContext.finish();
-
- // delete family1 for even rows, family2 for odd rows
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Delete d = new Delete(Bytes.toBytes("famrow" + i));
- d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
- txTable.delete(d);
- }
- txContext.finish();
-
- // verify deleted families
- txContext.start();
- for (int i = 0; i < rowCount; i++) {
- Get g = new Get(Bytes.toBytes("famrow" + i));
- Result r = txTable.get(g);
- assertEquals(1, r.size());
- if (i % 2 == 0) {
- assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
- } else {
- assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
- }
- }
- txContext.finish();
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test that put and delete attributes are preserved
- *
- * @throws Exception
- */
- @Test
- public void testAttributesPreserved() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestAttributesPreserved"),
- new byte[][]{TestBytes.family, TestBytes.family2}, false,
- Lists.newArrayList(TestRegionObserver.class.getName()));
- try {
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- txContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
- // set an attribute on the put, TestRegionObserver will verify it still exists
- put.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.put(put);
- txContext.finish();
-
- txContext.start();
- Result result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- value = result.getValue(TestBytes.family2, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value2, value);
-
- // test full row delete, TestRegionObserver will verify it still exists
- txContext.start();
- Delete delete = new Delete(TestBytes.row);
- delete.setAttribute(TEST_ATTRIBUTE, new byte[]{});
- txTable.delete(delete);
- txContext.finish();
-
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- txContext.finish();
- assertTrue(result.isEmpty());
- } finally {
- hTable.close();
- }
- }
-
- /**
- * Test aborted transactional delete requests, that must be rolled back.
- *
- * @throws Exception
- */
- @Test
- public void testAbortedTransactionalDelete() throws Exception {
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- transactionContext.start();
- Result result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
-
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row);
- transactionAwareHTable.delete(delete);
- transactionContext.abort();
-
- transactionContext.start();
- result = transactionAwareHTable.get(new Get(TestBytes.row));
- transactionContext.finish();
- value = result.getValue(TestBytes.family, TestBytes.qualifier);
- assertArrayEquals(TestBytes.value, value);
- }
-
- private void testDeleteRollback(TxConstants.ConflictDetection conflictDetection) throws Exception {
- String tableName = String.format("%s%s", "TestColFamilyDelete", conflictDetection);
- HTable hTable = createTable(Bytes.toBytes(tableName), new byte[][]{TestBytes.family});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, conflictDetection)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- // Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- // Start a tx, delete a column family and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family));
- txContext.abort();
-
- // Above operations should have no effect on the row, since they were aborted
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
- }
- }
-
- @Test
- public void testDeleteRollback() throws Exception {
- testDeleteRollback(TxConstants.ConflictDetection.ROW);
- testDeleteRollback(TxConstants.ConflictDetection.COLUMN);
- testDeleteRollback(TxConstants.ConflictDetection.NONE);
- }
-
- @Test
- public void testMultiColumnFamilyRowDeleteRollback() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestMultColFam"), new byte[][] {TestBytes.family, TestBytes.family2});
- try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext.finish();
-
- txContext.start();
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- Result result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
-
- //Start a tx, delete the row and then abort the tx
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.abort();
-
- //Start a tx and scan all the col families to make sure none of them have delete markers
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- result = txTable.get(new Get(TestBytes.row));
- Assert.assertEquals(1, result.getFamilyMap(TestBytes.family).size());
- Assert.assertEquals(0, result.getFamilyMap(TestBytes.family2).size());
- txContext.finish();
- }
- }
-
- @Test
- public void testRowDelete() throws Exception {
- HTable hTable = createTable(Bytes.toBytes("TestRowDelete"), new byte[][]{TestBytes.family, TestBytes.family2});
- TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW);
- try {
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Test 1: full row delete
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- Get get = new Get(TestBytes.row);
- Result result = txTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // delete entire row
- txContext.start();
- txTable.delete(new Delete(TestBytes.row));
- txContext.finish();
-
- // verify row is now empty
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertTrue(result.isEmpty());
-
- // verify row is empty for explicit column retrieval
- result = txTable.get(new Get(TestBytes.row)
- .addColumn(TestBytes.family, TestBytes.qualifier)
- .addFamily(TestBytes.family2));
- assertTrue(result.isEmpty());
-
- // verify row is empty for scan
- ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
- assertNull(scanner.next());
- scanner.close();
-
- // verify row is empty for scan with explicit column
- scanner = txTable.getScanner(new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
- assertNull(scanner.next());
- scanner.close();
- txContext.finish();
-
- // write swapped values to one column per family
- txContext.start();
- txTable.put(new Put(TestBytes.row)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
- txContext.finish();
-
- // verify new values appear
- txContext.start();
- result = txTable.get(new Get(TestBytes.row));
- assertFalse(result.isEmpty());
- assertEquals(2, result.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
-
- scanner = txTable.getScanner(new Scan(TestBytes.row));
- Result result1 = scanner.next();
- assertNotNull(result1);
- assertFalse(result1.isEmpty());
- assertEquals(2, result1.size());
- assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
- scanner.close();
- txContext.finish();
-
- // Test 2: delete of first column family
- txContext.start();
- txTable.put(new Put(TestBytes.row2)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
- txContext.finish();
-
- txContext.start();
- Result fam1Result = txTable.get(new Get(TestBytes.row2));
- assertFalse(fam1Result.isEmpty());
- assertEquals(2, fam1Result.size());
- assertArrayEquals(TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 3: delete of second column family
- txContext.start();
- txTable.put(new Put(TestBytes.row3)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
- .add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txContext.finish();
-
- txContext.start();
- txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
- txContext.finish();
-
- txContext.start();
- Result fam2Result = txTable.get(new Get(TestBytes.row3));
- assertFalse(fam2Result.isEmpty());
- assertEquals(2, fam2Result.size());
- assertArrayEquals(TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // Test 4: delete specific rows in a range
- txContext.start();
- for (int i = 0; i < 10; i++) {
- txTable.put(new Put(Bytes.toBytes("z" + i))
- .add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
- .add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
- }
- txContext.finish();
-
- txContext.start();
- // delete odd rows
- for (int i = 1; i < 10; i += 2) {
- txTable.delete(new Delete(Bytes.toBytes("z" + i)));
- }
- txContext.finish();
-
- txContext.start();
- int cnt = 0;
- ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
- Result res;
- while ((res = zScanner.next()) != null) {
- assertFalse(res.isEmpty());
- assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
- cnt += 2;
- }
-
- // Test 5: delete prior writes in the same transaction
- txContext.start();
- txTable.put(new Put(TestBytes.row4)
- .add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
- .add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
- txTable.delete(new Delete(TestBytes.row4));
- txContext.finish();
-
- txContext.start();
- Result row4Result = txTable.get(new Get(TestBytes.row4));
- assertTrue(row4Result.isEmpty());
- txContext.finish();
- } finally {
- txTable.close();
- }
- }
-
- /**
- * Expect an exception since a transaction hasn't been started.
- *
- * @throws Exception
- */
- @Test(expected = IOException.class)
- public void testTransactionlessFailure() throws Exception {
- transactionAwareHTable.get(new Get(TestBytes.row));
- }
-
- /**
- * Tests that each transaction can see its own persisted writes, while not seeing writes from other
- * in-progress transactions.
- */
- @Test
- public void testReadYourWrites() throws Exception {
- // In-progress tx1: started before our main transaction
- HTable hTable1 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable1 = new TransactionAwareHTable(hTable1);
- TransactionContext inprogressTxContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable1);
-
- // In-progress tx2: started while our main transaction is running
- HTable hTable2 = new HTable(testUtil.getConfiguration(), TestBytes.table);
- TransactionAwareHTable txHTable2 = new TransactionAwareHTable(hTable2);
- TransactionContext inprogressTxContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txHTable2);
-
- // create an in-progress write that should be ignored
- byte[] col2 = Bytes.toBytes("col2");
- inprogressTxContext1.start();
- Put putCol2 = new Put(TestBytes.row);
- byte[] valueCol2 = Bytes.toBytes("writing in progress");
- putCol2.add(TestBytes.family, col2, valueCol2);
- txHTable1.put(putCol2);
-
- // start a tx and write a value to test reading in same tx
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] value = Bytes.toBytes("writing");
- put.add(TestBytes.family, TestBytes.qualifier, value);
- transactionAwareHTable.put(put);
-
- // test that a write from a tx started after the first is not visible
- inprogressTxContext2.start();
- Put put2 = new Put(TestBytes.row);
- byte[] value2 = Bytes.toBytes("writing2");
- put2.add(TestBytes.family, TestBytes.qualifier, value2);
- txHTable2.put(put2);
-
- Get get = new Get(TestBytes.row);
- Result row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- byte[] col1Value = row.getValue(TestBytes.family, TestBytes.qualifier);
- Assert.assertNotNull(col1Value);
- Assert.assertArrayEquals(value, col1Value);
- // write from in-progress transaction should not be visible
- byte[] col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- // commit in-progress transaction, should still not be visible
- inprogressTxContext1.finish();
-
- get = new Get(TestBytes.row);
- row = transactionAwareHTable.get(get);
- assertFalse(row.isEmpty());
- col2Value = row.getValue(TestBytes.family, col2);
- assertNull(col2Value);
-
- transactionContext.finish();
-
- inprogressTxContext2.abort();
- }
-
- @Test
- public void testRowLevelConflictDetection() throws Exception {
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext1 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.ROW);
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- byte[] row1 = Bytes.toBytes("row1");
- byte[] row2 = Bytes.toBytes("row2");
- byte[] col1 = Bytes.toBytes("c1");
- byte[] col2 = Bytes.toBytes("c2");
- byte[] val1 = Bytes.toBytes("val1");
- byte[] val2 = Bytes.toBytes("val2");
-
- // test that concurrent writing to different rows succeeds
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col1, val2));
-
- // should be no conflicts
- txContext1.finish();
- txContext2.finish();
-
- transactionContext.start();
- Result res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- Cell cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
-
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // test that writing to different columns in the same row fails
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val2));
-
- txContext2.start();
- txTable2.put(new Put(row1).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row-level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row1));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col1);
- assertNotNull(cell);
- // should now be val2
- assertArrayEquals(val2, CellUtil.cloneValue(cell));
-
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- // col2 should not be visible due to conflict
- assertNull(cell);
- transactionContext.finish();
-
- // test that writing to the same column in the same row fails
- txContext1.start();
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val1));
-
- txContext2.start();
- txTable2.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- txContext1.finish();
- try {
- txContext2.finish();
- fail("txContext2 should have encountered a row and column level conflict during commit");
- } catch (TransactionConflictException tce) {
- txContext2.abort();
- }
-
- transactionContext.start();
- res = transactionAwareHTable.get(new Get(row2));
- assertFalse(res.isEmpty());
- cell = res.getColumnLatestCell(TestBytes.family, col2);
- assertNotNull(cell);
- // should now be val1
- assertArrayEquals(val1, CellUtil.cloneValue(cell));
- transactionContext.finish();
-
- // verify change set that is being reported only on rows
- txContext1.start();
- txTable1.put(new Put(row1).add(TestBytes.family, col1, val1));
- txTable1.put(new Put(row2).add(TestBytes.family, col2, val2));
-
- Collection<byte[]> changeSet = txTable1.getTxChanges();
- assertNotNull(changeSet);
- assertEquals(2, changeSet.size());
- assertTrue(changeSet.contains(txTable1.getChangeKey(row1, null, null)));
- assertTrue(changeSet.contains(txTable1.getChangeKey(row2, null, null)));
- txContext1.finish();
- }
-
- @Test
- public void testNoneLevelConflictDetection() throws Exception {
- InMemoryTxSystemClient txClient = new InMemoryTxSystemClient(txManager);
- TransactionAwareHTable txTable1 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext1 = new TransactionContext(txClient, txTable1);
-
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table),
- TxConstants.ConflictDetection.NONE);
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
-
- // overlapping writes to the same row + column should not conflict
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // changes should not be visible yet
- txContext2.start();
- Result row = txTable2.get(new Get(TestBytes.row));
- assertTrue(row.isEmpty());
-
- txTable2.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // both commits should succeed
- txContext1.finish();
- txContext2.finish();
-
- txContext1.start();
- row = txTable1.get(new Get(TestBytes.row));
- assertFalse(row.isEmpty());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
- txContext1.finish();
-
- // transaction abort should still rollback changes
-
- txContext1.start();
- txTable1.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txContext1.abort();
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row2 = txTable2.get(new Get(TestBytes.row2));
- assertTrue(row2.isEmpty());
- txContext2.finish();
-
- // transaction invalidate should still make changes invisible
-
- txContext1.start();
- Transaction tx1 = txContext1.getCurrentTransaction();
- txTable1.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- assertNotNull(tx1);
- txClient.invalidate(tx1.getWritePointer());
-
- // changes to row2 should be rolled back
- txContext2.start();
- Result row3 = txTable2.get(new Get(TestBytes.row3));
- assertTrue(row3.isEmpty());
- txContext2.finish();
- }
-
- @Test
- public void testCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionContext.checkpoint();
- Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
- assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
- long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
- assertEquals(1, checkpointPtrs.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
-
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
-
- assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
- assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
- long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
- assertEquals(2, checkpointPtrs2.length);
- assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
- assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
-
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- // by default, all rows should be visible with Read-Your-Writes
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
-
- // when disabling current write pointer, only the previous checkpoints should be visible
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- Get get = new Get(TestBytes.row);
- verifyRow(transactionAwareHTable, get, TestBytes.value);
- get = new Get(TestBytes.row2);
- verifyRow(transactionAwareHTable, get, TestBytes.value2);
- get = new Get(TestBytes.row3);
- verifyRow(transactionAwareHTable, get, null);
-
- // test scan results excluding current write pointer
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
-
- Result row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNotNull(row);
- assertArrayEquals(TestBytes.row2, row.getRow());
- assertEquals(1, row.size());
- assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
-
- row = scanner.next();
- assertNull(row);
- scanner.close();
- transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
-
- // commit transaction, verify writes are visible
- transactionContext.finish();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
- transactionContext.finish();
- }
-
- @Test
- public void testInProgressCheckpoint() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
-
- // check that writes are still not visible to other clients
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
-
- txContext2.start();
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- txContext2.finish();
- txTable2.close();
-
- transactionContext.finish();
-
- // verify writes are visible after commit
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
- verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointRollback() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- transactionContext.abort();
-
- transactionContext.start();
- verifyRow(transactionAwareHTable, TestBytes.row, null);
- verifyRow(transactionAwareHTable, TestBytes.row2, null);
- verifyRow(transactionAwareHTable, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = transactionAwareHTable.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- transactionContext.finish();
- }
-
- @Test
- public void testCheckpointInvalidate() throws Exception {
- // start a transaction, using checkpoints between writes
- transactionContext.start();
- Transaction origTx = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- transactionContext.checkpoint();
- Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
- transactionContext.checkpoint();
- Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
- transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
-
- TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
- txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());
-
- // check that writes are not visible
- TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
- TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
- txContext2.start();
- Transaction newTx = txContext2.getCurrentTransaction();
-
- // all 3 writes pointers from the previous transaction should now be excluded
- assertTrue(newTx.isExcluded(origTx.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
- assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));
-
- verifyRow(txTable2, TestBytes.row, null);
- verifyRow(txTable2, TestBytes.row2, null);
- verifyRow(txTable2, TestBytes.row3, null);
-
- Scan scan = new Scan();
- ResultScanner scanner = txTable2.getScanner(scan);
- assertNull(scanner.next());
- scanner.close();
- txContext2.finish();
- }
-
- @Test
- public void testExistingData() throws Exception {
- byte[] val11 = Bytes.toBytes("val11");
- byte[] val12 = Bytes.toBytes("val12");
- byte[] val21 = Bytes.toBytes("val21");
- byte[] val22 = Bytes.toBytes("val22");
- byte[] val31 = Bytes.toBytes("val31");
- byte[] val111 = Bytes.toBytes("val111");
-
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(createTable(Bytes.toBytes("testExistingData"), new byte[][]{TestBytes.family}, true,
- Collections.<String>emptyList()));
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // Add some pre-existing, non-transactional data
- HTable nonTxTable = new HTable(testUtil.getConfiguration(), txTable.getTableName());
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val11));
- nonTxTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, val12));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, val21));
- nonTxTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier2, val22));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER,
- HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.put(new Put(TestBytes.row4).add(TestBytes.family, TestBytes.qualifier, HConstants.EMPTY_BYTE_ARRAY));
- nonTxTable.flushCommits();
-
- // Add transactional data
- txContext.start();
- txTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, val31));
- txContext.finish();
-
- txContext.start();
- // test get
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val11);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), val21);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
-
- // test scan
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val11, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(val21, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
-
- // test update and delete
- txContext.start();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, val111));
- txTable.delete(new Delete(TestBytes.row2).deleteColumns(TestBytes.family, TestBytes.qualifier));
- txContext.finish();
-
- txContext.start();
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier), val111);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2), val12);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier), null);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier2), val22);
- verifyRow(txTable, new Get(TestBytes.row3).addColumn(TestBytes.family, TestBytes.qualifier), val31);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TxConstants.FAMILY_DELETE_QUALIFIER),
- HConstants.EMPTY_BYTE_ARRAY);
- verifyRow(txTable, new Get(TestBytes.row4).addColumn(TestBytes.family, TestBytes.qualifier),
- HConstants.EMPTY_BYTE_ARRAY);
- txContext.finish();
-
- // test scan
- txContext.start();
- try (ResultScanner scanner = txTable.getScanner(new Scan())) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val111, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val12, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row2, result.getRow());
- assertArrayEquals(null, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertArrayEquals(val22, result.getValue(TestBytes.family, TestBytes.qualifier2));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val31, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row4, result.getRow());
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family,
- TxConstants.FAMILY_DELETE_QUALIFIER));
- assertArrayEquals(HConstants.EMPTY_BYTE_ARRAY, result.getValue(TestBytes.family, TestBytes.qualifier));
- assertNull(scanner.next());
- }
- txContext.finish();
- }
-
- private void verifyRow(HTableInterface table, byte[] rowkey, byte[] expectedValue) throws Exception {
- verifyRow(table, new Get(rowkey), expectedValue);
- }
-
- private void verifyRow(HTableInterface table, Get get, byte[] expectedValue) throws Exception {
- verifyRows(table, get, expectedValue == null ? null : ImmutableList.of(expectedValue));
- }
-
- private void verifyRows(HTableInterface table, Get get, List<byte[]> expectedValues) throws Exception {
- Result result = table.get(get);
- if (expectedValues == null) {
- assertTrue(result.isEmpty());
- } else {
- assertFalse(result.isEmpty());
- byte[] family = TestBytes.family;
- byte[] col = TestBytes.qualifier;
- if (get.hasFamilies()) {
- family = get.getFamilyMap().keySet().iterator().next();
- col = get.getFamilyMap().get(family).first();
- }
- Iterator<Cell> it = result.getColumnCells(family, col).iterator();
- for (byte[] expectedValue : expectedValues) {
- Assert.assertTrue(it.hasNext());
- assertArrayEquals(expectedValue, CellUtil.cloneValue(it.next()));
- }
- }
- }
-
- private Cell[] getRow(HTableInterface table, Get get) throws Exception {
- Result result = table.get(get);
- return result.rawCells();
- }
-
- private void verifyScan(HTableInterface table, Scan scan, List<KeyValue> expectedCells) throws Exception {
- List<Cell> actualCells = new ArrayList<>();
- try (ResultScanner scanner = table.getScanner(scan)) {
- Result[] results = scanner.next(expectedCells.size() + 1);
- for (Result result : results) {
- actualCells.addAll(Lists.newArrayList(result.rawCells()));
- }
- Assert.assertEquals(expectedCells, actualCells);
- }
- }
-
- @Test
- public void testVisibilityAll() throws Exception {
- HTable nonTxTable = createTable(Bytes.toBytes("testVisibilityAll"),
- new byte[][]{TestBytes.family, TestBytes.family2}, true, Collections.<String>emptyList());
- TransactionAwareHTable txTable =
- new TransactionAwareHTable(nonTxTable,
- TxConstants.ConflictDetection.ROW); // ROW conflict detection to verify family deletes
- TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
-
- // start a transaction and create a delete marker
- txContext.start();
- //noinspection ConstantConditions
- long txWp0 = txContext.getCurrentTransaction().getWritePointer();
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier2));
- txContext.finish();
-
- // start a new transaction and write some values
- txContext.start();
- @SuppressWarnings("ConstantConditions")
- long txWp1 = txContext.getCurrentTransaction().getWritePointer();
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier, TestBytes.value));
- txTable.put(new Put(TestBytes.row).add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
-
- // verify written data
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- TestBytes.value);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- TestBytes.value2);
-
- // checkpoint and make changes to written data now
- txContext.checkpoint();
- long txWp2 = txContext.getCurrentTransaction().getWritePointer();
- // delete a column
- txTable.delete(new Delete(TestBytes.row).deleteColumn(TestBytes.family, TestBytes.qualifier));
- // no change to a column
- txTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2));
- // update a column
- txTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value3));
- // delete column family
- txTable.delete(new Delete(TestBytes.row).deleteFamily(TestBytes.family2));
-
- // verify changed values
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2),
- null);
-
- // run a scan with VisibilityLevel.ALL, this should return all raw changes by this transaction,
- // and the raw change by prior transaction
- //noinspection ConstantConditions
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- List<KeyValue> expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp0, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // verify a Get is also able to return all snapshot versions
- Get get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier);
- Cell[] cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(3, cells.length);
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
- Assert.assertTrue(CellUtil.isDeleteColumns(cells[2]));
-
- verifyRows(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- ImmutableList.of(TestBytes.value3, TestBytes.value));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value, CellUtil.cloneValue(cells[1]));
-
- get = new Get(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2);
- cells = getRow(txTable, get);
- Assert.assertEquals(2, cells.length);
- Assert.assertTrue(CellUtil.isDelete(cells[0]));
- Assert.assertArrayEquals(TestBytes.value2, CellUtil.cloneValue(cells[1]));
-
- // Verify VisibilityLevel.SNAPSHOT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- // Verify VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier, txWp1, TestBytes.value),
- new KeyValue(TestBytes.row, TestBytes.family2, TestBytes.qualifier2, txWp1, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp1, TestBytes.value)
- );
- verifyScan(txTable, new Scan(), expected);
- txContext.finish();
-
- // finally verify values once more after commit, this time we should get only committed raw values for
- // all visibility levels
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3)
- );
- verifyScan(txTable, new Scan(), expected);
-
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier),
- null);
- verifyRow(txTable, new Get(TestBytes.row).addColumn(TestBytes.family, TestBytes.qualifier2),
- TestBytes.value2);
- verifyRow(txTable, new Get(TestBytes.row2).addColumn(TestBytes.family, TestBytes.qualifier),
- TestBytes.value3);
- txContext.finish();
-
- // Test with regular HBase deletes in pre-existing data
- long now = System.currentTimeMillis();
- Delete deleteColumn = new Delete(TestBytes.row3).deleteColumn(TestBytes.family, TestBytes.qualifier, now - 1);
- // to prevent Tephra from replacing delete with delete marker
- deleteColumn.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteColumn);
- Delete deleteFamily = new Delete(TestBytes.row3).deleteFamily(TestBytes.family2, now);
- // to prevent Tephra from replacing delete with delete marker
- deleteFamily.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
- nonTxTable.delete(deleteFamily);
- nonTxTable.flushCommits();
-
- txContext.start();
- txContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT_ALL);
- expected = ImmutableList.of(
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier, txWp2, KeyValue.Type.DeleteColumn),
- new KeyValue(TestBytes.row, TestBytes.family, TestBytes.qualifier2, txWp2, TestBytes.value2),
- new KeyValue(TestBytes.row, TestBytes.family2, null, txWp2, KeyValue.Type.DeleteFamily),
- new KeyValue(TestBytes.row2, TestBytes.family, TestBytes.qualifier, txWp2, TestBytes.value3),
- new KeyValue(TestBytes.row3, TestBytes.family, TestBytes.qualifier, now - 1, KeyValue.Type.Delete),
- new KeyValue(TestBytes.row3, TestBytes.family2, null, now, KeyValue.Type.DeleteFamily)
- );
- // test scan
- Scan scan = new Scan();
- scan.setRaw(true);
- verifyScan(txTable, scan, expected);
- txContext.finish();
- }
-
- @Test
- public void testFilters() throws Exception {
- // Add some values to table
- transactionContext.start();
- Put put = new Put(TestBytes.row);
- byte[] val1 = Bytes.toBytes(1L);
- put.add(TestBytes.family, TestBytes.qualifier, val1);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row2);
- byte[] val2 = Bytes.toBytes(2L);
- put.add(TestBytes.family, TestBytes.qualifier, val2);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row3);
- byte[] val3 = Bytes.toBytes(3L);
- put.add(TestBytes.family, TestBytes.qualifier, val3);
- transactionAwareHTable.put(put);
- put = new Put(TestBytes.row4);
- byte[] val4 = Bytes.toBytes(4L);
- put.add(TestBytes.family, TestBytes.qualifier, val4);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Delete cell with value 2
- transactionContext.start();
- Delete delete = new Delete(TestBytes.row2);
- delete.addColumn(TestBytes.family, TestBytes.qualifier);
- transactionAwareHTable.delete(delete);
- transactionContext.finish();
-
- // Scan for values less than 4, should get only values 1 and 3
- transactionContext.start();
- Scan scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(4)));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- Result result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run a Get with a filter for less than 10 on row4, should get value 4
- transactionContext.start();
- Get get = new Get(TestBytes.row4);
- get.setFilter(new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
- Result result = transactionAwareHTable.get(get);
- assertFalse(result.isEmpty());
- assertArrayEquals(val4, result.getValue(TestBytes.family, TestBytes.qualifier));
- transactionContext.finish();
-
- // Change value of row4 to 40
- transactionContext.start();
- put = new Put(TestBytes.row4);
- byte[] val40 = Bytes.toBytes(40L);
- put.add(TestBytes.family, TestBytes.qualifier, val40);
- transactionAwareHTable.put(put);
- transactionContext.finish();
-
- // Scan for values less than 10, should get only values 1 and 3
- transactionContext.start();
- scan = new Scan(TestBytes.row, new ValueFilter(CompareFilter.CompareOp.LESS, new LongComparator(10)));
- try (ResultScanner scanner = transactionAwareHTable.getScanner(scan)) {
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row, result.getRow());
- assertArrayEquals(val1, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNotNull(result);
- assertArrayEquals(TestBytes.row3, result.getRow());
- assertArrayEquals(val3, result.getValue(TestBytes.family, TestBytes.qualifier));
- result = scanner.next();
- assertNull(result);
- }
- transactionContext.finish();
-
- // Run the Get again with a filter for less than 10 on row4, this time should not get any results
- transactionContext.start();
- result = transactionAwareHTable.get(get);
- assertTrue(result.isEmpty());
- transactionContext.finish();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilterTest.java b/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilterTest.java
deleted file mode 100644
index 99b300e..0000000
--- a/tephra-hbase-compat-1.0/src/test/java/co/cask/tephra/hbase10/coprocessor/CellSkipFilterTest.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase10.coprocessor;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * HBase 1.0 specific test for testing {@link CellSkipFilter}.
- */
-public class CellSkipFilterTest {
-
- private static final String ROW1KEY = "row1";
- private static final String ROW2KEY = "row2";
- private static final String FAM1KEY = "fam1";
- private static final String COL1KEY = "col1";
- private static final String FAM2KEY = "fam2";
- private static final String COL2KEY = "col2";
- private static final String VALUE = "value";
-
- @Test
- public void testSkipFiltering() throws Exception {
- long timestamp = System.currentTimeMillis();
- // Test to check that we get NEXT_COL once the INCLUDE_AND_NEXT_COL is returned for the same key
- Filter filter = new CellSkipFilter(new MyFilter(0));
- assertEquals(Filter.ReturnCode.INCLUDE, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 1)));
-
- // Next call should get NEXT_COL instead of SKIP, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give SKIP from MyFilter
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM2KEY, COL1KEY, VALUE,
- timestamp - 4)));
-
- // Test to check that we get NEXT_COL once the NEXT_COL is returned for the same key
- filter = new CellSkipFilter(new MyFilter(2));
- assertEquals(Filter.ReturnCode.SKIP, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp)));
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 1)));
-
- // Next call should get NEXT_COL instead of NEXT_ROW, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 2)));
-
- // Next call with the same key should return the NEXT_COL again, as it would be returned by CellSkipFilter
- assertEquals(Filter.ReturnCode.NEXT_COL, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL1KEY, VALUE,
- timestamp - 3)));
-
- // Since MyFilter counter is not incremented in the previous call, filtering for the different keyvalue should
- // give NEXT_ROW from MyFilter
- assertEquals(Filter.ReturnCode.NEXT_ROW, filter.filterKeyValue(newKeyValue(ROW1KEY, FAM1KEY, COL2KEY, VALUE,
- timestamp - 4)));
-
- // Next call with the new key should returned the SEEK_NEXT_USING_HINT
- assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(newKeyValue(ROW2KEY, FAM1KEY, COL1KEY,
- VALUE, timestamp - 5)));
- }
-
- private KeyValue newKeyValue(String rowkey, String family, String column, String value, long timestamp) {
- return new KeyValue(Bytes.toBytes(rowkey), Bytes.toBytes(family), Bytes.toBytes(column),
- timestamp, Bytes.toBytes(value));
- }
-
- /**
- * Sample filter for testing. This filter maintains the {@link List} of {@link ReturnCode}s. It accepts the
- * start index in the list and start serving the return codes corresponding that that index. Every time the
- * return code is served, index is incremented.
- */
- class MyFilter extends FilterBase {
-
- private final List<ReturnCode> returnCodes;
- private int counter;
-
- public MyFilter(int startIndex) {
- returnCodes = Arrays.asList(ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
- ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW, ReturnCode.SEEK_NEXT_USING_HINT);
- counter = startIndex;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- ReturnCode code = returnCodes.get(counter % returnCodes.size());
- counter++;
- return code;
- }
- }
-}
[55/56] [abbrv] incubator-tephra git commit: Support old clients on
operation attribute name change
Posted by po...@apache.org.
Support old clients on operation attribute name change
Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/e0a6e0a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/e0a6e0a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/e0a6e0a5
Branch: refs/heads/master
Commit: e0a6e0a569789afc2872a3a247976b7d08aa463c
Parents: 6cc3adb
Author: poorna <po...@cask.co>
Authored: Tue May 10 21:46:43 2016 -0700
Committer: poorna <po...@cask.co>
Committed: Tue May 10 21:46:43 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/tephra/TxConstants.java | 8 +++
.../tephra/hbase96/TransactionAwareHTable.java | 6 +-
.../coprocessor/TransactionProcessor.java | 12 +++-
.../hbase96/TransactionAwareHTableTest.java | 62 ++++++++++++++++++++
.../tephra/hbase98/TransactionAwareHTable.java | 6 +-
.../coprocessor/TransactionProcessor.java | 12 +++-
.../hbase98/TransactionAwareHTableTest.java | 62 ++++++++++++++++++++
.../hbase10cdh/TransactionAwareHTable.java | 6 +-
.../coprocessor/TransactionProcessor.java | 12 +++-
.../hbase10cdh/TransactionAwareHTableTest.java | 62 ++++++++++++++++++++
.../tephra/hbase10/TransactionAwareHTable.java | 6 +-
.../coprocessor/TransactionProcessor.java | 12 +++-
.../hbase10/TransactionAwareHTableTest.java | 62 ++++++++++++++++++++
.../tephra/hbase11/TransactionAwareHTable.java | 6 +-
.../coprocessor/TransactionProcessor.java | 12 +++-
.../hbase11/TransactionAwareHTableTest.java | 62 ++++++++++++++++++++
16 files changed, 398 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TxConstants.java b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
index d962486..61ee3cc 100644
--- a/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
+++ b/tephra-core/src/main/java/org/apache/tephra/TxConstants.java
@@ -88,11 +88,19 @@ public class TxConstants {
*/
public static final String TX_OPERATION_ATTRIBUTE_KEY = "tephra.tx";
/**
+ * @deprecated This constant is replaced by {@link #TX_OPERATION_ATTRIBUTE_KEY}
+ */
+ public static final String OLD_TX_OPERATION_ATTRIBUTE_KEY = "cask.tx";
+ /**
* Key used to flag a delete operation as part of a transaction rollback. This is used so that the
* {@code TransactionProcessor} coprocessor loaded on a table can differentiate between deletes issued
* as part of a normal client operation versus those performed when rolling back a transaction.
*/
public static final String TX_ROLLBACK_ATTRIBUTE_KEY = "tephra.tx.rollback";
+ /**
+ * @deprecated This constant is replaced by {@link #TX_ROLLBACK_ATTRIBUTE_KEY}
+ */
+ public static final String OLD_TX_ROLLBACK_ATTRIBUTE_KEY = "cask.tx.rollback";
/**
* Column qualifier used for a special delete marker tombstone, which identifies an entire column family as deleted.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java
index f9a3ff3..4633248 100644
--- a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java
@@ -139,7 +139,7 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
byte[] family = change.getFamily();
byte[] qualifier = change.getQualifier();
Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ makeRollbackOperation(rollbackDelete);
switch (conflictLevel) {
case ROW:
case NONE:
@@ -612,4 +612,8 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
}
+
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java
index 136dedc..c74f98d 100644
--- a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java
@@ -173,7 +173,7 @@ public class TransactionProcessor extends BaseRegionObserver {
// Deletes that are part of a transaction rollback do not need special handling.
// They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ if (isRollbackOperation(delete)) {
return;
}
@@ -301,12 +301,22 @@ public class TransactionProcessor extends BaseRegionObserver {
private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded == null) {
+ // to support old clients
+ encoded = op.getAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY);
+ }
if (encoded != null) {
return txCodec.decode(encoded);
}
return null;
}
+ private boolean isRollbackOperation(OperationWithAttributes op) throws IOException {
+ return op.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null ||
+ // to support old clients
+ op.getAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY) != null;
+ }
+
/**
* Derived classes can override this method to customize the filter used to return data visible for the current
* transaction.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java
index 6760ed7..1edd4d7 100644
--- a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java
+++ b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase96/TransactionAwareHTableTest.java
@@ -37,6 +37,7 @@ package org.apache.tephra.hbase96;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
+ import org.apache.hadoop.hbase.client.OperationWithAttributes;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -1552,4 +1553,65 @@ public class TransactionAwareHTableTest {
assertTrue(result.isEmpty());
transactionContext.finish();
}
+
+ /**
+ * Tests that transaction co-processor works with older clients
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testOlderClientOperations() throws Exception {
+ // Use old HTable to test
+ TransactionAwareHTable oldTxAware = new OldTransactionAwareHTable(hTable);
+ transactionContext.addTransactionAware(oldTxAware);
+
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ oldTxAware.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ long txId = transactionContext.getCurrentTransaction().getTransactionId();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value2);
+ oldTxAware.put(put);
+ // Invalidate the second Put
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(txId);
+
+ transactionContext.start();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value3);
+ oldTxAware.put(put);
+ // Abort the third Put
+ transactionContext.abort();
+
+ // Get should now return the first value
+ transactionContext.start();
+ Result result = oldTxAware.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Represents older transaction clients
+ */
+ private static class OldTransactionAwareHTable extends TransactionAwareHTable {
+ public OldTransactionAwareHTable(HTableInterface hTable) {
+ super(hTable);
+ }
+
+ @Override
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+
+ @Override
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java
index 54babd1..9c04d8f 100644
--- a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/TransactionAwareHTable.java
@@ -143,7 +143,7 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
byte[] family = change.getFamily();
byte[] qualifier = change.getQualifier();
Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ makeRollbackOperation(rollbackDelete);
switch (conflictLevel) {
case ROW:
case NONE:
@@ -639,4 +639,8 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
}
+
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java
index e8e045a..cc1915d 100644
--- a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase98/coprocessor/TransactionProcessor.java
@@ -173,7 +173,7 @@ public class TransactionProcessor extends BaseRegionObserver {
// Deletes that are part of a transaction rollback do not need special handling.
// They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ if (isRollbackOperation(delete)) {
return;
}
@@ -301,12 +301,22 @@ public class TransactionProcessor extends BaseRegionObserver {
private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded == null) {
+ // to support old clients
+ encoded = op.getAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY);
+ }
if (encoded != null) {
return txCodec.decode(encoded);
}
return null;
}
+ private boolean isRollbackOperation(OperationWithAttributes op) throws IOException {
+ return op.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null ||
+ // to support old clients
+ op.getAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY) != null;
+ }
+
/**
* Derived classes can override this method to customize the filter used to return data visible for the current
* transaction.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java
index 908f9c8..c4062df 100644
--- a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java
+++ b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase98/TransactionAwareHTableTest.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -1549,4 +1550,65 @@ public class TransactionAwareHTableTest {
assertTrue(result.isEmpty());
transactionContext.finish();
}
+
+ /**
+ * Tests that transaction co-processor works with older clients
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testOlderClientOperations() throws Exception {
+ // Use old HTable to test
+ TransactionAwareHTable oldTxAware = new OldTransactionAwareHTable(hTable);
+ transactionContext.addTransactionAware(oldTxAware);
+
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ oldTxAware.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ long txId = transactionContext.getCurrentTransaction().getTransactionId();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value2);
+ oldTxAware.put(put);
+ // Invalidate the second Put
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(txId);
+
+ transactionContext.start();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value3);
+ oldTxAware.put(put);
+ // Abort the third Put
+ transactionContext.abort();
+
+ // Get should now return the first value
+ transactionContext.start();
+ Result result = oldTxAware.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Represents older transaction clients
+ */
+ private static class OldTransactionAwareHTable extends TransactionAwareHTable {
+ public OldTransactionAwareHTable(HTableInterface hTable) {
+ super(hTable);
+ }
+
+ @Override
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+
+ @Override
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java
index ea9fdad..62cdafd 100644
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/TransactionAwareHTable.java
@@ -143,7 +143,7 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
byte[] family = change.getFamily();
byte[] qualifier = change.getQualifier();
Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ makeRollbackOperation(rollbackDelete);
switch (conflictLevel) {
case ROW:
case NONE:
@@ -671,4 +671,8 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
}
+
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
index 45ac114..f219373 100644
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase10cdh/coprocessor/TransactionProcessor.java
@@ -173,7 +173,7 @@ public class TransactionProcessor extends BaseRegionObserver {
// Deletes that are part of a transaction rollback do not need special handling.
// They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ if (isRollbackOperation(delete)) {
return;
}
@@ -301,12 +301,22 @@ public class TransactionProcessor extends BaseRegionObserver {
private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded == null) {
+ // to support old clients
+ encoded = op.getAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY);
+ }
if (encoded != null) {
return txCodec.decode(encoded);
}
return null;
}
+ private boolean isRollbackOperation(OperationWithAttributes op) throws IOException {
+ return op.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null ||
+ // to support old clients
+ op.getAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY) != null;
+ }
+
/**
* Derived classes can override this method to customize the filter used to return data visible for the current
* transaction.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java
index d31b972..1360252 100644
--- a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java
+++ b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase10cdh/TransactionAwareHTableTest.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -1549,4 +1550,65 @@ public class TransactionAwareHTableTest {
assertTrue(result.isEmpty());
transactionContext.finish();
}
+
+ /**
+ * Tests that transaction co-processor works with older clients
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testOlderClientOperations() throws Exception {
+ // Use old HTable to test
+ TransactionAwareHTable oldTxAware = new OldTransactionAwareHTable(hTable);
+ transactionContext.addTransactionAware(oldTxAware);
+
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ oldTxAware.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ long txId = transactionContext.getCurrentTransaction().getTransactionId();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value2);
+ oldTxAware.put(put);
+ // Invalidate the second Put
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(txId);
+
+ transactionContext.start();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value3);
+ oldTxAware.put(put);
+ // Abort the third Put
+ transactionContext.abort();
+
+ // Get should now return the first value
+ transactionContext.start();
+ Result result = oldTxAware.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Represents older transaction clients
+ */
+ private static class OldTransactionAwareHTable extends TransactionAwareHTable {
+ public OldTransactionAwareHTable(HTableInterface hTable) {
+ super(hTable);
+ }
+
+ @Override
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+
+ @Override
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java
index 3d0fb20..ed8cc25 100644
--- a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/TransactionAwareHTable.java
@@ -143,7 +143,7 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
byte[] family = change.getFamily();
byte[] qualifier = change.getQualifier();
Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ makeRollbackOperation(rollbackDelete);
switch (conflictLevel) {
case ROW:
case NONE:
@@ -671,4 +671,8 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
}
+
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java
index 0d6ef17..1cb6564 100644
--- a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase10/coprocessor/TransactionProcessor.java
@@ -173,7 +173,7 @@ public class TransactionProcessor extends BaseRegionObserver {
// Deletes that are part of a transaction rollback do not need special handling.
// They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ if (isRollbackOperation(delete)) {
return;
}
@@ -301,12 +301,22 @@ public class TransactionProcessor extends BaseRegionObserver {
private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded == null) {
+ // to support old clients
+ encoded = op.getAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY);
+ }
if (encoded != null) {
return txCodec.decode(encoded);
}
return null;
}
+ private boolean isRollbackOperation(OperationWithAttributes op) throws IOException {
+ return op.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null ||
+ // to support old clients
+ op.getAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY) != null;
+ }
+
/**
* Derived classes can override this method to customize the filter used to return data visible for the current
* transaction.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java
index 6ff4bb1..0d54e89 100644
--- a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java
+++ b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase10/TransactionAwareHTableTest.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -1549,4 +1550,65 @@ public class TransactionAwareHTableTest {
assertTrue(result.isEmpty());
transactionContext.finish();
}
+
+ /**
+ * Tests that transaction co-processor works with older clients
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testOlderClientOperations() throws Exception {
+ // Use old HTable to test
+ TransactionAwareHTable oldTxAware = new OldTransactionAwareHTable(hTable);
+ transactionContext.addTransactionAware(oldTxAware);
+
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ oldTxAware.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ long txId = transactionContext.getCurrentTransaction().getTransactionId();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value2);
+ oldTxAware.put(put);
+ // Invalidate the second Put
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(txId);
+
+ transactionContext.start();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value3);
+ oldTxAware.put(put);
+ // Abort the third Put
+ transactionContext.abort();
+
+ // Get should now return the first value
+ transactionContext.start();
+ Result result = oldTxAware.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Represents older transaction clients
+ */
+ private static class OldTransactionAwareHTable extends TransactionAwareHTable {
+ public OldTransactionAwareHTable(HTableInterface hTable) {
+ super(hTable);
+ }
+
+ @Override
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+
+ @Override
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/TransactionAwareHTable.java b/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/TransactionAwareHTable.java
index f28930e..4740f6b 100644
--- a/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/TransactionAwareHTable.java
+++ b/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/TransactionAwareHTable.java
@@ -143,7 +143,7 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
byte[] family = change.getFamily();
byte[] qualifier = change.getQualifier();
Delete rollbackDelete = new Delete(row);
- rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ makeRollbackOperation(rollbackDelete);
switch (conflictLevel) {
case ROW:
case NONE:
@@ -671,4 +671,8 @@ public class TransactionAwareHTable extends AbstractTransactionAwareTable
public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
}
+
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/coprocessor/TransactionProcessor.java
index 8051279..27a7ef2 100644
--- a/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-1.1/src/main/java/org/apache/tephra/hbase11/coprocessor/TransactionProcessor.java
@@ -173,7 +173,7 @@ public class TransactionProcessor extends BaseRegionObserver {
// Deletes that are part of a transaction rollback do not need special handling.
// They will never be rolled back, so are performed as normal HBase deletes.
- if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ if (isRollbackOperation(delete)) {
return;
}
@@ -300,12 +300,22 @@ public class TransactionProcessor extends BaseRegionObserver {
private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded == null) {
+ // to support old clients
+ encoded = op.getAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY);
+ }
if (encoded != null) {
return txCodec.decode(encoded);
}
return null;
}
+ private boolean isRollbackOperation(OperationWithAttributes op) throws IOException {
+ return op.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null ||
+ // to support old clients
+ op.getAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY) != null;
+ }
+
/**
* Derived classes can override this method to customize the filter used to return data visible for the current
* transaction.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/e0a6e0a5/tephra-hbase-compat-1.1/src/test/java/org/apache/tephra/hbase11/TransactionAwareHTableTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1/src/test/java/org/apache/tephra/hbase11/TransactionAwareHTableTest.java b/tephra-hbase-compat-1.1/src/test/java/org/apache/tephra/hbase11/TransactionAwareHTableTest.java
index 8578230..9968fb3 100644
--- a/tephra-hbase-compat-1.1/src/test/java/org/apache/tephra/hbase11/TransactionAwareHTableTest.java
+++ b/tephra-hbase-compat-1.1/src/test/java/org/apache/tephra/hbase11/TransactionAwareHTableTest.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -1543,4 +1544,65 @@ public class TransactionAwareHTableTest {
assertTrue(result.isEmpty());
transactionContext.finish();
}
+
+ /**
+ * Tests that transaction co-processor works with older clients
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testOlderClientOperations() throws Exception {
+ // Use old HTable to test
+ TransactionAwareHTable oldTxAware = new OldTransactionAwareHTable(hTable);
+ transactionContext.addTransactionAware(oldTxAware);
+
+ transactionContext.start();
+ Put put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
+ oldTxAware.put(put);
+ transactionContext.finish();
+
+ transactionContext.start();
+ long txId = transactionContext.getCurrentTransaction().getTransactionId();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value2);
+ oldTxAware.put(put);
+ // Invalidate the second Put
+ TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
+ txClient.invalidate(txId);
+
+ transactionContext.start();
+ put = new Put(TestBytes.row);
+ put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value3);
+ oldTxAware.put(put);
+ // Abort the third Put
+ transactionContext.abort();
+
+ // Get should now return the first value
+ transactionContext.start();
+ Result result = oldTxAware.get(new Get(TestBytes.row));
+ transactionContext.finish();
+
+ byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
+ assertArrayEquals(TestBytes.value, value);
+ }
+
+ /**
+ * Represents older transaction clients
+ */
+ private static class OldTransactionAwareHTable extends TransactionAwareHTable {
+ public OldTransactionAwareHTable(HTableInterface hTable) {
+ super(hTable);
+ }
+
+ @Override
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.OLD_TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+
+ @Override
+ protected void makeRollbackOperation(Delete delete) {
+ delete.setAttribute(TxConstants.OLD_TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ }
+ }
}
[23/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java
new file mode 100644
index 0000000..f9a3ff3
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/TransactionAwareHTable.java
@@ -0,0 +1,615 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tephra.hbase96;
+
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.tephra.AbstractTransactionAwareTable;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionAware;
+import org.apache.tephra.TxConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+/**
+ * A Transaction Aware HTable implementation for HBase 0.96. Operations are committed as usual,
+ * but upon a failed or aborted transaction, they are rolled back to the state before the transaction
+ * was started.
+ */
+public class TransactionAwareHTable extends AbstractTransactionAwareTable
+ implements HTableInterface, TransactionAware {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareHTable.class);
+ private final HTableInterface hTable;
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ */
+ public TransactionAwareHTable(HTableInterface hTable) {
+ this(hTable, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable
+ *
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel) {
+ this(hTable, conflictLevel, false);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, boolean allowNonTransactional) {
+ this(hTable, TxConstants.ConflictDetection.COLUMN, allowNonTransactional);
+ }
+
+ /**
+ * Create a transactional aware instance of the passed HTable, with the option
+ * of allowing non-transactional operations.
+ * @param hTable underlying HBase table to use
+ * @param conflictLevel level of conflict detection to perform (defaults to {@code COLUMN})
+ * @param allowNonTransactional if true, additional operations (checkAndPut, increment, checkAndDelete)
+ * will be available, though non-transactional
+ */
+ public TransactionAwareHTable(HTableInterface hTable, TxConstants.ConflictDetection conflictLevel,
+ boolean allowNonTransactional) {
+ super(conflictLevel, allowNonTransactional);
+ this.hTable = hTable;
+ }
+
+ /* AbstractTransactionAwareTable implementation */
+
+ @Override
+ protected byte[] getTableKey() {
+ return getTableName();
+ }
+
+ @Override
+ protected boolean doCommit() throws IOException {
+ hTable.flushCommits();
+ return true;
+ }
+
+ @Override
+ protected boolean doRollback() throws Exception {
+ try {
+ // pre-size arraylist of deletes
+ int size = 0;
+ for (Set<ActionChange> cs : changeSets.values()) {
+ size += cs.size();
+ }
+ List<Delete> rollbackDeletes = new ArrayList<>(size);
+ for (Map.Entry<Long, Set<ActionChange>> entry : changeSets.entrySet()) {
+ long transactionTimestamp = entry.getKey();
+ for (ActionChange change : entry.getValue()) {
+ byte[] row = change.getRow();
+ byte[] family = change.getFamily();
+ byte[] qualifier = change.getQualifier();
+ Delete rollbackDelete = new Delete(row);
+ rollbackDelete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
+ switch (conflictLevel) {
+ case ROW:
+ case NONE:
+ // issue family delete for the tx write pointer
+ rollbackDelete.deleteFamilyVersion(change.getFamily(), transactionTimestamp);
+ break;
+ case COLUMN:
+ if (family != null && qualifier == null) {
+ rollbackDelete.deleteFamilyVersion(family, transactionTimestamp);
+ } else if (family != null && qualifier != null) {
+ rollbackDelete.deleteColumn(family, qualifier, transactionTimestamp);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unknown conflict detection level: " + conflictLevel);
+ }
+ rollbackDeletes.add(rollbackDelete);
+ }
+ }
+ hTable.delete(rollbackDeletes);
+ return true;
+ } finally {
+ try {
+ hTable.flushCommits();
+ } catch (Exception e) {
+ LOG.error("Could not flush HTable commits", e);
+ }
+ tx = null;
+ changeSets.clear();
+ }
+ }
+
+ /* HTableInterface implementation */
+
+ @Override
+ public byte[] getTableName() {
+ return hTable.getTableName();
+ }
+
+ @Override
+ public TableName getName() {
+ return hTable.getName();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hTable.getConfiguration();
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return hTable.getTableDescriptor();
+ }
+
+ @Override
+ public boolean exists(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.exists(transactionalizeAction(get));
+ }
+
+ @Override
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Get> transactionalizedGets = new ArrayList<Get>(gets.size());
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.exists(transactionalizedGets);
+ }
+
+ @Override
+ public void batch(List<? extends Row> actions, Object[] results) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batch(transactionalizeActions(actions), results);
+ }
+
+ @Override
+ public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batch(transactionalizeActions(actions));
+ }
+
+ @Override
+ public <R> void batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback) throws
+ IOException, InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.batchCallback(transactionalizeActions(actions), results, callback);
+ }
+
+ @Override
+ public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException,
+ InterruptedException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.batchCallback(transactionalizeActions(actions), callback);
+ }
+
+ @Override
+ public Result get(Get get) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.get(transactionalizeAction(get));
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ ArrayList<Get> transactionalizedGets = new ArrayList<Get>();
+ for (Get get : gets) {
+ transactionalizedGets.add(transactionalizeAction(get));
+ }
+ return hTable.get(transactionalizedGets);
+ }
+
+ @Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.getRowOrBefore(row, family);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan scan) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Scan scan = new Scan();
+ scan.addColumn(family, qualifier);
+ return hTable.getScanner(transactionalizeAction(scan));
+ }
+
+ @Override
+ public void put(Put put) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ Put txPut = transactionalizeAction(put);
+ hTable.put(txPut);
+ }
+
+ @Override
+ public void put(List<Put> puts) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Put> transactionalizedPuts = new ArrayList<Put>(puts.size());
+ for (Put put : puts) {
+ Put txPut = transactionalizeAction(put);
+ transactionalizedPuts.add(txPut);
+ }
+ hTable.put(transactionalizedPuts);
+ }
+
+ @Override
+ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndPut(row, family, qualifier, value, put);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public void delete(Delete delete) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ hTable.delete(transactionalizeAction(delete));
+ }
+
+ @Override
+ public void delete(List<Delete> deletes) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ List<Delete> transactionalizedDeletes = new ArrayList<Delete>(deletes.size());
+ for (Delete delete : deletes) {
+ Delete txDelete = transactionalizeAction(delete);
+ transactionalizedDeletes.add(txDelete);
+ }
+ hTable.delete(transactionalizedDeletes);
+ }
+
+ @Override
+ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.checkAndDelete(row, family, qualifier, value, delete);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public void mutateRow(RowMutations rm) throws IOException {
+ if (tx == null) {
+ throw new IOException("Transaction not started");
+ }
+ RowMutations transactionalMutations = new RowMutations();
+ for (Mutation mutation : rm.getMutations()) {
+ if (mutation instanceof Put) {
+ transactionalMutations.add(transactionalizeAction((Put) mutation));
+ } else if (mutation instanceof Delete) {
+ transactionalMutations.add(transactionalizeAction((Delete) mutation));
+ }
+ }
+ hTable.mutateRow(transactionalMutations);
+ }
+
+ @Override
+ public Result append(Append append) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.append(append);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public Result increment(Increment increment) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.increment(increment);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, durability);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)
+ throws IOException {
+ if (allowNonTransactional) {
+ return hTable.incrementColumnValue(row, family, qualifier, amount, writeToWAL);
+ } else {
+ throw new UnsupportedOperationException("Operation is not supported transactionally");
+ }
+ }
+
+ @Override
+ public boolean isAutoFlush() {
+ return hTable.isAutoFlush();
+ }
+
+ @Override
+ public void flushCommits() throws IOException {
+ hTable.flushCommits();
+ }
+
+ @Override
+ public void close() throws IOException {
+ hTable.close();
+ }
+
+ @Override
+ public CoprocessorRpcChannel coprocessorService(byte[] row) {
+ return hTable.coprocessorService(row);
+ }
+
+ @Override
+ public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable)
+ throws ServiceException, Throwable {
+ return hTable.coprocessorService(service, startKey, endKey, callable);
+ }
+
+ @Override
+ public <T extends Service, R> void coprocessorService(Class<T> service, byte[] startKey, byte[] endKey,
+ Batch.Call<T, R> callable, Batch.Callback<R> callback)
+ throws ServiceException, Throwable {
+ hTable.coprocessorService(service, startKey, endKey, callable, callback);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush) {
+ setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ hTable.setAutoFlush(autoFlush, clearBufferOnFail);
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ hTable.setAutoFlushTo(autoFlush);
+ }
+
+ @Override
+ public long getWriteBufferSize() {
+ return hTable.getWriteBufferSize();
+ }
+
+ @Override
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ hTable.setWriteBufferSize(writeBufferSize);
+ }
+
+ // Helpers to get copies of objects with the timestamp set to the current transaction timestamp.
+
+ private Get transactionalizeAction(Get get) throws IOException {
+ addToOperation(get, tx);
+ return get;
+ }
+
+ private Scan transactionalizeAction(Scan scan) throws IOException {
+ addToOperation(scan, tx);
+ return scan;
+ }
+
+ private Put transactionalizeAction(Put put) throws IOException {
+ Put txPut = new Put(put.getRow(), tx.getWritePointer());
+ Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
+ if (!familyMap.isEmpty()) {
+ for (Map.Entry<byte[], List<Cell>> family : familyMap) {
+ List<Cell> familyValues = family.getValue();
+ if (!familyValues.isEmpty()) {
+ for (Cell value : familyValues) {
+ txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
+ addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
+ txPut.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txPut.setDurability(put.getDurability());
+ addToOperation(txPut, tx);
+ return txPut;
+ }
+
+ private Delete transactionalizeAction(Delete delete) throws IOException {
+ long transactionTimestamp = tx.getWritePointer();
+
+ byte[] deleteRow = delete.getRow();
+ Delete txDelete = new Delete(deleteRow, transactionTimestamp);
+
+ Map<byte[], List<Cell>> familyToDelete = delete.getFamilyCellMap();
+ if (familyToDelete.isEmpty()) {
+ // perform a row delete if we are using row-level conflict detection
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // Row delete leaves delete markers in all column families of the table
+ // Therefore get all the column families of the hTable from the HTableDescriptor and add them to the changeSet
+ for (HColumnDescriptor columnDescriptor : hTable.getTableDescriptor().getColumnFamilies()) {
+ // no need to identify individual columns deleted
+ addToChangeSet(deleteRow, columnDescriptor.getName(), null);
+ }
+ } else {
+ Result result = get(new Get(delete.getRow()));
+ // Delete everything
+ NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultMap = result.getNoVersionMap();
+ for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> familyEntry : resultMap.entrySet()) {
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(familyEntry.getKey());
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(familyEntry.getKey(), column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, familyEntry.getKey(), column.getKey());
+ }
+ }
+ }
+ } else {
+ for (Map.Entry<byte [], List<Cell>> familyEntry : familyToDelete.entrySet()) {
+ byte[] family = familyEntry.getKey();
+ List<Cell> entries = familyEntry.getValue();
+ boolean isFamilyDelete = false;
+ if (entries.size() == 1) {
+ Cell cell = entries.get(0);
+ isFamilyDelete = CellUtil.isDeleteFamily(cell);
+ }
+ if (isFamilyDelete) {
+ if (conflictLevel == TxConstants.ConflictDetection.ROW ||
+ conflictLevel == TxConstants.ConflictDetection.NONE) {
+ // no need to identify individual columns deleted
+ // Older versions of HBase 0.96 lack HBASE-10964, so family deletes do not correctly
+ // inherit the common Delete timestamp, so must explicitly set the timestamp here.
+ txDelete.deleteFamily(family, transactionTimestamp);
+ addToChangeSet(deleteRow, family, null);
+ } else {
+ Result result = get(new Get(delete.getRow()).addFamily(family));
+ // Delete entire family
+ NavigableMap<byte[], byte[]> familyColumns = result.getFamilyMap(family);
+ for (Map.Entry<byte[], byte[]> column : familyColumns.entrySet()) {
+ txDelete.deleteColumns(family, column.getKey(), transactionTimestamp);
+ addToChangeSet(deleteRow, family, column.getKey());
+ }
+ }
+ } else {
+ for (Cell value : entries) {
+ txDelete.deleteColumns(value.getFamily(), value.getQualifier(), transactionTimestamp);
+ addToChangeSet(deleteRow, value.getFamily(), value.getQualifier());
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ txDelete.setAttribute(entry.getKey(), entry.getValue());
+ }
+ txDelete.setDurability(delete.getDurability());
+ return txDelete;
+ }
+
+ private List<? extends Row> transactionalizeActions(List<? extends Row> actions) throws IOException {
+ List<Row> transactionalizedActions = new ArrayList<>(actions.size());
+ for (Row action : actions) {
+ if (action instanceof Get) {
+ transactionalizedActions.add(transactionalizeAction((Get) action));
+ } else if (action instanceof Put) {
+ transactionalizedActions.add(transactionalizeAction((Put) action));
+ } else if (action instanceof Delete) {
+ transactionalizedActions.add(transactionalizeAction((Delete) action));
+ } else {
+ transactionalizedActions.add(action);
+ }
+ }
+ return transactionalizedActions;
+ }
+
+ public void addToOperation(OperationWithAttributes op, Transaction tx) throws IOException {
+ op.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilter.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilter.java
new file mode 100644
index 0000000..526498d
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/CellSkipFilter.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96.coprocessor;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * {@link Filter} that encapsulates another {@link Filter}. It remembers the last {@link KeyValue}
+ * for which the underlying filter returned the {@link ReturnCode#NEXT_COL} or {@link ReturnCode#INCLUDE_AND_NEXT_COL},
+ * so that when {@link #filterKeyValue} is called again for the same {@link KeyValue} with different
+ * version, it returns {@link ReturnCode#NEXT_COL} directly without consulting the underlying {@link Filter}.
+ * Please see TEPHRA-169 for more details.
+ */
+public class CellSkipFilter extends FilterBase {
+ private final Filter filter;
+ // remember the previous keyvalue processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
+ private KeyValue skipColumn = null;
+
+ public CellSkipFilter(Filter filter) {
+ this.filter = filter;
+ }
+
+ /**
+ * Determines whether the current cell should be skipped. The cell will be skipped
+ * if the previous keyvalue had the same key as the current cell. This means filter already responded
+ * for the previous keyvalue with ReturnCode.NEXT_COL or ReturnCode.INCLUDE_AND_NEXT_COL.
+ * @param cell the {@link Cell} to be tested for skipping
+ * @return true is current cell should be skipped, false otherwise
+ */
+ private boolean skipCellVersion(Cell cell) {
+ return skipColumn != null
+ && skipColumn.matchingRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
+ && skipColumn.matchingColumn(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (skipCellVersion(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+
+ ReturnCode code = filter.filterKeyValue(cell);
+ if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
+ // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
+ skipColumn = KeyValue.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ } else {
+ skipColumn = null;
+ }
+ return code;
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ return filter.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ return filter.transformCell(cell);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ filter.reset();
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ return filter.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ return filter.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ filter.filterRowCells(kvs);
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ return filter.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ return filter.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ return filter.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ return filter.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return filter.toByteArray();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionFilters.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionFilters.java
new file mode 100644
index 0000000..e6334ed
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionFilters.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96.coprocessor;
+
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.tephra.Transaction;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Factory class for providing {@link Filter} instances.
+ */
+public class TransactionFilters {
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+ }
+
+ /**
+ * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java
new file mode 100644
index 0000000..136dedc
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionProcessor.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96.coprocessor;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.OperationWithAttributes;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TransactionCodec;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.coprocessor.TransactionStateCache;
+import org.apache.tephra.coprocessor.TransactionStateCacheSupplier;
+import org.apache.tephra.persist.TransactionVisibilityState;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+
+/**
+ * {@code org.apache.hadoop.hbase.coprocessor.RegionObserver} coprocessor that handles server-side processing
+ * for transactions:
+ * <ul>
+ * <li>applies filtering to exclude data from invalid and in-progress transactions</li>
+ * <li>overrides the scanner returned for flush and compaction to drop data written by invalidated transactions,
+ * or expired due to TTL.</li>
+ * </ul>
+ *
+ * <p>In order to use this coprocessor for transactions, configure the class on any table involved in transactions,
+ * or on all user tables by adding the following to hbase-site.xml:
+ * {@code
+ * <property>
+ * <name>hbase.coprocessor.region.classes</name>
+ * <value>TransactionProcessor</value>
+ * </property>
+ * }
+ * </p>
+ *
+ * <p>HBase {@code Get} and {@code Scan} operations should have the current transaction serialized on to the operation
+ * as an attribute:
+ * {@code
+ * Transaction t = ...;
+ * Get get = new Get(...);
+ * TransactionCodec codec = new TransactionCodec();
+ * codec.addToOperation(get, t);
+ * }
+ * </p>
+ */
+public class TransactionProcessor extends BaseRegionObserver {
+ private static final Log LOG = LogFactory.getLog(TransactionProcessor.class);
+
+ private TransactionStateCache cache;
+ private final TransactionCodec txCodec;
+ protected Map<byte[], Long> ttlByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ protected boolean allowEmptyValues = TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;
+ protected boolean readNonTxnData = TxConstants.DEFAULT_READ_NON_TX_DATA;
+
+ public TransactionProcessor() {
+ this.txCodec = new TransactionCodec();
+ }
+
+ /* RegionObserver implementation */
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ if (e instanceof RegionCoprocessorEnvironment) {
+ RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
+ Supplier<TransactionStateCache> cacheSupplier = getTransactionStateCacheSupplier(env);
+ this.cache = cacheSupplier.get();
+
+ HTableDescriptor tableDesc = env.getRegion().getTableDesc();
+ for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
+ String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
+ long ttl = 0;
+ if (columnTTL != null) {
+ try {
+ ttl = Long.parseLong(columnTTL);
+ LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
+ ", value = " + columnTTL);
+ }
+ }
+ ttlByFamily.put(columnDesc.getName(), ttl);
+ }
+
+ this.allowEmptyValues = env.getConfiguration().getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
+ TxConstants.ALLOW_EMPTY_VALUES_DEFAULT);
+ this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
+ if (readNonTxnData) {
+ LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
+ }
+ }
+ }
+
+ protected Supplier<TransactionStateCache> getTransactionStateCacheSupplier(RegionCoprocessorEnvironment env) {
+ return new TransactionStateCacheSupplier(env.getConfiguration());
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ // nothing to do
+ }
+
+ @Override
+ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
+ throws IOException {
+ Transaction tx = getFromOperation(get);
+ if (tx != null) {
+ projectFamilyDeletes(get);
+ get.setMaxVersions();
+ get.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, get.getFilter());
+ get.setFilter(newFilter);
+ }
+ }
+
+ @Override
+ public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit,
+ Durability durability) throws IOException {
+ // Translate deletes into our own delete tombstones
+ // Since HBase deletes cannot be undone, we need to translate deletes into special puts, which allows
+ // us to rollback the changes (by a real delete) if the transaction fails
+
+ // Deletes that are part of a transaction rollback do not need special handling.
+ // They will never be rolled back, so are performed as normal HBase deletes.
+ if (delete.getAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY) != null) {
+ return;
+ }
+
+ // Other deletes are client-initiated and need to be translated into our own tombstones
+ // TODO: this should delegate to the DeleteStrategy implementation.
+ Put deleteMarkers = new Put(delete.getRow(), delete.getTimeStamp());
+ for (byte[] family : delete.getFamilyCellMap().keySet()) {
+ List<Cell> familyCells = delete.getFamilyCellMap().get(family);
+ if (isFamilyDelete(familyCells)) {
+ deleteMarkers.add(family, TxConstants.FAMILY_DELETE_QUALIFIER, familyCells.get(0).getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ } else {
+ int cellSize = familyCells.size();
+ for (int i = 0; i < cellSize; i++) {
+ Cell cell = familyCells.get(i);
+ deleteMarkers.add(family, CellUtil.cloneQualifier(cell), cell.getTimestamp(),
+ HConstants.EMPTY_BYTE_ARRAY);
+ }
+ }
+ }
+ for (Map.Entry<String, byte[]> entry : delete.getAttributesMap().entrySet()) {
+ deleteMarkers.setAttribute(entry.getKey(), entry.getValue());
+ }
+ e.getEnvironment().getRegion().put(deleteMarkers);
+ // skip normal delete handling
+ e.bypass();
+ }
+
+ private boolean isFamilyDelete(List<Cell> familyCells) {
+ return familyCells.size() == 1 && CellUtil.isDeleteFamily(familyCells.get(0));
+ }
+
+ @Override
+ public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
+ throws IOException {
+ Transaction tx = getFromOperation(scan);
+ if (tx != null) {
+ projectFamilyDeletes(scan);
+ scan.setMaxVersions();
+ scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
+ TxUtils.getMaxVisibleTimestamp(tx));
+ Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
+ scan.setFilter(newFilter);
+ }
+ return s;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any scan operation.
+ * @param scan The original scan request
+ * @return The modified scan request with the family delete qualifiers represented
+ */
+ private Scan projectFamilyDeletes(Scan scan) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ scan.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return scan;
+ }
+
+ /**
+ * Ensures that family delete markers are present in the columns requested for any get operation.
+ * @param get The original get request
+ * @return The modified get request with the family delete qualifiers represented
+ */
+ private Get projectFamilyDeletes(Get get) {
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
+ NavigableSet<byte[]> columns = entry.getValue();
+ // wildcard scans will automatically include the delete marker, so only need to add it when we have
+ // explicit columns listed
+ if (columns != null && !columns.isEmpty()) {
+ get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
+ }
+ }
+ return get;
+ }
+
+ @Override
+ public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ KeyValueScanner memstoreScanner, InternalScanner scanner)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "flush", cache.getLatestState(), store,
+ Collections.singletonList(memstoreScanner), ScanType.COMPACT_RETAIN_DELETES,
+ HConstants.OLDEST_TIMESTAMP);
+ }
+
+ @Override
+ public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
+ CompactionRequest request)
+ throws IOException {
+ return createStoreScanner(c.getEnvironment(), "compaction", cache.getLatestState(), store, scanners, scanType,
+ earliestPutTs);
+ }
+
+ protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
+ TransactionVisibilityState snapshot, Store store,
+ List<? extends KeyValueScanner> scanners, ScanType type,
+ long earliestPutTs) throws IOException {
+ if (snapshot == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
+ ", no current transaction state found, defaulting to normal " + action + " scanner");
+ }
+ return null;
+ }
+
+ // construct a dummy transaction from the latest snapshot
+ Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
+ Scan scan = new Scan();
+ // need to see all versions, since we filter out excludes and applications may rely on multiple versions
+ scan.setMaxVersions();
+ scan.setFilter(
+ new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
+ snapshot.getInvalid(),
+ getTransactionFilter(dummyTx, type, null)));
+
+ return new StoreScanner(store, store.getScanInfo(), scan, scanners,
+ type, store.getSmallestReadPoint(), earliestPutTs);
+ }
+
+ private Transaction getFromOperation(OperationWithAttributes op) throws IOException {
+ byte[] encoded = op.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
+ if (encoded != null) {
+ return txCodec.decode(encoded);
+ }
+ return null;
+ }
+
+ /**
+ * Derived classes can override this method to customize the filter used to return data visible for the current
+ * transaction.
+ *
+ * @param tx the current transaction to apply
+ * @param type the type of scan operation being performed
+ */
+ protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
+ return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+ }
+
+ /**
+ * Filter used to include cells visible to in-progress transactions on flush and commit.
+ */
+ static class IncludeInProgressFilter extends FilterBase {
+ private final long visibilityUpperBound;
+ private final Set<Long> invalidIds;
+ private final Filter txFilter;
+
+ public IncludeInProgressFilter(long upperBound, Collection<Long> invalids, Filter transactionFilter) {
+ this.visibilityUpperBound = upperBound;
+ this.invalidIds = Sets.newHashSet(invalids);
+ this.txFilter = transactionFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ // include all cells visible to in-progress transactions, except for those already marked as invalid
+ long ts = cell.getTimestamp();
+ if (ts > visibilityUpperBound) {
+ // include everything that could still be in-progress except invalids
+ if (invalidIds.contains(ts)) {
+ return ReturnCode.SKIP;
+ }
+ return ReturnCode.INCLUDE;
+ }
+ return txFilter.filterKeyValue(cell);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java
new file mode 100644
index 0000000..d675db7
--- /dev/null
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase96/coprocessor/TransactionVisibilityFilter.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.hbase96.coprocessor;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.util.TxUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Applies filtering of data based on transactional visibility (HBase 0.96+ specific version).
+ * Note: this is intended for server-side use only, as additional properties need to be set on
+ * any {@code Scan} or {@code Get} operation performed.
+ */
+public class TransactionVisibilityFilter extends FilterBase {
+ private final Transaction tx;
+ // oldest visible timestamp by column family, used to apply TTL when reading
+ private final Map<byte[], Long> oldestTsByFamily;
+ // if false, empty values will be interpreted as deletes
+ private final boolean allowEmptyValues;
+ // whether or not we can remove delete markers
+ // these can only be safely removed when we are traversing all storefiles
+ private final boolean clearDeletes;
+ // optional sub-filter to apply to visible cells
+ private final Filter cellFilter;
+
+ // since we traverse KVs in order, cache the current oldest TS to avoid map lookups per KV
+ private byte[] currentFamily = new byte[0];
+ private long currentOldestTs;
+
+ private DeleteTracker deleteTracker = new DeleteTracker();
+
+ /**
+ * Creates a new {@link Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType) {
+ this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+ }
+
+ /**
+ * Creates a new {@link Filter} for returning data only from visible transactions.
+ *
+ * @param tx the current transaction to apply. Only data visible to this transaction will be returned.
+ * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
+ * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
+ * these will be interpreted as "delete" markers and the column will be filtered out
+ * @param scanType the type of scan operation being performed
+ * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
+ * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then
+ * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
+ */
+ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+ ScanType scanType, @Nullable Filter cellFilter) {
+ this.tx = tx;
+ this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
+ long familyTTL = ttlEntry.getValue();
+ oldestTsByFamily.put(ttlEntry.getKey(),
+ familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+ }
+ this.allowEmptyValues = allowEmptyValues;
+ this.clearDeletes =
+ scanType == ScanType.COMPACT_DROP_DELETES ||
+ (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL);
+ this.cellFilter = cellFilter;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell cell) throws IOException {
+ if (!CellUtil.matchingFamily(cell, currentFamily)) {
+ // column family changed
+ currentFamily = CellUtil.cloneFamily(cell);
+ Long familyOldestTs = oldestTsByFamily.get(currentFamily);
+ currentOldestTs = familyOldestTs != null ? familyOldestTs : 0;
+ deleteTracker.reset();
+ }
+ // need to apply TTL for the column family here
+ long kvTimestamp = cell.getTimestamp();
+ if (TxUtils.getTimestampForTTL(kvTimestamp) < currentOldestTs) {
+ // passed TTL for this column, seek to next
+ return ReturnCode.NEXT_COL;
+ } else if (tx.isVisible(kvTimestamp)) {
+ // Return all writes done by current transaction (including deletes) for VisibilityLevel.SNAPSHOT_ALL
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
+ // cell is visible
+ // visibility SNAPSHOT_ALL needs all matches
+ return runSubFilter(ReturnCode.INCLUDE, cell);
+ }
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ deleteTracker.addFamilyDelete(cell);
+ if (clearDeletes) {
+ return ReturnCode.NEXT_COL;
+ } else {
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // check if masked by family delete
+ if (deleteTracker.isDeleted(cell)) {
+ return ReturnCode.NEXT_COL;
+ }
+ // check for column delete
+ if (isColumnDelete(cell)) {
+ if (clearDeletes) {
+ // skip "deleted" cell
+ return ReturnCode.NEXT_COL;
+ } else {
+ // keep the marker but skip any remaining versions
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ }
+ }
+ // cell is visible
+ // as soon as we find a KV to include we can move to the next column
+ return runSubFilter(ReturnCode.INCLUDE_AND_NEXT_COL, cell);
+ } else {
+ return ReturnCode.SKIP;
+ }
+ }
+
+ private ReturnCode runSubFilter(ReturnCode txFilterCode, Cell cell) throws IOException {
+ if (cellFilter != null) {
+ ReturnCode subFilterCode = cellFilter.filterKeyValue(cell);
+ return determineReturnCode(txFilterCode, subFilterCode);
+ }
+ return txFilterCode;
+ }
+
+ /**
+ * Determines the return code of TransactionVisibilityFilter based on sub-filter's return code.
+ * Sub-filter can only exclude cells included by TransactionVisibilityFilter, i.e., sub-filter's
+ * INCLUDE will be ignored. This behavior makes sure that sub-filter only sees cell versions valid for the
+ * given transaction. If sub-filter needs to see older versions of cell, then this method can be overridden.
+ *
+ * @param txFilterCode return code from TransactionVisibilityFilter
+ * @param subFilterCode return code from sub-filter
+ * @return final return code
+ */
+ protected ReturnCode determineReturnCode(ReturnCode txFilterCode, ReturnCode subFilterCode) {
+ // Return the more restrictive of the two filter responses
+ switch (subFilterCode) {
+ case INCLUDE:
+ return txFilterCode;
+ case INCLUDE_AND_NEXT_COL:
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ case SKIP:
+ return txFilterCode == ReturnCode.INCLUDE ? ReturnCode.SKIP : ReturnCode.NEXT_COL;
+ default:
+ return subFilterCode;
+ }
+ }
+
+ @Override
+ public boolean filterRow() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRow();
+ }
+ return super.filterRow();
+ }
+
+ @Override
+ public Cell transformCell(Cell cell) throws IOException {
+ // Convert Tephra deletes back into HBase deletes
+ if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
+ if (DeleteTracker.isFamilyDelete(cell)) {
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
+ KeyValue.Type.DeleteFamily);
+ } else if (isColumnDelete(cell)) {
+ // Note: in some cases KeyValue.Type.Delete is used in Delete object,
+ // and in some other cases KeyValue.Type.DeleteColumn is used.
+ // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
+ // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
+ // work in both cases.
+ return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
+ cell.getTimestamp(), KeyValue.Type.DeleteColumn);
+ }
+ }
+ return cell;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ deleteTracker.reset();
+ if (cellFilter != null) {
+ cellFilter.reset();
+ }
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterRowKey(buffer, offset, length);
+ }
+ return super.filterRowKey(buffer, offset, length);
+ }
+
+ @Override
+ public boolean filterAllRemaining() throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.filterAllRemaining();
+ }
+ return super.filterAllRemaining();
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ if (cellFilter != null) {
+ cellFilter.filterRowCells(kvs);
+ } else {
+ super.filterRowCells(kvs);
+ }
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ if (cellFilter != null) {
+ return cellFilter.hasFilterRow();
+ }
+ return super.hasFilterRow();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextKeyHint(currentKV);
+ }
+ return super.getNextKeyHint(currentKV);
+ }
+
+ @Override
+ public Cell getNextCellHint(Cell currentKV) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.getNextCellHint(currentKV);
+ }
+ return super.getNextCellHint(currentKV);
+ }
+
+ @Override
+ public boolean isFamilyEssential(byte[] name) throws IOException {
+ if (cellFilter != null) {
+ return cellFilter.isFamilyEssential(name);
+ }
+ return super.isFamilyEssential(name);
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return super.toByteArray();
+ }
+
+ private boolean isColumnDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
+ }
+
+ private static final class DeleteTracker {
+ private long familyDeleteTs;
+
+ public static boolean isFamilyDelete(Cell cell) {
+ return !TxUtils.isPreExistingVersion(cell.getTimestamp()) &&
+ CellUtil.matchingQualifier(cell, TxConstants.FAMILY_DELETE_QUALIFIER) &&
+ CellUtil.matchingValue(cell, HConstants.EMPTY_BYTE_ARRAY);
+ }
+
+ public void addFamilyDelete(Cell delete) {
+ this.familyDeleteTs = delete.getTimestamp();
+ }
+
+ public boolean isDeleted(Cell cell) {
+ return cell.getTimestamp() <= familyDeleteTs;
+ }
+
+ public void reset() {
+ this.familyDeleteTs = 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/HBase96ConfigurationProviderTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/HBase96ConfigurationProviderTest.java b/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/HBase96ConfigurationProviderTest.java
deleted file mode 100644
index a1a2e97..0000000
--- a/tephra-hbase-compat-0.96/src/test/java/co/cask/tephra/hbase96/HBase96ConfigurationProviderTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package co.cask.tephra.hbase96;
-
-import co.cask.tephra.util.AbstractConfigurationProviderTest;
-import co.cask.tephra.util.HBaseVersion;
-
-/**
- * Test for HBase 0.96 version specific behavior.
- */
-public class HBase96ConfigurationProviderTest extends AbstractConfigurationProviderTest {
- @Override
- protected HBaseVersion.Version getExpectedVersion() {
- return HBaseVersion.Version.HBASE_96;
- }
-}
[37/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftHandler.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftHandler.java b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftHandler.java
new file mode 100644
index 0000000..8edc1a6
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/TransactionServiceThriftHandler.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra.distributed;
+
+import com.google.common.collect.Sets;
+import org.apache.tephra.InvalidTruncateTimeException;
+import org.apache.tephra.TransactionManager;
+import org.apache.tephra.TransactionNotInProgressException;
+import org.apache.tephra.TxConstants;
+import org.apache.tephra.distributed.thrift.TBoolean;
+import org.apache.tephra.distributed.thrift.TInvalidTruncateTimeException;
+import org.apache.tephra.distributed.thrift.TTransaction;
+import org.apache.tephra.distributed.thrift.TTransactionCouldNotTakeSnapshotException;
+import org.apache.tephra.distributed.thrift.TTransactionNotInProgressException;
+import org.apache.tephra.distributed.thrift.TTransactionServer;
+import org.apache.tephra.rpc.RPCServiceHandler;
+import org.apache.thrift.TException;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Set;
+
+/**
+ * The implementation of a thrift service for tx service.
+ * All operations arrive over the wire as Thrift objects.
+ * <p/>
+ * Why all this conversion (wrap/unwrap), and not define all Operations
+ * themselves as Thrift objects?
+ * <ul><li>
+ * All the non-thrift executors would have to use the Thrift objects
+ * </li><li>
+ * Thrift's object model is too restrictive: it has only limited inheritance
+ * and no overloading
+ * </li><li>
+ * Thrift objects are bare-bone, all they have are getters, setters, and
+ * basic object methods.
+ * </li></ul>
+ */
+public class TransactionServiceThriftHandler implements TTransactionServer.Iface, RPCServiceHandler {
+
+ private final TransactionManager txManager;
+
+ public TransactionServiceThriftHandler(TransactionManager txManager) {
+ this.txManager = txManager;
+ }
+
+ @Override
+ public TTransaction startLong() throws TException {
+ return TransactionConverterUtils.wrap(txManager.startLong());
+ }
+
+ @Override
+ public TTransaction startShort() throws TException {
+ return TransactionConverterUtils.wrap(txManager.startShort());
+ }
+
+ @Override
+ public TTransaction startShortTimeout(int timeout) throws TException {
+ return TransactionConverterUtils.wrap(txManager.startShort(timeout));
+ }
+
+
+ @Override
+ public TBoolean canCommitTx(TTransaction tx, Set<ByteBuffer> changes) throws TException {
+
+ Set<byte[]> changeIds = Sets.newHashSet();
+ for (ByteBuffer bb : changes) {
+ byte[] changeId = new byte[bb.remaining()];
+ bb.get(changeId);
+ changeIds.add(changeId);
+ }
+ try {
+ return new TBoolean(txManager.canCommit(TransactionConverterUtils.unwrap(tx), changeIds));
+ } catch (TransactionNotInProgressException e) {
+ throw new TTransactionNotInProgressException(e.getMessage());
+ }
+ }
+
+ @Override
+ public TBoolean commitTx(TTransaction tx) throws TException {
+ try {
+ return new TBoolean(txManager.commit(TransactionConverterUtils.unwrap(tx)));
+ } catch (TransactionNotInProgressException e) {
+ throw new TTransactionNotInProgressException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void abortTx(TTransaction tx) throws TException {
+ txManager.abort(TransactionConverterUtils.unwrap(tx));
+ }
+
+ @Override
+ public boolean invalidateTx(long tx) throws TException {
+ return txManager.invalidate(tx);
+ }
+
+ @Override
+ public ByteBuffer getSnapshot() throws TException {
+ try {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try {
+ boolean snapshotTaken = txManager.takeSnapshot(out);
+ if (!snapshotTaken) {
+ throw new TTransactionCouldNotTakeSnapshotException("Transaction manager could not get a snapshot.");
+ }
+ } finally {
+ out.close();
+ }
+ // todo find a way to encode directly to the stream, without having the snapshot in memory twice
+ return ByteBuffer.wrap(out.toByteArray());
+ } catch (IOException e) {
+ throw new TTransactionCouldNotTakeSnapshotException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void resetState() throws TException {
+ txManager.resetState();
+ }
+
+ @Override
+ public String status() throws TException {
+ return txManager.isRunning() ? TxConstants.STATUS_OK : TxConstants.STATUS_NOTOK;
+ }
+
+ @Override
+ public TBoolean truncateInvalidTx(Set<Long> txns) throws TException {
+ return new TBoolean(txManager.truncateInvalidTx(txns));
+ }
+
+ @Override
+ public TBoolean truncateInvalidTxBefore(long time) throws TException {
+ try {
+ return new TBoolean(txManager.truncateInvalidTxBefore(time));
+ } catch (InvalidTruncateTimeException e) {
+ throw new TInvalidTruncateTimeException(e.getMessage());
+ }
+ }
+
+ @Override
+ public int invalidTxSize() throws TException {
+ return txManager.getInvalidSize();
+ }
+
+ @Override
+ public TTransaction checkpoint(TTransaction originalTx) throws TException {
+ try {
+ return TransactionConverterUtils.wrap(
+ txManager.checkpoint(TransactionConverterUtils.unwrap(originalTx)));
+ } catch (TransactionNotInProgressException e) {
+ throw new TTransactionNotInProgressException(e.getMessage());
+ }
+ }
+
+ /* RPCServiceHandler implementation */
+
+ @Override
+ public void init() throws Exception {
+ txManager.startAndWait();
+ }
+
+ @Override
+ public void destroy() throws Exception {
+ txManager.stopAndWait();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/package-info.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/package-info.java b/tephra-core/src/main/java/org/apache/tephra/distributed/package-info.java
new file mode 100644
index 0000000..e001968
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains standalone server of the transaction system v2. It is now simple netty piece, but will change
+ * we integrate new RPC
+ */
+package org.apache.tephra.distributed;
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TBoolean.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TBoolean.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TBoolean.java
new file mode 100644
index 0000000..d33c7aa
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TBoolean.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+import org.apache.thrift.scheme.TupleScheme;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TBoolean implements org.apache.thrift.TBase<TBoolean, TBoolean._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TBoolean");
+
+ private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.BOOL, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TBooleanStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TBooleanTupleSchemeFactory());
+ }
+
+ public boolean value; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ VALUE((short)1, "value");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // VALUE
+ return VALUE;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __VALUE_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TBoolean.class, metaDataMap);
+ }
+
+ public TBoolean() {
+ }
+
+ public TBoolean(
+ boolean value)
+ {
+ this();
+ this.value = value;
+ setValueIsSet(true);
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TBoolean(TBoolean other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.value = other.value;
+ }
+
+ public TBoolean deepCopy() {
+ return new TBoolean(this);
+ }
+
+ @Override
+ public void clear() {
+ setValueIsSet(false);
+ this.value = false;
+ }
+
+ public boolean isValue() {
+ return this.value;
+ }
+
+ public TBoolean setValue(boolean value) {
+ this.value = value;
+ setValueIsSet(true);
+ return this;
+ }
+
+ public void unsetValue() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VALUE_ISSET_ID);
+ }
+
+ /** Returns true if field value is set (has been assigned a value) and false otherwise */
+ public boolean isSetValue() {
+ return EncodingUtils.testBit(__isset_bitfield, __VALUE_ISSET_ID);
+ }
+
+ public void setValueIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VALUE_ISSET_ID, value);
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case VALUE:
+ if (value == null) {
+ unsetValue();
+ } else {
+ setValue((Boolean)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case VALUE:
+ return Boolean.valueOf(isValue());
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case VALUE:
+ return isSetValue();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TBoolean)
+ return this.equals((TBoolean)that);
+ return false;
+ }
+
+ public boolean equals(TBoolean that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_value = true;
+ boolean that_present_value = true;
+ if (this_present_value || that_present_value) {
+ if (!(this_present_value && that_present_value))
+ return false;
+ if (this.value != that.value)
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(TBoolean other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ TBoolean typedOther = (TBoolean)other;
+
+ lastComparison = Boolean.valueOf(isSetValue()).compareTo(typedOther.isSetValue());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetValue()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, typedOther.value);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TBoolean(");
+ boolean first = true;
+
+ sb.append("value:");
+ sb.append(this.value);
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TBooleanStandardSchemeFactory implements SchemeFactory {
+ public TBooleanStandardScheme getScheme() {
+ return new TBooleanStandardScheme();
+ }
+ }
+
+ private static class TBooleanStandardScheme extends StandardScheme<TBoolean> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, TBoolean struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // VALUE
+ if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
+ struct.value = iprot.readBool();
+ struct.setValueIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, TBoolean struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(VALUE_FIELD_DESC);
+ oprot.writeBool(struct.value);
+ oprot.writeFieldEnd();
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TBooleanTupleSchemeFactory implements SchemeFactory {
+ public TBooleanTupleScheme getScheme() {
+ return new TBooleanTupleScheme();
+ }
+ }
+
+ private static class TBooleanTupleScheme extends TupleScheme<TBoolean> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, TBoolean struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetValue()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetValue()) {
+ oprot.writeBool(struct.value);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, TBoolean struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.value = iprot.readBool();
+ struct.setValueIsSet(true);
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TInvalidTruncateTimeException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TInvalidTruncateTimeException.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TInvalidTruncateTimeException.java
new file mode 100644
index 0000000..6587ae0
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TInvalidTruncateTimeException.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+import org.apache.thrift.scheme.TupleScheme;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TInvalidTruncateTimeException extends TException implements org.apache.thrift.TBase<TInvalidTruncateTimeException, TInvalidTruncateTimeException._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TInvalidTruncateTimeException");
+
+ private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TInvalidTruncateTimeExceptionStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TInvalidTruncateTimeExceptionTupleSchemeFactory());
+ }
+
+ public String message; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ MESSAGE((short)1, "message");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // MESSAGE
+ return MESSAGE;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.MESSAGE, new org.apache.thrift.meta_data.FieldMetaData("message", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TInvalidTruncateTimeException.class, metaDataMap);
+ }
+
+ public TInvalidTruncateTimeException() {
+ }
+
+ public TInvalidTruncateTimeException(
+ String message)
+ {
+ this();
+ this.message = message;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TInvalidTruncateTimeException(TInvalidTruncateTimeException other) {
+ if (other.isSetMessage()) {
+ this.message = other.message;
+ }
+ }
+
+ public TInvalidTruncateTimeException deepCopy() {
+ return new TInvalidTruncateTimeException(this);
+ }
+
+ @Override
+ public void clear() {
+ this.message = null;
+ }
+
+ public String getMessage() {
+ return this.message;
+ }
+
+ public TInvalidTruncateTimeException setMessage(String message) {
+ this.message = message;
+ return this;
+ }
+
+ public void unsetMessage() {
+ this.message = null;
+ }
+
+ /** Returns true if field message is set (has been assigned a value) and false otherwise */
+ public boolean isSetMessage() {
+ return this.message != null;
+ }
+
+ public void setMessageIsSet(boolean value) {
+ if (!value) {
+ this.message = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case MESSAGE:
+ if (value == null) {
+ unsetMessage();
+ } else {
+ setMessage((String)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case MESSAGE:
+ return getMessage();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case MESSAGE:
+ return isSetMessage();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TInvalidTruncateTimeException)
+ return this.equals((TInvalidTruncateTimeException)that);
+ return false;
+ }
+
+ public boolean equals(TInvalidTruncateTimeException that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_message = true && this.isSetMessage();
+ boolean that_present_message = true && that.isSetMessage();
+ if (this_present_message || that_present_message) {
+ if (!(this_present_message && that_present_message))
+ return false;
+ if (!this.message.equals(that.message))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(TInvalidTruncateTimeException other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ TInvalidTruncateTimeException typedOther = (TInvalidTruncateTimeException)other;
+
+ lastComparison = Boolean.valueOf(isSetMessage()).compareTo(typedOther.isSetMessage());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetMessage()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.message, typedOther.message);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TInvalidTruncateTimeException(");
+ boolean first = true;
+
+ sb.append("message:");
+ if (this.message == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.message);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TInvalidTruncateTimeExceptionStandardSchemeFactory implements SchemeFactory {
+ public TInvalidTruncateTimeExceptionStandardScheme getScheme() {
+ return new TInvalidTruncateTimeExceptionStandardScheme();
+ }
+ }
+
+ private static class TInvalidTruncateTimeExceptionStandardScheme extends StandardScheme<TInvalidTruncateTimeException> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // MESSAGE
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.message != null) {
+ oprot.writeFieldBegin(MESSAGE_FIELD_DESC);
+ oprot.writeString(struct.message);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TInvalidTruncateTimeExceptionTupleSchemeFactory implements SchemeFactory {
+ public TInvalidTruncateTimeExceptionTupleScheme getScheme() {
+ return new TInvalidTruncateTimeExceptionTupleScheme();
+ }
+ }
+
+ private static class TInvalidTruncateTimeExceptionTupleScheme extends TupleScheme<TInvalidTruncateTimeException> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetMessage()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetMessage()) {
+ oprot.writeString(struct.message);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, TInvalidTruncateTimeException struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransaction.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransaction.java b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransaction.java
new file mode 100644
index 0000000..ae2f72d
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/distributed/thrift/TTransaction.java
@@ -0,0 +1,1364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.tephra.distributed.thrift;
+
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+import org.apache.thrift.scheme.TupleScheme;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TTransaction implements org.apache.thrift.TBase<TTransaction, TTransaction._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTransaction");
+
+ private static final org.apache.thrift.protocol.TField TRANSACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("transactionId", org.apache.thrift.protocol.TType.I64, (short)1);
+ private static final org.apache.thrift.protocol.TField READ_POINTER_FIELD_DESC = new org.apache.thrift.protocol.TField("readPointer", org.apache.thrift.protocol.TType.I64, (short)2);
+ private static final org.apache.thrift.protocol.TField INVALIDS_FIELD_DESC = new org.apache.thrift.protocol.TField("invalids", org.apache.thrift.protocol.TType.LIST, (short)3);
+ private static final org.apache.thrift.protocol.TField IN_PROGRESS_FIELD_DESC = new org.apache.thrift.protocol.TField("inProgress", org.apache.thrift.protocol.TType.LIST, (short)4);
+ private static final org.apache.thrift.protocol.TField FIRST_SHORT_FIELD_DESC = new org.apache.thrift.protocol.TField("firstShort", org.apache.thrift.protocol.TType.I64, (short)5);
+ private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)6);
+ private static final org.apache.thrift.protocol.TField WRITE_POINTER_FIELD_DESC = new org.apache.thrift.protocol.TField("writePointer", org.apache.thrift.protocol.TType.I64, (short)7);
+ private static final org.apache.thrift.protocol.TField CHECKPOINT_WRITE_POINTERS_FIELD_DESC = new org.apache.thrift.protocol.TField("checkpointWritePointers", org.apache.thrift.protocol.TType.LIST, (short)8);
+ private static final org.apache.thrift.protocol.TField VISIBILITY_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("visibilityLevel", org.apache.thrift.protocol.TType.I32, (short)9);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TTransactionStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TTransactionTupleSchemeFactory());
+ }
+
+ public long transactionId; // required
+ public long readPointer; // required
+ public List<Long> invalids; // required
+ public List<Long> inProgress; // required
+ public long firstShort; // required
+ /**
+ *
+ * @see TTransactionType
+ */
+ public TTransactionType type; // required
+ public long writePointer; // required
+ public List<Long> checkpointWritePointers; // required
+ /**
+ *
+ * @see TVisibilityLevel
+ */
+ public TVisibilityLevel visibilityLevel; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ TRANSACTION_ID((short)1, "transactionId"),
+ READ_POINTER((short)2, "readPointer"),
+ INVALIDS((short)3, "invalids"),
+ IN_PROGRESS((short)4, "inProgress"),
+ FIRST_SHORT((short)5, "firstShort"),
+ /**
+ *
+ * @see TTransactionType
+ */
+ TYPE((short)6, "type"),
+ WRITE_POINTER((short)7, "writePointer"),
+ CHECKPOINT_WRITE_POINTERS((short)8, "checkpointWritePointers"),
+ /**
+ *
+ * @see TVisibilityLevel
+ */
+ VISIBILITY_LEVEL((short)9, "visibilityLevel");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // TRANSACTION_ID
+ return TRANSACTION_ID;
+ case 2: // READ_POINTER
+ return READ_POINTER;
+ case 3: // INVALIDS
+ return INVALIDS;
+ case 4: // IN_PROGRESS
+ return IN_PROGRESS;
+ case 5: // FIRST_SHORT
+ return FIRST_SHORT;
+ case 6: // TYPE
+ return TYPE;
+ case 7: // WRITE_POINTER
+ return WRITE_POINTER;
+ case 8: // CHECKPOINT_WRITE_POINTERS
+ return CHECKPOINT_WRITE_POINTERS;
+ case 9: // VISIBILITY_LEVEL
+ return VISIBILITY_LEVEL;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __TRANSACTIONID_ISSET_ID = 0;
+ private static final int __READPOINTER_ISSET_ID = 1;
+ private static final int __FIRSTSHORT_ISSET_ID = 2;
+ private static final int __WRITEPOINTER_ISSET_ID = 3;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.TRANSACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("transactionId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.READ_POINTER, new org.apache.thrift.meta_data.FieldMetaData("readPointer", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.INVALIDS, new org.apache.thrift.meta_data.FieldMetaData("invalids", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
+ tmpMap.put(_Fields.IN_PROGRESS, new org.apache.thrift.meta_data.FieldMetaData("inProgress", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
+ tmpMap.put(_Fields.FIRST_SHORT, new org.apache.thrift.meta_data.FieldMetaData("firstShort", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TTransactionType.class)));
+ tmpMap.put(_Fields.WRITE_POINTER, new org.apache.thrift.meta_data.FieldMetaData("writePointer", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.CHECKPOINT_WRITE_POINTERS, new org.apache.thrift.meta_data.FieldMetaData("checkpointWritePointers", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
+ tmpMap.put(_Fields.VISIBILITY_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("visibilityLevel", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TVisibilityLevel.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTransaction.class, metaDataMap);
+ }
+
+ public TTransaction() {
+ }
+
+ public TTransaction(
+ long transactionId,
+ long readPointer,
+ List<Long> invalids,
+ List<Long> inProgress,
+ long firstShort,
+ TTransactionType type,
+ long writePointer,
+ List<Long> checkpointWritePointers,
+ TVisibilityLevel visibilityLevel)
+ {
+ this();
+ this.transactionId = transactionId;
+ setTransactionIdIsSet(true);
+ this.readPointer = readPointer;
+ setReadPointerIsSet(true);
+ this.invalids = invalids;
+ this.inProgress = inProgress;
+ this.firstShort = firstShort;
+ setFirstShortIsSet(true);
+ this.type = type;
+ this.writePointer = writePointer;
+ setWritePointerIsSet(true);
+ this.checkpointWritePointers = checkpointWritePointers;
+ this.visibilityLevel = visibilityLevel;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TTransaction(TTransaction other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.transactionId = other.transactionId;
+ this.readPointer = other.readPointer;
+ if (other.isSetInvalids()) {
+ List<Long> __this__invalids = new ArrayList<Long>();
+ for (Long other_element : other.invalids) {
+ __this__invalids.add(other_element);
+ }
+ this.invalids = __this__invalids;
+ }
+ if (other.isSetInProgress()) {
+ List<Long> __this__inProgress = new ArrayList<Long>();
+ for (Long other_element : other.inProgress) {
+ __this__inProgress.add(other_element);
+ }
+ this.inProgress = __this__inProgress;
+ }
+ this.firstShort = other.firstShort;
+ if (other.isSetType()) {
+ this.type = other.type;
+ }
+ this.writePointer = other.writePointer;
+ if (other.isSetCheckpointWritePointers()) {
+ List<Long> __this__checkpointWritePointers = new ArrayList<Long>();
+ for (Long other_element : other.checkpointWritePointers) {
+ __this__checkpointWritePointers.add(other_element);
+ }
+ this.checkpointWritePointers = __this__checkpointWritePointers;
+ }
+ if (other.isSetVisibilityLevel()) {
+ this.visibilityLevel = other.visibilityLevel;
+ }
+ }
+
+ public TTransaction deepCopy() {
+ return new TTransaction(this);
+ }
+
+ @Override
+ public void clear() {
+ setTransactionIdIsSet(false);
+ this.transactionId = 0;
+ setReadPointerIsSet(false);
+ this.readPointer = 0;
+ this.invalids = null;
+ this.inProgress = null;
+ setFirstShortIsSet(false);
+ this.firstShort = 0;
+ this.type = null;
+ setWritePointerIsSet(false);
+ this.writePointer = 0;
+ this.checkpointWritePointers = null;
+ this.visibilityLevel = null;
+ }
+
+ public long getTransactionId() {
+ return this.transactionId;
+ }
+
+ public TTransaction setTransactionId(long transactionId) {
+ this.transactionId = transactionId;
+ setTransactionIdIsSet(true);
+ return this;
+ }
+
+ public void unsetTransactionId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TRANSACTIONID_ISSET_ID);
+ }
+
+ /** Returns true if field transactionId is set (has been assigned a value) and false otherwise */
+ public boolean isSetTransactionId() {
+ return EncodingUtils.testBit(__isset_bitfield, __TRANSACTIONID_ISSET_ID);
+ }
+
+ public void setTransactionIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TRANSACTIONID_ISSET_ID, value);
+ }
+
+ public long getReadPointer() {
+ return this.readPointer;
+ }
+
+ public TTransaction setReadPointer(long readPointer) {
+ this.readPointer = readPointer;
+ setReadPointerIsSet(true);
+ return this;
+ }
+
+ public void unsetReadPointer() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __READPOINTER_ISSET_ID);
+ }
+
+ /** Returns true if field readPointer is set (has been assigned a value) and false otherwise */
+ public boolean isSetReadPointer() {
+ return EncodingUtils.testBit(__isset_bitfield, __READPOINTER_ISSET_ID);
+ }
+
+ public void setReadPointerIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __READPOINTER_ISSET_ID, value);
+ }
+
+ public int getInvalidsSize() {
+ return (this.invalids == null) ? 0 : this.invalids.size();
+ }
+
+ public java.util.Iterator<Long> getInvalidsIterator() {
+ return (this.invalids == null) ? null : this.invalids.iterator();
+ }
+
+ public void addToInvalids(long elem) {
+ if (this.invalids == null) {
+ this.invalids = new ArrayList<Long>();
+ }
+ this.invalids.add(elem);
+ }
+
+ public List<Long> getInvalids() {
+ return this.invalids;
+ }
+
+ public TTransaction setInvalids(List<Long> invalids) {
+ this.invalids = invalids;
+ return this;
+ }
+
+ public void unsetInvalids() {
+ this.invalids = null;
+ }
+
+ /** Returns true if field invalids is set (has been assigned a value) and false otherwise */
+ public boolean isSetInvalids() {
+ return this.invalids != null;
+ }
+
+ public void setInvalidsIsSet(boolean value) {
+ if (!value) {
+ this.invalids = null;
+ }
+ }
+
+ public int getInProgressSize() {
+ return (this.inProgress == null) ? 0 : this.inProgress.size();
+ }
+
+ public java.util.Iterator<Long> getInProgressIterator() {
+ return (this.inProgress == null) ? null : this.inProgress.iterator();
+ }
+
+ public void addToInProgress(long elem) {
+ if (this.inProgress == null) {
+ this.inProgress = new ArrayList<Long>();
+ }
+ this.inProgress.add(elem);
+ }
+
+ public List<Long> getInProgress() {
+ return this.inProgress;
+ }
+
+ public TTransaction setInProgress(List<Long> inProgress) {
+ this.inProgress = inProgress;
+ return this;
+ }
+
+ public void unsetInProgress() {
+ this.inProgress = null;
+ }
+
+ /** Returns true if field inProgress is set (has been assigned a value) and false otherwise */
+ public boolean isSetInProgress() {
+ return this.inProgress != null;
+ }
+
+ public void setInProgressIsSet(boolean value) {
+ if (!value) {
+ this.inProgress = null;
+ }
+ }
+
+ public long getFirstShort() {
+ return this.firstShort;
+ }
+
+ public TTransaction setFirstShort(long firstShort) {
+ this.firstShort = firstShort;
+ setFirstShortIsSet(true);
+ return this;
+ }
+
+ public void unsetFirstShort() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __FIRSTSHORT_ISSET_ID);
+ }
+
+ /** Returns true if field firstShort is set (has been assigned a value) and false otherwise */
+ public boolean isSetFirstShort() {
+ return EncodingUtils.testBit(__isset_bitfield, __FIRSTSHORT_ISSET_ID);
+ }
+
+ public void setFirstShortIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __FIRSTSHORT_ISSET_ID, value);
+ }
+
+ /**
+ *
+ * @see TTransactionType
+ */
+ public TTransactionType getType() {
+ return this.type;
+ }
+
+ /**
+ *
+ * @see TTransactionType
+ */
+ public TTransaction setType(TTransactionType type) {
+ this.type = type;
+ return this;
+ }
+
+ public void unsetType() {
+ this.type = null;
+ }
+
+ /** Returns true if field type is set (has been assigned a value) and false otherwise */
+ public boolean isSetType() {
+ return this.type != null;
+ }
+
+ public void setTypeIsSet(boolean value) {
+ if (!value) {
+ this.type = null;
+ }
+ }
+
+ public long getWritePointer() {
+ return this.writePointer;
+ }
+
+ public TTransaction setWritePointer(long writePointer) {
+ this.writePointer = writePointer;
+ setWritePointerIsSet(true);
+ return this;
+ }
+
+ public void unsetWritePointer() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __WRITEPOINTER_ISSET_ID);
+ }
+
+ /** Returns true if field writePointer is set (has been assigned a value) and false otherwise */
+ public boolean isSetWritePointer() {
+ return EncodingUtils.testBit(__isset_bitfield, __WRITEPOINTER_ISSET_ID);
+ }
+
+ public void setWritePointerIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __WRITEPOINTER_ISSET_ID, value);
+ }
+
+ public int getCheckpointWritePointersSize() {
+ return (this.checkpointWritePointers == null) ? 0 : this.checkpointWritePointers.size();
+ }
+
+ public java.util.Iterator<Long> getCheckpointWritePointersIterator() {
+ return (this.checkpointWritePointers == null) ? null : this.checkpointWritePointers.iterator();
+ }
+
+ public void addToCheckpointWritePointers(long elem) {
+ if (this.checkpointWritePointers == null) {
+ this.checkpointWritePointers = new ArrayList<Long>();
+ }
+ this.checkpointWritePointers.add(elem);
+ }
+
+ public List<Long> getCheckpointWritePointers() {
+ return this.checkpointWritePointers;
+ }
+
+ public TTransaction setCheckpointWritePointers(List<Long> checkpointWritePointers) {
+ this.checkpointWritePointers = checkpointWritePointers;
+ return this;
+ }
+
+ public void unsetCheckpointWritePointers() {
+ this.checkpointWritePointers = null;
+ }
+
+ /** Returns true if field checkpointWritePointers is set (has been assigned a value) and false otherwise */
+ public boolean isSetCheckpointWritePointers() {
+ return this.checkpointWritePointers != null;
+ }
+
+ public void setCheckpointWritePointersIsSet(boolean value) {
+ if (!value) {
+ this.checkpointWritePointers = null;
+ }
+ }
+
+ /**
+ *
+ * @see TVisibilityLevel
+ */
+ public TVisibilityLevel getVisibilityLevel() {
+ return this.visibilityLevel;
+ }
+
+ /**
+ *
+ * @see TVisibilityLevel
+ */
+ public TTransaction setVisibilityLevel(TVisibilityLevel visibilityLevel) {
+ this.visibilityLevel = visibilityLevel;
+ return this;
+ }
+
+ public void unsetVisibilityLevel() {
+ this.visibilityLevel = null;
+ }
+
+ /** Returns true if field visibilityLevel is set (has been assigned a value) and false otherwise */
+ public boolean isSetVisibilityLevel() {
+ return this.visibilityLevel != null;
+ }
+
+ public void setVisibilityLevelIsSet(boolean value) {
+ if (!value) {
+ this.visibilityLevel = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case TRANSACTION_ID:
+ if (value == null) {
+ unsetTransactionId();
+ } else {
+ setTransactionId((Long)value);
+ }
+ break;
+
+ case READ_POINTER:
+ if (value == null) {
+ unsetReadPointer();
+ } else {
+ setReadPointer((Long)value);
+ }
+ break;
+
+ case INVALIDS:
+ if (value == null) {
+ unsetInvalids();
+ } else {
+ setInvalids((List<Long>)value);
+ }
+ break;
+
+ case IN_PROGRESS:
+ if (value == null) {
+ unsetInProgress();
+ } else {
+ setInProgress((List<Long>)value);
+ }
+ break;
+
+ case FIRST_SHORT:
+ if (value == null) {
+ unsetFirstShort();
+ } else {
+ setFirstShort((Long)value);
+ }
+ break;
+
+ case TYPE:
+ if (value == null) {
+ unsetType();
+ } else {
+ setType((TTransactionType)value);
+ }
+ break;
+
+ case WRITE_POINTER:
+ if (value == null) {
+ unsetWritePointer();
+ } else {
+ setWritePointer((Long)value);
+ }
+ break;
+
+ case CHECKPOINT_WRITE_POINTERS:
+ if (value == null) {
+ unsetCheckpointWritePointers();
+ } else {
+ setCheckpointWritePointers((List<Long>)value);
+ }
+ break;
+
+ case VISIBILITY_LEVEL:
+ if (value == null) {
+ unsetVisibilityLevel();
+ } else {
+ setVisibilityLevel((TVisibilityLevel)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case TRANSACTION_ID:
+ return Long.valueOf(getTransactionId());
+
+ case READ_POINTER:
+ return Long.valueOf(getReadPointer());
+
+ case INVALIDS:
+ return getInvalids();
+
+ case IN_PROGRESS:
+ return getInProgress();
+
+ case FIRST_SHORT:
+ return Long.valueOf(getFirstShort());
+
+ case TYPE:
+ return getType();
+
+ case WRITE_POINTER:
+ return Long.valueOf(getWritePointer());
+
+ case CHECKPOINT_WRITE_POINTERS:
+ return getCheckpointWritePointers();
+
+ case VISIBILITY_LEVEL:
+ return getVisibilityLevel();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case TRANSACTION_ID:
+ return isSetTransactionId();
+ case READ_POINTER:
+ return isSetReadPointer();
+ case INVALIDS:
+ return isSetInvalids();
+ case IN_PROGRESS:
+ return isSetInProgress();
+ case FIRST_SHORT:
+ return isSetFirstShort();
+ case TYPE:
+ return isSetType();
+ case WRITE_POINTER:
+ return isSetWritePointer();
+ case CHECKPOINT_WRITE_POINTERS:
+ return isSetCheckpointWritePointers();
+ case VISIBILITY_LEVEL:
+ return isSetVisibilityLevel();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TTransaction)
+ return this.equals((TTransaction)that);
+ return false;
+ }
+
+ public boolean equals(TTransaction that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_transactionId = true;
+ boolean that_present_transactionId = true;
+ if (this_present_transactionId || that_present_transactionId) {
+ if (!(this_present_transactionId && that_present_transactionId))
+ return false;
+ if (this.transactionId != that.transactionId)
+ return false;
+ }
+
+ boolean this_present_readPointer = true;
+ boolean that_present_readPointer = true;
+ if (this_present_readPointer || that_present_readPointer) {
+ if (!(this_present_readPointer && that_present_readPointer))
+ return false;
+ if (this.readPointer != that.readPointer)
+ return false;
+ }
+
+ boolean this_present_invalids = true && this.isSetInvalids();
+ boolean that_present_invalids = true && that.isSetInvalids();
+ if (this_present_invalids || that_present_invalids) {
+ if (!(this_present_invalids && that_present_invalids))
+ return false;
+ if (!this.invalids.equals(that.invalids))
+ return false;
+ }
+
+ boolean this_present_inProgress = true && this.isSetInProgress();
+ boolean that_present_inProgress = true && that.isSetInProgress();
+ if (this_present_inProgress || that_present_inProgress) {
+ if (!(this_present_inProgress && that_present_inProgress))
+ return false;
+ if (!this.inProgress.equals(that.inProgress))
+ return false;
+ }
+
+ boolean this_present_firstShort = true;
+ boolean that_present_firstShort = true;
+ if (this_present_firstShort || that_present_firstShort) {
+ if (!(this_present_firstShort && that_present_firstShort))
+ return false;
+ if (this.firstShort != that.firstShort)
+ return false;
+ }
+
+ boolean this_present_type = true && this.isSetType();
+ boolean that_present_type = true && that.isSetType();
+ if (this_present_type || that_present_type) {
+ if (!(this_present_type && that_present_type))
+ return false;
+ if (!this.type.equals(that.type))
+ return false;
+ }
+
+ boolean this_present_writePointer = true;
+ boolean that_present_writePointer = true;
+ if (this_present_writePointer || that_present_writePointer) {
+ if (!(this_present_writePointer && that_present_writePointer))
+ return false;
+ if (this.writePointer != that.writePointer)
+ return false;
+ }
+
+ boolean this_present_checkpointWritePointers = true && this.isSetCheckpointWritePointers();
+ boolean that_present_checkpointWritePointers = true && that.isSetCheckpointWritePointers();
+ if (this_present_checkpointWritePointers || that_present_checkpointWritePointers) {
+ if (!(this_present_checkpointWritePointers && that_present_checkpointWritePointers))
+ return false;
+ if (!this.checkpointWritePointers.equals(that.checkpointWritePointers))
+ return false;
+ }
+
+ boolean this_present_visibilityLevel = true && this.isSetVisibilityLevel();
+ boolean that_present_visibilityLevel = true && that.isSetVisibilityLevel();
+ if (this_present_visibilityLevel || that_present_visibilityLevel) {
+ if (!(this_present_visibilityLevel && that_present_visibilityLevel))
+ return false;
+ if (!this.visibilityLevel.equals(that.visibilityLevel))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(TTransaction other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ TTransaction typedOther = (TTransaction)other;
+
+ lastComparison = Boolean.valueOf(isSetTransactionId()).compareTo(typedOther.isSetTransactionId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTransactionId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.transactionId, typedOther.transactionId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetReadPointer()).compareTo(typedOther.isSetReadPointer());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetReadPointer()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.readPointer, typedOther.readPointer);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetInvalids()).compareTo(typedOther.isSetInvalids());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetInvalids()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.invalids, typedOther.invalids);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetInProgress()).compareTo(typedOther.isSetInProgress());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetInProgress()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.inProgress, typedOther.inProgress);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetFirstShort()).compareTo(typedOther.isSetFirstShort());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetFirstShort()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.firstShort, typedOther.firstShort);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetType()).compareTo(typedOther.isSetType());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetType()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, typedOther.type);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetWritePointer()).compareTo(typedOther.isSetWritePointer());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetWritePointer()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.writePointer, typedOther.writePointer);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetCheckpointWritePointers()).compareTo(typedOther.isSetCheckpointWritePointers());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetCheckpointWritePointers()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.checkpointWritePointers, typedOther.checkpointWritePointers);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetVisibilityLevel()).compareTo(typedOther.isSetVisibilityLevel());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetVisibilityLevel()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.visibilityLevel, typedOther.visibilityLevel);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TTransaction(");
+ boolean first = true;
+
+ sb.append("transactionId:");
+ sb.append(this.transactionId);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("readPointer:");
+ sb.append(this.readPointer);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("invalids:");
+ if (this.invalids == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.invalids);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("inProgress:");
+ if (this.inProgress == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.inProgress);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("firstShort:");
+ sb.append(this.firstShort);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("type:");
+ if (this.type == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.type);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("writePointer:");
+ sb.append(this.writePointer);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("checkpointWritePointers:");
+ if (this.checkpointWritePointers == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.checkpointWritePointers);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("visibilityLevel:");
+ if (this.visibilityLevel == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.visibilityLevel);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TTransactionStandardSchemeFactory implements SchemeFactory {
+ public TTransactionStandardScheme getScheme() {
+ return new TTransactionStandardScheme();
+ }
+ }
+
+ private static class TTransactionStandardScheme extends StandardScheme<TTransaction> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, TTransaction struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // TRANSACTION_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.transactionId = iprot.readI64();
+ struct.setTransactionIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // READ_POINTER
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.readPointer = iprot.readI64();
+ struct.setReadPointerIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 3: // INVALIDS
+ if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+ {
+ org.apache.thrift.protocol.TList _list0 = iprot.readListBegin();
+ struct.invalids = new ArrayList<Long>(_list0.size);
+ for (int _i1 = 0; _i1 < _list0.size; ++_i1)
+ {
+ long _elem2; // required
+ _elem2 = iprot.readI64();
+ struct.invalids.add(_elem2);
+ }
+ iprot.readListEnd();
+ }
+ struct.setInvalidsIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 4: // IN_PROGRESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+ {
+ org.apache.thrift.protocol.TList _list3 = iprot.readListBegin();
+ struct.inProgress = new ArrayList<Long>(_list3.size);
+ for (int _i4 = 0; _i4 < _list3.size; ++_i4)
+ {
+ long _elem5; // required
+ _elem5 = iprot.readI64();
+ struct.inProgress.add(_elem5);
+ }
+ iprot.readListEnd();
+ }
+ struct.setInProgressIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 5: // FIRST_SHORT
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.firstShort = iprot.readI64();
+ struct.setFirstShortIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 6: // TYPE
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.type = TTransactionType.findByValue(iprot.readI32());
+ struct.setTypeIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 7: // WRITE_POINTER
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.writePointer = iprot.readI64();
+ struct.setWritePointerIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 8: // CHECKPOINT_WRITE_POINTERS
+ if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+ {
+ org.apache.thrift.protocol.TList _list6 = iprot.readListBegin();
+ struct.checkpointWritePointers = new ArrayList<Long>(_list6.size);
+ for (int _i7 = 0; _i7 < _list6.size; ++_i7)
+ {
+ long _elem8; // required
+ _elem8 = iprot.readI64();
+ struct.checkpointWritePointers.add(_elem8);
+ }
+ iprot.readListEnd();
+ }
+ struct.setCheckpointWritePointersIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 9: // VISIBILITY_LEVEL
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.visibilityLevel = TVisibilityLevel.findByValue(iprot.readI32());
+ struct.setVisibilityLevelIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, TTransaction struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(TRANSACTION_ID_FIELD_DESC);
+ oprot.writeI64(struct.transactionId);
+ oprot.writeFieldEnd();
+ oprot.writeFieldBegin(READ_POINTER_FIELD_DESC);
+ oprot.writeI64(struct.readPointer);
+ oprot.writeFieldEnd();
+ if (struct.invalids != null) {
+ oprot.writeFieldBegin(INVALIDS_FIELD_DESC);
+ {
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.invalids.size()));
+ for (long _iter9 : struct.invalids)
+ {
+ oprot.writeI64(_iter9);
+ }
+ oprot.writeListEnd();
+ }
+ oprot.writeFieldEnd();
+ }
+ if (struct.inProgress != null) {
+ oprot.writeFieldBegin(IN_PROGRESS_FIELD_DESC);
+ {
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.inProgress.size()));
+ for (long _iter10 : struct.inProgress)
+ {
+ oprot.writeI64(_iter10);
+ }
+ oprot.writeListEnd();
+ }
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldBegin(FIRST_SHORT_FIELD_DESC);
+ oprot.writeI64(struct.firstShort);
+ oprot.writeFieldEnd();
+ if (struct.type != null) {
+ oprot.writeFieldBegin(TYPE_FIELD_DESC);
+ oprot.writeI32(struct.type.getValue());
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldBegin(WRITE_POINTER_FIELD_DESC);
+ oprot.writeI64(struct.writePointer);
+ oprot.writeFieldEnd();
+ if (struct.checkpointWritePointers != null) {
+ oprot.writeFieldBegin(CHECKPOINT_WRITE_POINTERS_FIELD_DESC);
+ {
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.checkpointWritePointers.size()));
+ for (long _iter11 : struct.checkpointWritePointers)
+ {
+ oprot.writeI64(_iter11);
+ }
+ oprot.writeListEnd();
+ }
+ oprot.writeFieldEnd();
+ }
+ if (struct.visibilityLevel != null) {
+ oprot.writeFieldBegin(VISIBILITY_LEVEL_FIELD_DESC);
+ oprot.writeI32(struct.visibilityLevel.getValue());
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TTransactionTupleSchemeFactory implements SchemeFactory {
+ public TTransactionTupleScheme getScheme() {
+ return new TTransactionTupleScheme();
+ }
+ }
+
+ private static class TTransactionTupleScheme extends TupleScheme<TTransaction> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, TTransaction struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetTransactionId()) {
+ optionals.set(0);
+ }
+ if (struct.isSetReadPointer()) {
+ optionals.set(1);
+ }
+ if (struct.isSetInvalids()) {
+ optionals.set(2);
+ }
+ if (struct.isSetInProgress()) {
+ optionals.set(3);
+ }
+ if (struct.isSetFirstShort()) {
+ optionals.set(4);
+ }
+ if (struct.isSetType()) {
+ optionals.set(5);
+ }
+ if (struct.isSetWritePointer()) {
+ optionals.set(6);
+ }
+ if (struct.isSetCheckpointWritePointers()) {
+ optionals.set(7);
+ }
+ if (struct.isSetVisibilityLevel()) {
+ optionals.set(8);
+ }
+ oprot.writeBitSet(optionals, 9);
+ if (struct.isSetTransactionId()) {
+ oprot.writeI64(struct.transactionId);
+ }
+ if (struct.isSetReadPointer()) {
+ oprot.writeI64(struct.readPointer);
+ }
+ if (struct.isSetInvalids()) {
+ {
+ oprot.writeI32(struct.invalids.size());
+ for (long _iter12 : struct.invalids)
+ {
+ oprot.writeI64(_iter12);
+ }
+ }
+ }
+ if (struct.isSetInProgress()) {
+ {
+ oprot.writeI32(struct.inProgress.size());
+ for (long _iter13 : struct.inProgress)
+ {
+ oprot.writeI64(_iter13);
+ }
+ }
+ }
+ if (struct.isSetFirstShort()) {
+ oprot.writeI64(struct.firstShort);
+ }
+ if (struct.isSetType()) {
+ oprot.writeI32(struct.type.getValue());
+ }
+ if (struct.isSetWritePointer()) {
+ oprot.writeI64(struct.writePointer);
+ }
+ if (struct.isSetCheckpointWritePointers()) {
+ {
+ oprot.writeI32(struct.checkpointWritePointers.size());
+ for (long _iter14 : struct.checkpointWritePointers)
+ {
+ oprot.writeI64(_iter14);
+ }
+ }
+ }
+ if (struct.isSetVisibilityLevel()) {
+ oprot.writeI32(struct.visibilityLevel.getValue());
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, TTransaction struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(9);
+ if (incoming.get(0)) {
+ struct.transactionId = iprot.readI64();
+ struct.setTransactionIdIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.readPointer = iprot.readI64();
+ struct.setReadPointerIsSet(true);
+ }
+ if (incoming.get(2)) {
+ {
+ org.apache.thrift.protocol.TList _list15 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
+ struct.invalids = new ArrayList<Long>(_list15.size);
+ for (int _i16 = 0; _i16 < _list15.size; ++_i16)
+ {
+ long _elem17; // required
+ _elem17 = iprot.readI64();
+ struct.invalids.add(_elem17);
+ }
+ }
+ struct.setInvalidsIsSet(true);
+ }
+ if (incoming.get(3)) {
+ {
+ org.apache.thrift.protocol.TList _list18 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
+ struct.inProgress = new ArrayList<Long>(_list18.size);
+ for (int _i19 = 0; _i19 < _list18.size; ++_i19)
+ {
+ long _elem20; // required
+ _elem20 = iprot.readI64();
+ struct.inProgress.add(_elem20);
+ }
+ }
+ struct.setInProgressIsSet(true);
+ }
+ if (incoming.get(4)) {
+ struct.firstShort = iprot.readI64();
+ struct.setFirstShortIsSet(true);
+ }
+ if (incoming.get(5)) {
+ struct.type = TTransactionType.findByValue(iprot.readI32());
+ struct.setTypeIsSet(true);
+ }
+ if (incoming.get(6)) {
+ struct.writePointer = iprot.readI64();
+ struct.setWritePointerIsSet(true);
+ }
+ if (incoming.get(7)) {
+ {
+ org.apache.thrift.protocol.TList _list21 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
+ struct.checkpointWritePointers = new ArrayList<Long>(_list21.size);
+ for (int _i22 = 0; _i22 < _list21.size; ++_i22)
+ {
+ long _elem23; // required
+ _elem23 = iprot.readI64();
+ struct.checkpointWritePointers.add(_elem23);
+ }
+ }
+ struct.setCheckpointWritePointersIsSet(true);
+ }
+ if (incoming.get(8)) {
+ struct.visibilityLevel = TVisibilityLevel.findByValue(iprot.readI32());
+ struct.setVisibilityLevelIsSet(true);
+ }
+ }
+ }
+
+}
+
[40/56] [abbrv] [partial] incubator-tephra git commit: Rename package
to org.apache.tephra
Posted by po...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/RetryStrategy.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/RetryStrategy.java b/tephra-core/src/main/java/org/apache/tephra/RetryStrategy.java
new file mode 100644
index 0000000..4c33538
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/RetryStrategy.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Retry strategy for failed transactions
+ */
+public interface RetryStrategy {
+ /**
+ * Returns the number of milliseconds to wait before retrying the operation.
+ *
+ * @param reason Reason for transaction failure.
+ * @param failureCount Number of times that the request has been failed.
+ * @return Number of milliseconds to wait before retrying the operation. Returning {@code 0} means
+ * retry it immediately, while negative means abort the operation.
+ */
+ long nextRetry(TransactionFailureException reason, int failureCount);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionAdmin.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionAdmin.java b/tephra-core/src/main/java/org/apache/tephra/TransactionAdmin.java
new file mode 100644
index 0000000..e5902a7
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionAdmin.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tephra.runtime.ConfigModule;
+import org.apache.tephra.runtime.DiscoveryModules;
+import org.apache.tephra.runtime.TransactionClientModule;
+import org.apache.tephra.runtime.TransactionModules;
+import org.apache.tephra.runtime.ZKModule;
+import org.apache.twill.zookeeper.ZKClientService;
+
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Allows calling some methods on {@link TransactionManager} from command line.
+ */
+public class TransactionAdmin {
+ private static final String OPT_TRUNCATE_INVALID_TX = "--truncate-invalid-tx";
+ private static final String OPT_TRUNCATE_INVALID_TX_BEFORE = "--truncate-invalid-tx-before";
+ private static final String OPT_GET_INVALID_TX_SIZE = "--get-invalid-tx-size";
+
+ private final PrintStream out;
+ private final PrintStream err;
+
+ public static void main(String[] args) {
+ TransactionAdmin txAdmin = new TransactionAdmin(System.out, System.err);
+ int status = txAdmin.doMain(args, new Configuration());
+ System.exit(status);
+ }
+
+ public TransactionAdmin(PrintStream out, PrintStream err) {
+ this.out = out;
+ this.err = err;
+ }
+
+ @VisibleForTesting
+ int doMain(String[] args, Configuration conf) {
+ if (args.length < 1) {
+ printUsage();
+ return 1;
+ }
+
+ Injector injector = Guice.createInjector(
+ new ConfigModule(conf),
+ new ZKModule(),
+ new DiscoveryModules().getDistributedModules(),
+ new TransactionModules().getDistributedModules(),
+ new TransactionClientModule()
+ );
+
+ ZKClientService zkClient = injector.getInstance(ZKClientService.class);
+ zkClient.startAndWait();
+
+ try {
+ TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
+ String option = args[0];
+
+ if (option.equals(OPT_TRUNCATE_INVALID_TX)) {
+ if (args.length != 2) {
+ printUsage();
+ return 1;
+ }
+ Set<Long> txIds;
+ try {
+ txIds = parseTxIds(args[1]);
+ } catch (NumberFormatException e) {
+ err.println("NumberFormatException: " + e.getMessage());
+ return 1;
+ }
+ if (!txIds.isEmpty()) {
+ out.println("Invalid list size before truncation: " + txClient.getInvalidSize());
+ txClient.truncateInvalidTx(txIds);
+ out.println("Invalid list size after truncation: " + txClient.getInvalidSize());
+ }
+ } else if (option.equals(OPT_TRUNCATE_INVALID_TX_BEFORE)) {
+ if (args.length != 2) {
+ printUsage();
+ return 1;
+ }
+ try {
+ long time = Long.parseLong(args[1]);
+ out.println("Invalid list size before truncation: " + txClient.getInvalidSize());
+ txClient.truncateInvalidTxBefore(time);
+ out.println("Invalid list size after truncation: " + txClient.getInvalidSize());
+ } catch (InvalidTruncateTimeException e) {
+ err.println(e.getMessage());
+ return 1;
+ } catch (NumberFormatException e) {
+ err.println("NumberFormatException: " + e.getMessage());
+ return 1;
+ }
+ } else if (option.equals(OPT_GET_INVALID_TX_SIZE)) {
+ if (args.length != 1) {
+ printUsage();
+ return 1;
+ }
+ out.println("Invalid list size: " + txClient.getInvalidSize());
+ } else {
+ printUsage();
+ return 1;
+ }
+ } finally {
+ zkClient.stopAndWait();
+ }
+ return 0;
+ }
+
+ private Set<Long> parseTxIds(String option) throws NumberFormatException {
+ Set<Long> txIds = Sets.newHashSet();
+ for (String str : Splitter.on(',').split(option)) {
+ txIds.add(Long.parseLong(str));
+ }
+ return txIds;
+ }
+
+ private void printUsage() {
+ String programName = TransactionAdmin.class.getSimpleName();
+ String spaces = " ";
+ List<String> options = Lists.newArrayList();
+ options.add(join("Usage: "));
+ options.add(join(spaces, programName, OPT_TRUNCATE_INVALID_TX, "<tx1,tx2,...>"));
+ options.add(join(spaces, programName, OPT_TRUNCATE_INVALID_TX_BEFORE, "<time in secs>"));
+ options.add(join(spaces, programName, OPT_GET_INVALID_TX_SIZE));
+
+ String usage = Joiner.on(System.getProperty("line.separator")).join(options);
+ err.println(usage);
+ }
+
+ private static String join(String... args) {
+ return Joiner.on(" ").join(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionCodec.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionCodec.java b/tephra-core/src/main/java/org/apache/tephra/TransactionCodec.java
new file mode 100644
index 0000000..c147917
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionCodec.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import org.apache.tephra.distributed.TransactionConverterUtils;
+import org.apache.tephra.distributed.thrift.TTransaction;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+
+import java.io.IOException;
+
+/**
+ * Handles serialization and deserialization of {@link Transaction} instances to and from {@code byte[]}.
+ */
+public class TransactionCodec {
+
+ public TransactionCodec() {
+ }
+
+ public byte[] encode(Transaction tx) throws IOException {
+ TTransaction thriftTx = TransactionConverterUtils.wrap(tx);
+ TSerializer serializer = new TSerializer();
+ try {
+ return serializer.serialize(thriftTx);
+ } catch (TException te) {
+ throw new IOException(te);
+ }
+ }
+
+ public Transaction decode(byte[] encoded) throws IOException {
+ TTransaction thriftTx = new TTransaction();
+ TDeserializer deserializer = new TDeserializer();
+ try {
+ deserializer.deserialize(thriftTx, encoded);
+ return TransactionConverterUtils.unwrap(thriftTx);
+ } catch (TException te) {
+ throw new IOException(te);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionContext.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionContext.java b/tephra-core/src/main/java/org/apache/tephra/TransactionContext.java
new file mode 100644
index 0000000..22a59c6
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionContext.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import javax.annotation.Nullable;
+
+/**
+ * Utility class that encapsulates the transaction life cycle over a given set of
+ * transaction-aware datasets. It is not thread-safe for concurrent execution.
+ */
+public class TransactionContext {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionContext.class);
+
+ private final Collection<TransactionAware> txAwares;
+ private final TransactionSystemClient txClient;
+
+ private Transaction currentTx;
+
+ public TransactionContext(TransactionSystemClient txClient, TransactionAware... txAwares) {
+ this(txClient, ImmutableList.copyOf(txAwares));
+ }
+
+ public TransactionContext(TransactionSystemClient txClient, Iterable<TransactionAware> txAwares) {
+ // Use a set to avoid adding the same TransactionAware twice and to make removal faster.
+ // Use a linked hash set so that insertion order is preserved (same behavior as when it was using a List).
+ this.txAwares = Sets.newLinkedHashSet(txAwares);
+ this.txClient = txClient;
+ }
+
+ /**
+ * Adds a new transaction-aware to participate in the transaction.
+ * @param txAware the new transaction-aware
+ */
+ public boolean addTransactionAware(TransactionAware txAware) {
+ // If the txAware is newly added, call startTx as well if there is an active transaction
+ boolean added = txAwares.add(txAware);
+ if (added && currentTx != null) {
+ txAware.startTx(currentTx);
+ }
+ return added;
+ }
+
+ /**
+ * Removes a {@link TransactionAware} and withdraws from participation in the transaction.
+ * Withdrawal is only allowed if there is no active transaction.
+ *
+ * @param txAware the {@link TransactionAware} to be removed
+ * @return true if the given {@link TransactionAware} is removed; false otherwise.
+ * @throws IllegalStateException if there is an active transaction going on with this TransactionContext.
+ */
+ public boolean removeTransactionAware(TransactionAware txAware) {
+ Preconditions.checkState(currentTx == null, "Cannot remove TransactionAware while there is an active transaction.");
+ return txAwares.remove(txAware);
+ }
+
+ /**
+ * Starts a new transaction. Calling this will initiate a new transaction using the {@link TransactionSystemClient},
+ * and pass the returned transaction to {@link TransactionAware#startTx(Transaction)} for each registered
+ * TransactionAware. If an exception is encountered, the transaction will be aborted and a
+ * {@code TransactionFailureException} wrapping the root cause will be thrown.
+ *
+ * @throws TransactionFailureException if an exception occurs starting the transaction with any registered
+ * TransactionAware
+ */
+ public void start() throws TransactionFailureException {
+ currentTx = txClient.startShort();
+ for (TransactionAware txAware : txAwares) {
+ try {
+ txAware.startTx(currentTx);
+ } catch (Throwable e) {
+ String message = String.format("Unable to start transaction-aware '%s' for transaction %d. ",
+ txAware.getTransactionAwareName(), currentTx.getTransactionId());
+ LOG.warn(message, e);
+ txClient.abort(currentTx);
+ throw new TransactionFailureException(message, e);
+ }
+ }
+ }
+
+ /**
+ * Commits the current transaction. This will: check for any conflicts, based on the change set aggregated from
+ * all registered {@link TransactionAware} instances; flush any pending writes from the {@code TransactionAware}s;
+ * commit the current transaction with the {@link TransactionSystemClient}; and clear the current transaction state.
+ *
+ * @throws TransactionConflictException if a conflict is detected with a recently committed transaction
+ * @throws TransactionFailureException if an error occurs while committing
+ */
+ public void finish() throws TransactionFailureException {
+ Preconditions.checkState(currentTx != null, "Cannot finish tx that has not been started");
+ // each of these steps will abort and rollback the tx in case if errors, and throw an exception
+ checkForConflicts();
+ persist();
+ commit();
+ postCommit();
+ currentTx = null;
+ }
+
+ /**
+ * Aborts the given transaction, and rolls back all data set changes. If rollback fails,
+ * the transaction is invalidated. If an exception is caught during rollback, the exception
+ * is rethrown wrapped in a TransactionFailureException, after all remaining TransactionAwares have
+ * completed rollback.
+ *
+ * @throws TransactionFailureException for any exception that is encountered.
+ */
+ public void abort() throws TransactionFailureException {
+ abort(null);
+ }
+
+ /**
+ * Checkpoints the current transaction by flushing any pending writes for the registered {@link TransactionAware}
+ * instances, and obtaining a new current write pointer for the transaction. By performing a checkpoint,
+ * the client can ensure that all previous writes were flushed and are visible. By default, the current write
+ * pointer for the transaction is also visible. The current write pointer can be excluded from read
+ * operations by calling {@link Transaction#setVisibility(Transaction.VisibilityLevel)} with the visibility level set
+ * to {@link Transaction.VisibilityLevel#SNAPSHOT_EXCLUDE_CURRENT} on the {@link Transaction} instance created
+ * by the checkpoint call, which can be retrieved by calling {@link #getCurrentTransaction()}.
+ *
+ * After the checkpoint operation is performed, the updated
+ * {@link Transaction} instance will be passed to {@link TransactionAware#startTx(Transaction)} for each
+ * registered {@code TransactionAware} instance.
+ *
+ * @throws TransactionFailureException if an error occurs while performing the checkpoint
+ */
+ public void checkpoint() throws TransactionFailureException {
+ Preconditions.checkState(currentTx != null, "Cannot checkpoint tx that has not been started");
+ persist();
+ try {
+ currentTx = txClient.checkpoint(currentTx);
+ // update the current transaction with all TransactionAwares
+ for (TransactionAware txAware : txAwares) {
+ txAware.updateTx(currentTx);
+ }
+ } catch (TransactionNotInProgressException e) {
+ String message = String.format("Transaction %d is not in progress.", currentTx.getTransactionId());
+ LOG.warn(message, e);
+ abort(new TransactionFailureException(message, e));
+ // abort will throw that exception
+ } catch (Throwable e) {
+ String message = String.format("Exception from checkpoint for transaction %d.", currentTx.getTransactionId());
+ LOG.warn(message, e);
+ abort(new TransactionFailureException(message, e));
+ // abort will throw that exception
+ }
+ }
+
+ /**
+ * Returns the current transaction or null if no transaction is currently in progress.
+ */
+ @Nullable
+ public Transaction getCurrentTransaction() {
+ return currentTx;
+ }
+
+ // CHECKSTYLE IGNORE "@throws" FOR 11 LINES
+ /**
+ * Aborts the given transaction, and rolls back all data set changes. If rollback fails,
+ * the transaction is invalidated. If an exception is caught during rollback, the exception
+ * is rethrown wrapped into a TransactionFailureException, after all remaining TransactionAwares have
+ * completed rollback. If an existing exception is passed in, that exception is thrown in either
+ * case, whether the rollback is successful or not. In other words, this method always throws the
+ * first exception that it encounters.
+ * @param cause the original exception that caused the abort
+ * @throws TransactionFailureException for any exception that is encountered.
+ */
+ public void abort(TransactionFailureException cause) throws TransactionFailureException {
+ if (currentTx == null) {
+ // might be called by some generic exception handler even though already aborted/finished - we allow that
+ return;
+ }
+ try {
+ boolean success = true;
+ for (TransactionAware txAware : txAwares) {
+ try {
+ if (!txAware.rollbackTx()) {
+ success = false;
+ }
+ } catch (Throwable e) {
+ String message = String.format("Unable to roll back changes in transaction-aware '%s' for transaction %d. ",
+ txAware.getTransactionAwareName(), currentTx.getTransactionId());
+ LOG.warn(message, e);
+ if (cause == null) {
+ cause = new TransactionFailureException(message, e);
+ }
+ success = false;
+ }
+ }
+ if (success) {
+ txClient.abort(currentTx);
+ } else {
+ txClient.invalidate(currentTx.getTransactionId());
+ }
+ if (cause != null) {
+ throw cause;
+ }
+ } finally {
+ currentTx = null;
+ }
+ }
+
+ private void checkForConflicts() throws TransactionFailureException {
+ Collection<byte[]> changes = Lists.newArrayList();
+ for (TransactionAware txAware : txAwares) {
+ try {
+ changes.addAll(txAware.getTxChanges());
+ } catch (Throwable e) {
+ String message = String.format("Unable to retrieve changes from transaction-aware '%s' for transaction %d. ",
+ txAware.getTransactionAwareName(), currentTx.getTransactionId());
+ LOG.warn(message, e);
+ abort(new TransactionFailureException(message, e));
+ // abort will throw that exception
+ }
+ }
+
+ boolean canCommit = false;
+ try {
+ canCommit = txClient.canCommit(currentTx, changes);
+ } catch (TransactionNotInProgressException e) {
+ String message = String.format("Transaction %d is not in progress.", currentTx.getTransactionId());
+ LOG.warn(message, e);
+ abort(new TransactionFailureException(message, e));
+ // abort will throw that exception
+ } catch (Throwable e) {
+ String message = String.format("Exception from canCommit for transaction %d.", currentTx.getTransactionId());
+ LOG.warn(message, e);
+ abort(new TransactionFailureException(message, e));
+ // abort will throw that exception
+ }
+ if (!canCommit) {
+ String message = String.format("Conflict detected for transaction %d.", currentTx.getTransactionId());
+ abort(new TransactionConflictException(message));
+ // abort will throw
+ }
+ }
+
+ private void persist() throws TransactionFailureException {
+ for (TransactionAware txAware : txAwares) {
+ boolean success;
+ Throwable cause = null;
+ try {
+ success = txAware.commitTx();
+ } catch (Throwable e) {
+ success = false;
+ cause = e;
+ }
+ if (!success) {
+ String message = String.format("Unable to persist changes of transaction-aware '%s' for transaction %d. ",
+ txAware.getTransactionAwareName(), currentTx.getTransactionId());
+ if (cause == null) {
+ LOG.warn(message);
+ } else {
+ LOG.warn(message, cause);
+ }
+ abort(new TransactionFailureException(message, cause));
+ // abort will throw that exception
+ }
+ }
+ }
+
+ private void commit() throws TransactionFailureException {
+ boolean commitSuccess = false;
+ try {
+ commitSuccess = txClient.commit(currentTx);
+ } catch (TransactionNotInProgressException e) {
+ String message = String.format("Transaction %d is not in progress.", currentTx.getTransactionId());
+ LOG.warn(message, e);
+ abort(new TransactionFailureException(message, e));
+ // abort will throw that exception
+ } catch (Throwable e) {
+ String message = String.format("Exception from commit for transaction %d.", currentTx.getTransactionId());
+ LOG.warn(message, e);
+ abort(new TransactionFailureException(message, e));
+ // abort will throw that exception
+ }
+ if (!commitSuccess) {
+ String message = String.format("Conflict detected for transaction %d.", currentTx.getTransactionId());
+ abort(new TransactionConflictException(message));
+ // abort will throw
+ }
+ }
+
+ private void postCommit() throws TransactionFailureException {
+ TransactionFailureException cause = null;
+ for (TransactionAware txAware : txAwares) {
+ try {
+ txAware.postTxCommit();
+ } catch (Throwable e) {
+ String message = String.format("Unable to perform post-commit in transaction-aware '%s' for transaction %d. ",
+ txAware.getTransactionAwareName(), currentTx.getTransactionId());
+ LOG.warn(message, e);
+ cause = new TransactionFailureException(message, e);
+ }
+ }
+ if (cause != null) {
+ throw cause;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionCouldNotTakeSnapshotException.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionCouldNotTakeSnapshotException.java b/tephra-core/src/main/java/org/apache/tephra/TransactionCouldNotTakeSnapshotException.java
new file mode 100644
index 0000000..3af9596
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionCouldNotTakeSnapshotException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * Throw when taking a snapshot fails.
+ */
+public class TransactionCouldNotTakeSnapshotException extends Exception {
+ public TransactionCouldNotTakeSnapshotException(String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionExecutor.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionExecutor.java b/tephra-core/src/main/java/org/apache/tephra/TransactionExecutor.java
new file mode 100644
index 0000000..ed5029e
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionExecutor.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Utility that wraps the execution of a function into the context of a transaction.
+ */
+// todo: implementations should throw different from TransactionFailureException in case of user code error?
+// todo: accept only Callable? Executors util has a way to convert everything to Callable...
+public interface TransactionExecutor {
+
+ /**
+ * A function is a class with a single method that takes an argument and returns a result.
+ * @param <I> the type of the argument
+ * @param <O> the type of the result
+ */
+ public interface Function<I, O> {
+ O apply(I input) throws Exception;
+ }
+
+ /**
+ * A procedure is a class with a single void method that takes an argument.
+ * @param <I> the type of the argument
+ */
+ public interface Procedure<I> {
+ void apply(I input) throws Exception;
+ }
+
+ /**
+ * A subroutine is a class with a single void method without arguments.
+ */
+ public interface Subroutine {
+ void apply() throws Exception;
+ }
+
+ // Due to a bug in checkstyle, it would emit false positives here of the form
+ // "Unable to get class information for @throws tag '<exn>' (...)".
+ // This comment disables that check up to the corresponding ON comments below
+
+ // CHECKSTYLE OFF: @throws
+
+ /**
+ * Execute a function under transactional semantics. A transaction is started and all datasets
+ * are initialized with the transaction. Then the passed function is executed, the transaction
+ * is committed, and the function return value is returned as the return value of this method.
+ * If any exception is caught, the transaction is aborted and the original exception is rethrown,
+ * wrapped into a TransactionFailureException. If the transaction fails due to a write conflict,
+ * a TransactionConflictException is thrown.
+ * @param function the function to execute
+ * @param input the input parameter for the function
+ * @param <I> the input type of the function
+ * @param <O> the result type of the function
+ * @return the function's return value
+ * @throws TransactionConflictException if there is a write conflict with another transaction.
+ * @throws TransactionFailureException if any exception is caught, be it from the function or from the datasets.
+ */
+ <I, O> O execute(Function<I, O> function, I input) throws TransactionFailureException, InterruptedException;
+
+ // CHECKSTYLE ON
+
+ /**
+ * Like {@link #execute(Function, Object)} but without a return value.
+ */
+ <I> void execute(Procedure<I> procedure, I input) throws TransactionFailureException, InterruptedException;
+
+ /**
+ * Like {@link #execute(Function, Object)} but the callable has no argument.
+ */
+ <O> O execute(Callable<O> callable) throws TransactionFailureException, InterruptedException;
+
+ /**
+ * Like {@link #execute(Function, Object)} but without argument or return value.
+ */
+ void execute(Subroutine subroutine) throws TransactionFailureException, InterruptedException;
+
+ /**
+ * Same as {@link #execute(Function, Object)} but
+ * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
+ */
+ <I, O> O executeUnchecked(Function<I, O> function, I input);
+
+ /**
+ * Same as {@link #execute(Procedure, Object)} but
+ * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
+ */
+ <I> void executeUnchecked(Procedure<I> procedure, I input);
+
+ /**
+ * Same as {@link #execute(Callable)} but
+ * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
+ */
+ <O> O executeUnchecked(Callable<O> callable);
+
+ /**
+ * Same as {@link #execute(Subroutine)} but
+ * suppresses exception with {@link com.google.common.base.Throwables#propagate(Throwable)}
+ */
+ void executeUnchecked(Subroutine subroutine);
+
+ /**
+ * Same as {@link #execute(Function, Object)} but executes asynchronously
+ */
+ <I, O> ListenableFuture<O> submit(Function<I, O> function, I input);
+
+ /**
+ * Same as {@link #execute(Procedure, Object)} but executes asynchronously
+ */
+ <I> ListenableFuture<?> submit(Procedure<I> procedure, I input);
+
+ /**
+ * Same as {@link #execute(Callable)} but executes asynchronously
+ */
+ <O> ListenableFuture<O> submit(Callable<O> callable);
+
+ /**
+ * Same as {@link #execute(Subroutine)} but executes asynchronously
+ */
+ ListenableFuture<?> submit(Subroutine subroutine);
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/9c693743/tephra-core/src/main/java/org/apache/tephra/TransactionExecutorFactory.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/TransactionExecutorFactory.java b/tephra-core/src/main/java/org/apache/tephra/TransactionExecutorFactory.java
new file mode 100644
index 0000000..afe0f33
--- /dev/null
+++ b/tephra-core/src/main/java/org/apache/tephra/TransactionExecutorFactory.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tephra;
+
+/**
+ * A factory for transaction executors.
+ */
+public interface TransactionExecutorFactory {
+
+ TransactionExecutor createExecutor(Iterable<TransactionAware> txAwares);
+
+}