You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2014/12/09 16:55:16 UTC
[2/4] hbase git commit: HBASE-12655 WALPerformanceEvaluation should
only add stat gathering listener once per WAL.
HBASE-12655 WALPerformanceEvaluation should only add stat gathering listener once per WAL.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/33d79bb5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/33d79bb5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/33d79bb5
Branch: refs/heads/master
Commit: 33d79bb5eca16edc61a81c87676f866613c06ce8
Parents: a609313
Author: Sean Busbey <bu...@apache.org>
Authored: Mon Dec 8 10:12:27 2014 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 9 09:52:12 2014 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/wal/WAL.java | 3 +
.../hbase/wal/TestDefaultWALProvider.java | 27 +++++++++
.../hbase/wal/WALPerformanceEvaluation.java | 63 +++++++++++---------
3 files changed, 65 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/33d79bb5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 787a34b..23f8c9f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
/**
* A Write Ahead Log (WAL) provides service for reading, writing waledits. This interface provides
* APIs for WAL users (such as RegionServer) to use the WAL (do append, sync, etc).
+ *
+ * Note that some internals, such as log rolling and performance evaluation tools, will use
+ * WAL.equals to determine if they have already seen a given WAL.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hbase/blob/33d79bb5/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
index 9798b6c..e008b60 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
@@ -19,9 +19,14 @@
package org.apache.hadoop.hbase.wal;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
@@ -330,4 +335,26 @@ public class TestDefaultWALProvider {
new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
assertEquals(0, errCode);
}
+
+ /**
+ * Ensure that we can use Set.add to deduplicate WALs
+ */
+ @Test
+ public void setMembershipDedups() throws IOException {
+ final Configuration localConf = new Configuration(conf);
+ localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
+ final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
+ try {
+ final Set<WAL> seen = new HashSet<WAL>(1);
+ final Random random = new Random();
+ assertTrue("first attempt to add WAL from default provider should work.",
+ seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
+ for (int i = 0; i < 1000; i++) {
+ assertFalse("default wal provider is only supposed to return a single wal, which should " +
+ "compare as .equals itself.", seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
+ }
+ } finally {
+ wals.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/33d79bb5/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
index c00ebca..bfc7147 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
@@ -21,9 +21,11 @@ package org.apache.hadoop.hbase.wal;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -460,43 +462,48 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
System.exit(1);
}
+ private final Set<WAL> walsListenedTo = new HashSet<WAL>();
+
private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd,
final WALFactory wals, final long whenToRoll) throws IOException {
// Initialize HRegion
HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
// Initialize WAL
final WAL wal = wals.getWAL(regionInfo.getEncodedNameAsBytes());
- wal.registerWALActionsListener(new WALActionsListener.Base() {
- private int appends = 0;
-
- @Override
- public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
- WALEdit logEdit) {
- this.appends++;
- if (this.appends % whenToRoll == 0) {
- LOG.info("Rolling after " + appends + " edits");
- // We used to do explicit call to rollWriter but changed it to a request
- // to avoid dead lock (there are less threads going on in this class than
- // in the regionserver -- regionserver does not have the issue).
- // TODO I think this means no rolling actually happens; the request relies on there
- // being a LogRoller.
- DefaultWALProvider.requestLogRoll(wal);
+ // If we haven't already, attach a listener to this wal to handle rolls and metrics.
+ if (walsListenedTo.add(wal)) {
+ wal.registerWALActionsListener(new WALActionsListener.Base() {
+ private int appends = 0;
+
+ @Override
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
+ WALEdit logEdit) {
+ this.appends++;
+ if (this.appends % whenToRoll == 0) {
+ LOG.info("Rolling after " + appends + " edits");
+ // We used to do explicit call to rollWriter but changed it to a request
+ // to avoid dead lock (there are less threads going on in this class than
+ // in the regionserver -- regionserver does not have the issue).
+ // TODO I think this means no rolling actually happens; the request relies on there
+ // being a LogRoller.
+ DefaultWALProvider.requestLogRoll(wal);
+ }
}
- }
- @Override
- public void postSync(final long timeInNanos, final int handlerSyncs) {
- syncMeter.mark();
- syncHistogram.update(timeInNanos);
- syncCountHistogram.update(handlerSyncs);
- }
+ @Override
+ public void postSync(final long timeInNanos, final int handlerSyncs) {
+ syncMeter.mark();
+ syncHistogram.update(timeInNanos);
+ syncCountHistogram.update(handlerSyncs);
+ }
- @Override
- public void postAppend(final long size, final long elapsedTime) {
- appendMeter.mark(size);
- }
- });
- wal.rollWriter();
+ @Override
+ public void postAppend(final long size, final long elapsedTime) {
+ appendMeter.mark(size);
+ }
+ });
+ wal.rollWriter();
+ }
return HRegion.createHRegion(regionInfo, dir, getConf(), htd, wal);
}