You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2014/12/09 16:55:15 UTC
[1/4] hbase git commit: HBASE-12661 add missing licenses.
Repository: hbase
Updated Branches:
refs/heads/branch-1 553c386c8 -> d2d6945e8
refs/heads/master c42c36521 -> 33d79bb5e
HBASE-12661 add missing licenses.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a6093139
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a6093139
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a6093139
Branch: refs/heads/master
Commit: a6093139ebceec76ef5092486221f872983419e6
Parents: c42c365
Author: Sean Busbey <bu...@apache.org>
Authored: Tue Dec 9 01:09:31 2014 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 9 09:51:55 2014 -0600
----------------------------------------------------------------------
.../MetricsReplicationSinkSource.java | 18 ++++++++++++++++++
.../MetricsReplicationSourceFactoryImpl.java | 17 +++++++++++++++++
.../MetricsReplicationSourceSourceImpl.java | 17 +++++++++++++++++
.../TestMetricsReplicationSourceFactoryImpl.java | 19 ++++++++++++++++++-
4 files changed, 70 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a6093139/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
index d73b24b..805dfca 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hbase.replication.regionserver;
public interface MetricsReplicationSinkSource {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a6093139/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
index cb78d3e..b07790f 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.replication.regionserver;
public class MetricsReplicationSourceFactoryImpl implements MetricsReplicationSourceFactory {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a6093139/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index 0e6b8c8..89ef4de 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a6093139/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
index 20162c2..d370b17 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
@@ -15,4 +32,4 @@ public class TestMetricsReplicationSourceFactoryImpl {
assertTrue(rms instanceof MetricsReplicationSourceFactoryImpl);
}
-}
\ No newline at end of file
+}
[4/4] hbase git commit: HBASE-12655 WALPerformanceEvaluation should
only add stat gathering listener once per WAL.
Posted by bu...@apache.org.
HBASE-12655 WALPerformanceEvaluation should only add stat gathering listener once per WAL.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d2d6945e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d2d6945e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d2d6945e
Branch: refs/heads/branch-1
Commit: d2d6945e8951391a3b04d89f2879b51d7df26bb7
Parents: 2372739
Author: Sean Busbey <bu...@apache.org>
Authored: Mon Dec 8 10:12:27 2014 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 9 09:53:25 2014 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/wal/WAL.java | 3 +
.../hbase/wal/TestDefaultWALProvider.java | 27 +++++++++
.../hbase/wal/WALPerformanceEvaluation.java | 63 +++++++++++---------
3 files changed, 65 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d2d6945e/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 787a34b..23f8c9f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
/**
* A Write Ahead Log (WAL) provides service for reading, writing waledits. This interface provides
* APIs for WAL users (such as RegionServer) to use the WAL (do append, sync, etc).
+ *
+ * Note that some internals, such as log rolling and performance evaluation tools, will use
+ * WAL.equals to determine if they have already seen a given WAL.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hbase/blob/d2d6945e/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
index 8eef74c..28cd849 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
@@ -19,9 +19,14 @@
package org.apache.hadoop.hbase.wal;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
@@ -329,4 +334,26 @@ public class TestDefaultWALProvider {
new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
assertEquals(0, errCode);
}
+
+ /**
+ * Ensure that we can use Set.add to deduplicate WALs
+ */
+ @Test
+ public void setMembershipDedups() throws IOException {
+ final Configuration localConf = new Configuration(conf);
+ localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
+ final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
+ try {
+ final Set<WAL> seen = new HashSet<WAL>(1);
+ final Random random = new Random();
+ assertTrue("first attempt to add WAL from default provider should work.",
+ seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
+ for (int i = 0; i < 1000; i++) {
+ assertFalse("default wal provider is only supposed to return a single wal, which should " +
+ "compare as .equals itself.", seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
+ }
+ } finally {
+ wals.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d2d6945e/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
index c00ebca..bfc7147 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
@@ -21,9 +21,11 @@ package org.apache.hadoop.hbase.wal;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -460,43 +462,48 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
System.exit(1);
}
+ private final Set<WAL> walsListenedTo = new HashSet<WAL>();
+
private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd,
final WALFactory wals, final long whenToRoll) throws IOException {
// Initialize HRegion
HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
// Initialize WAL
final WAL wal = wals.getWAL(regionInfo.getEncodedNameAsBytes());
- wal.registerWALActionsListener(new WALActionsListener.Base() {
- private int appends = 0;
-
- @Override
- public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
- WALEdit logEdit) {
- this.appends++;
- if (this.appends % whenToRoll == 0) {
- LOG.info("Rolling after " + appends + " edits");
- // We used to do explicit call to rollWriter but changed it to a request
- // to avoid dead lock (there are less threads going on in this class than
- // in the regionserver -- regionserver does not have the issue).
- // TODO I think this means no rolling actually happens; the request relies on there
- // being a LogRoller.
- DefaultWALProvider.requestLogRoll(wal);
+ // If we haven't already, attach a listener to this wal to handle rolls and metrics.
+ if (walsListenedTo.add(wal)) {
+ wal.registerWALActionsListener(new WALActionsListener.Base() {
+ private int appends = 0;
+
+ @Override
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
+ WALEdit logEdit) {
+ this.appends++;
+ if (this.appends % whenToRoll == 0) {
+ LOG.info("Rolling after " + appends + " edits");
+ // We used to do explicit call to rollWriter but changed it to a request
+ // to avoid dead lock (there are less threads going on in this class than
+ // in the regionserver -- regionserver does not have the issue).
+ // TODO I think this means no rolling actually happens; the request relies on there
+ // being a LogRoller.
+ DefaultWALProvider.requestLogRoll(wal);
+ }
}
- }
- @Override
- public void postSync(final long timeInNanos, final int handlerSyncs) {
- syncMeter.mark();
- syncHistogram.update(timeInNanos);
- syncCountHistogram.update(handlerSyncs);
- }
+ @Override
+ public void postSync(final long timeInNanos, final int handlerSyncs) {
+ syncMeter.mark();
+ syncHistogram.update(timeInNanos);
+ syncCountHistogram.update(handlerSyncs);
+ }
- @Override
- public void postAppend(final long size, final long elapsedTime) {
- appendMeter.mark(size);
- }
- });
- wal.rollWriter();
+ @Override
+ public void postAppend(final long size, final long elapsedTime) {
+ appendMeter.mark(size);
+ }
+ });
+ wal.rollWriter();
+ }
return HRegion.createHRegion(regionInfo, dir, getConf(), htd, wal);
}
[3/4] hbase git commit: HBASE-12661 add missing licenses.
Posted by bu...@apache.org.
HBASE-12661 add missing licenses.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2372739e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2372739e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2372739e
Branch: refs/heads/branch-1
Commit: 2372739e462bc4ee404815657f37141a2599177b
Parents: 553c386
Author: Sean Busbey <bu...@apache.org>
Authored: Tue Dec 9 01:09:31 2014 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 9 09:52:55 2014 -0600
----------------------------------------------------------------------
.../MetricsReplicationSinkSource.java | 18 ++++++++++++++++++
.../MetricsReplicationSourceFactoryImpl.java | 17 +++++++++++++++++
.../MetricsReplicationSourceSourceImpl.java | 17 +++++++++++++++++
.../TestMetricsReplicationSourceFactoryImpl.java | 19 ++++++++++++++++++-
4 files changed, 70 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2372739e/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
index d73b24b..805dfca 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hbase.replication.regionserver;
public interface MetricsReplicationSinkSource {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2372739e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
index cb78d3e..b07790f 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceFactoryImpl.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.replication.regionserver;
public class MetricsReplicationSourceFactoryImpl implements MetricsReplicationSourceFactory {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2372739e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index 0e6b8c8..89ef4de 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
http://git-wip-us.apache.org/repos/asf/hbase/blob/2372739e/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
index 20162c2..d370b17 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetricsReplicationSourceFactoryImpl.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
@@ -15,4 +32,4 @@ public class TestMetricsReplicationSourceFactoryImpl {
assertTrue(rms instanceof MetricsReplicationSourceFactoryImpl);
}
-}
\ No newline at end of file
+}
[2/4] hbase git commit: HBASE-12655 WALPerformanceEvaluation should
only add stat gathering listener once per WAL.
Posted by bu...@apache.org.
HBASE-12655 WALPerformanceEvaluation should only add stat gathering listener once per WAL.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/33d79bb5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/33d79bb5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/33d79bb5
Branch: refs/heads/master
Commit: 33d79bb5eca16edc61a81c87676f866613c06ce8
Parents: a609313
Author: Sean Busbey <bu...@apache.org>
Authored: Mon Dec 8 10:12:27 2014 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 9 09:52:12 2014 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/wal/WAL.java | 3 +
.../hbase/wal/TestDefaultWALProvider.java | 27 +++++++++
.../hbase/wal/WALPerformanceEvaluation.java | 63 +++++++++++---------
3 files changed, 65 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/33d79bb5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 787a34b..23f8c9f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
/**
* A Write Ahead Log (WAL) provides service for reading, writing waledits. This interface provides
* APIs for WAL users (such as RegionServer) to use the WAL (do append, sync, etc).
+ *
+ * Note that some internals, such as log rolling and performance evaluation tools, will use
+ * WAL.equals to determine if they have already seen a given WAL.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hbase/blob/33d79bb5/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
index 9798b6c..e008b60 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
@@ -19,9 +19,14 @@
package org.apache.hadoop.hbase.wal;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
@@ -330,4 +335,26 @@ public class TestDefaultWALProvider {
new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
assertEquals(0, errCode);
}
+
+ /**
+ * Ensure that we can use Set.add to deduplicate WALs
+ */
+ @Test
+ public void setMembershipDedups() throws IOException {
+ final Configuration localConf = new Configuration(conf);
+ localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
+ final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
+ try {
+ final Set<WAL> seen = new HashSet<WAL>(1);
+ final Random random = new Random();
+ assertTrue("first attempt to add WAL from default provider should work.",
+ seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
+ for (int i = 0; i < 1000; i++) {
+ assertFalse("default wal provider is only supposed to return a single wal, which should " +
+ "compare as .equals itself.", seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
+ }
+ } finally {
+ wals.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/33d79bb5/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
index c00ebca..bfc7147 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
@@ -21,9 +21,11 @@ package org.apache.hadoop.hbase.wal;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -460,43 +462,48 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
System.exit(1);
}
+ private final Set<WAL> walsListenedTo = new HashSet<WAL>();
+
private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd,
final WALFactory wals, final long whenToRoll) throws IOException {
// Initialize HRegion
HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
// Initialize WAL
final WAL wal = wals.getWAL(regionInfo.getEncodedNameAsBytes());
- wal.registerWALActionsListener(new WALActionsListener.Base() {
- private int appends = 0;
-
- @Override
- public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
- WALEdit logEdit) {
- this.appends++;
- if (this.appends % whenToRoll == 0) {
- LOG.info("Rolling after " + appends + " edits");
- // We used to do explicit call to rollWriter but changed it to a request
- // to avoid dead lock (there are less threads going on in this class than
- // in the regionserver -- regionserver does not have the issue).
- // TODO I think this means no rolling actually happens; the request relies on there
- // being a LogRoller.
- DefaultWALProvider.requestLogRoll(wal);
+ // If we haven't already, attach a listener to this wal to handle rolls and metrics.
+ if (walsListenedTo.add(wal)) {
+ wal.registerWALActionsListener(new WALActionsListener.Base() {
+ private int appends = 0;
+
+ @Override
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
+ WALEdit logEdit) {
+ this.appends++;
+ if (this.appends % whenToRoll == 0) {
+ LOG.info("Rolling after " + appends + " edits");
+ // We used to do explicit call to rollWriter but changed it to a request
+ // to avoid dead lock (there are less threads going on in this class than
+ // in the regionserver -- regionserver does not have the issue).
+ // TODO I think this means no rolling actually happens; the request relies on there
+ // being a LogRoller.
+ DefaultWALProvider.requestLogRoll(wal);
+ }
}
- }
- @Override
- public void postSync(final long timeInNanos, final int handlerSyncs) {
- syncMeter.mark();
- syncHistogram.update(timeInNanos);
- syncCountHistogram.update(handlerSyncs);
- }
+ @Override
+ public void postSync(final long timeInNanos, final int handlerSyncs) {
+ syncMeter.mark();
+ syncHistogram.update(timeInNanos);
+ syncCountHistogram.update(handlerSyncs);
+ }
- @Override
- public void postAppend(final long size, final long elapsedTime) {
- appendMeter.mark(size);
- }
- });
- wal.rollWriter();
+ @Override
+ public void postAppend(final long size, final long elapsedTime) {
+ appendMeter.mark(size);
+ }
+ });
+ wal.rollWriter();
+ }
return HRegion.createHRegion(regionInfo, dir, getConf(), htd, wal);
}