You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/03/15 00:28:34 UTC
svn commit: r1577760 - in /hbase/branches/0.96: ./
hbase-common/src/main/java/org/apache/hadoop/hbase/util/
hbase-common/src/test/java/org/apache/hadoop/hbase/util/ hbase-server/
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ hbase-server/src/...
Author: stack
Date: Fri Mar 14 23:28:34 2014
New Revision: 1577760
URL: http://svn.apache.org/r1577760
Log:
HBASE-10656 high-scale-lib's Counter depends on Oracle (Sun) JRE, and also has some bug
Added:
hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Counter.java
hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCounter.java
Modified:
hbase/branches/0.96/hbase-server/pom.xml
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
hbase/branches/0.96/pom.xml
Added: hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Counter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Counter.java?rev=1577760&view=auto
==============================================================================
--- hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Counter.java (added)
+++ hbase/branches/0.96/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Counter.java Fri Mar 14 23:28:34 2014
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * High scalable counter. Thread safe.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Counter {
+ private static final int MAX_CELLS_LENGTH = 1 << 20;
+ private static final int SUFFERABLE_SPIN_COUNT = 2;
+
+ private static class Cell {
+ // Pads are added around the value to avoid cache-line contention with
+ // another cell's value. The cache-line size is expected to be equal to or
+ // less than about 128 Bytes (= 64 Bits * 16).
+
+ @SuppressWarnings("unused")
+ volatile long p0, p1, p2, p3, p4, p5, p6;
+ volatile long value;
+ @SuppressWarnings("unused")
+ volatile long q0, q1, q2, q3, q4, q5, q6;
+
+ static final AtomicLongFieldUpdater<Cell> valueUpdater =
+ AtomicLongFieldUpdater.newUpdater(Cell.class, "value");
+
+ Cell() {}
+
+ Cell(long initValue) {
+ value = initValue;
+ }
+
+ long get() {
+ return value;
+ }
+
+ boolean addAndIsCongested(long delta) {
+ for(int i = 0; i < SUFFERABLE_SPIN_COUNT; i++) {
+ if(add(delta)) {
+ return false;
+ }
+ }
+
+ while(! add(delta)) {}
+
+ return true;
+ }
+
+ boolean add(long delta) {
+ long current = value;
+ return valueUpdater.compareAndSet(this, current, current + delta);
+ }
+ }
+
+ private static class Container {
+ /** The length should be a power of 2. */
+ final Cell[] cells;
+
+ /** True if a new extended container is going to replace this. */
+ final AtomicBoolean demoted = new AtomicBoolean();
+
+ Container(Cell cell) {
+ this(new Cell[] { cell });
+ }
+
+ /**
+ * @param cells the length should be a power of 2
+ */
+ Container(Cell[] cells) {
+ this.cells = cells;
+ }
+ }
+
+ private final AtomicReference<Container> containerRef;
+
+ public Counter() {
+ this(new Cell());
+ }
+
+ public Counter(long initValue) {
+ this(new Cell(initValue));
+ }
+
+ private Counter(Cell initCell) {
+ containerRef = new AtomicReference<Container>(new Container(initCell));
+ }
+
+ private static int hash() {
+ return (int) Thread.currentThread().getId();
+ }
+
+ public void add(long delta) {
+ Container container = containerRef.get();
+ Cell[] cells = container.cells;
+ int index = hash() & (cells.length - 1);
+ Cell cell = cells[index];
+
+ if(cell.addAndIsCongested(delta) && cells.length < MAX_CELLS_LENGTH &&
+ container.demoted.compareAndSet(false, true)) {
+
+ if(containerRef.get() == container) {
+ Cell[] newCells = new Cell[cells.length * 2];
+ System.arraycopy(cells, 0, newCells, 0, cells.length);
+ for(int i = cells.length; i < newCells.length; i++) {
+ newCells[i] = new Cell();
+ // Fill all of the elements with instances. Creating a cell on demand
+ // and putting it into the array makes a concurrent problem about
+ // visibility or, in other words, happens-before relation, because
+ // each element of the array is not volatile so that you should
+ // establish the relation by some piggybacking.
+ }
+ containerRef.compareAndSet(container, new Container(newCells));
+ }
+ }
+ }
+
+ public void increment() {
+ add(1);
+ }
+
+ public void decrement() {
+ add(-1);
+ }
+
+ public void set(long value) {
+ containerRef.set(new Container(new Cell(value)));
+ }
+
+ public long get() {
+ long sum = 0;
+ for(Cell cell : containerRef.get().cells) {
+ sum += cell.get();
+ }
+ return sum;
+ }
+
+ @Override
+ public String toString() {
+ Cell[] cells = containerRef.get().cells;
+
+ long min = Long.MAX_VALUE;
+ long max = Long.MIN_VALUE;
+ long sum = 0;
+
+ for(Cell cell : cells) {
+ long value = cell.get();
+ sum += value;
+ if(min > value) { min = value; }
+ if(max < value) { max = value; }
+ }
+
+ return new StringBuilder(100)
+ .append("[value=").append(sum)
+ .append(", cells=[length=").append(cells.length)
+ .append(", min=").append(min)
+ .append(", max=").append(max)
+ .append("]]").toString();
+ }
+}
Added: hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCounter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCounter.java?rev=1577760&view=auto
==============================================================================
--- hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCounter.java (added)
+++ hbase/branches/0.96/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCounter.java Fri Mar 14 23:28:34 2014
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestCounter {
+ private static final int[] THREAD_COUNTS = {1, 10, 100};
+ private static final int DATA_COUNT = 1000000;
+
+ private interface Operation {
+ void execute();
+ }
+
+ @Test
+ public void testIncrement() throws Exception {
+ for(int threadCount : THREAD_COUNTS) {
+ final Counter counter = new Counter();
+
+ execute(new Operation() {
+ @Override
+ public void execute() {
+ counter.increment();
+ }
+ }, threadCount);
+
+ Assert.assertEquals(threadCount * (long)DATA_COUNT, counter.get());
+ }
+ }
+
+ @Test
+ public void testIncrementAndGet() throws Exception {
+ for(int threadCount: THREAD_COUNTS) {
+ final Counter counter = new Counter();
+
+ execute(new Operation() {
+ @Override
+ public void execute() {
+ counter.increment();
+ counter.get();
+ }
+ }, threadCount);
+
+ Assert.assertEquals(threadCount * (long)DATA_COUNT, counter.get());
+ }
+ }
+
+ private static void execute(final Operation op, int threadCount)
+ throws InterruptedException {
+
+ final CountDownLatch prepareLatch = new CountDownLatch(threadCount);
+ final CountDownLatch startLatch = new CountDownLatch(1);
+ final CountDownLatch endLatch = new CountDownLatch(threadCount);
+
+ class OperationThread extends Thread {
+ @Override
+ public void run() {
+ try {
+ prepareLatch.countDown();
+ startLatch.await();
+
+ for(int i=0; i<DATA_COUNT; i++) {
+ op.execute();
+ }
+
+ endLatch.countDown();
+
+ } catch(Exception e) {}
+ }
+ }
+
+ for(int j=0; j<threadCount; j++) {
+ new OperationThread().start();
+ }
+
+ prepareLatch.await();
+ startLatch.countDown();
+ endLatch.await();
+ }
+}
Modified: hbase/branches/0.96/hbase-server/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/pom.xml?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/pom.xml (original)
+++ hbase/branches/0.96/hbase-server/pom.xml Fri Mar 14 23:28:34 2014
@@ -342,10 +342,6 @@
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
- <groupId>com.github.stephenc.high-scale-lib</groupId>
- <artifactId>high-scale-lib</artifactId>
- </dependency>
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java Fri Mar 14 23:28:34 2014
@@ -96,6 +96,7 @@ import org.apache.hadoop.hbase.security.
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.token.AuthenticationTokenSecretManager;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
@@ -114,7 +115,6 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.StringUtils;
-import org.cliffc.high_scale_lib.Counter;
import org.cloudera.htrace.Trace;
import org.cloudera.htrace.TraceInfo;
import org.cloudera.htrace.TraceScope;
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java Fri Mar 14 23:28:34 2014
@@ -66,7 +66,6 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.KeeperException;
-import org.cliffc.high_scale_lib.Counter;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -300,9 +299,6 @@ public class TableMapReduceUtil {
TableSnapshotInputFormat.setInput(job, snapshotName, tmpRestoreDir);
initTableMapperJob(snapshotName, scan, mapper, outputKeyClass,
outputValueClass, job, addDependencyJars, false, TableSnapshotInputFormat.class);
-
- // We would need even more libraries that hbase-server depends on
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Counter.class);
}
/**
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/metrics/ExactCounterMetric.java Fri Mar 14 23:28:34 2014
@@ -27,11 +27,11 @@ import java.util.concurrent.ConcurrentMa
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.metrics.MetricsRecord;
import org.apache.hadoop.metrics.util.MetricsBase;
import org.apache.hadoop.metrics.util.MetricsRegistry;
-import org.cliffc.high_scale_lib.Counter;
import com.google.common.collect.Lists;
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java Fri Mar 14 23:28:34 2014
@@ -356,9 +356,6 @@ public class CompactionTool extends Conf
// add dependencies (including HBase ones)
TableMapReduceUtil.addDependencyJars(job);
- // This job instantiates HRegions, which requires the Counter class from the high_scale library
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
- org.cliffc.high_scale_lib.Counter.class);
Path stagingDir = JobUtil.getStagingDir(conf);
try {
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Fri Mar 14 23:28:34 2014
@@ -130,6 +130,7 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.CompressionTest;
+import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HashedBytes;
@@ -137,7 +138,6 @@ import org.apache.hadoop.hbase.util.Pair
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.util.StringUtils;
-import org.cliffc.high_scale_lib.Counter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -5129,9 +5129,9 @@ public class HRegion implements HeapSize
// woefully out of date - currently missing:
// 1 x HashMap - coprocessorServiceHandlers
- // 6 org.cliffc.high_scale_lib.Counter - numMutationsWithoutWAL, dataInMemoryWithoutWAL,
+ // 6 x Counter - numMutationsWithoutWAL, dataInMemoryWithoutWAL,
// checkAndMutateChecksPassed, checkAndMutateChecksFailed, readRequestsCount,
- // writeRequestsCount, updatesBlockedMs
+ // writeRequestsCount
// 1 x HRegion$WriteState - writestate
// 1 x RegionCoprocessorHost - coprocessorHost
// 1 x RegionSplitPolicy - splitPolicy
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri Mar 14 23:28:34 2014
@@ -204,6 +204,7 @@ import org.apache.hadoop.hbase.security.
import org.apache.hadoop.hbase.trace.SpanReceiverHost;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CompressionTest;
+import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -229,7 +230,6 @@ import org.apache.hadoop.util.Reflection
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
-import org.cliffc.high_scale_lib.Counter;
import com.google.protobuf.BlockingRpcChannel;
import com.google.protobuf.ByteString;
Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java (original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java Fri Mar 14 23:28:34 2014
@@ -47,11 +47,11 @@ import org.apache.hadoop.hbase.util.Envi
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.util.StringUtils;
-import org.cliffc.high_scale_lib.Counter;
import com.google.common.base.Preconditions;
import org.cloudera.htrace.Trace;
import org.cloudera.htrace.TraceScope;
+import org.apache.hadoop.hbase.util.Counter;
/**
* Thread that flushes cache on request
Modified: hbase/branches/0.96/pom.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/pom.xml?rev=1577760&r1=1577759&r2=1577760&view=diff
==============================================================================
--- hbase/branches/0.96/pom.xml (original)
+++ hbase/branches/0.96/pom.xml Fri Mar 14 23:28:34 2014
@@ -1086,11 +1086,6 @@
<version>${commons-cli.version}</version>
</dependency>
<dependency>
- <groupId>com.github.stephenc.high-scale-lib</groupId>
- <artifactId>high-scale-lib</artifactId>
- <version>1.1.1</version>
- </dependency>
- <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons-codec.version}</version>