You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jd...@apache.org on 2013/07/18 00:05:57 UTC
svn commit: r1504300 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/io/hfile/
test/java/org/apache/hadoop/hbase/replication/
Author: jdcryans
Date: Wed Jul 17 22:05:56 2013
New Revision: 1504300
URL: http://svn.apache.org/r1504300
Log:
HBASE-8732 HFileBlockDefaultEncodingContext isn't thread-safe but is used by all readers,
breaks column encoding
HBASE-8961 Make TestReplicationQueueFailover less integration-y by splitting it up
Added:
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java
Removed:
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java?rev=1504300&r1=1504299&r2=1504300&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java Wed Jul 17 22:05:56 2013
@@ -41,7 +41,7 @@ import com.google.common.base.Preconditi
public class HFileDataBlockEncoderImpl implements HFileDataBlockEncoder {
private final DataBlockEncoding onDisk;
private final DataBlockEncoding inCache;
- private final HFileBlockEncodingContext inCacheEncodeCtx;
+ private final byte[] dummyHeader;
public HFileDataBlockEncoderImpl(DataBlockEncoding encoding) {
this(encoding, encoding);
@@ -75,16 +75,7 @@ public class HFileDataBlockEncoderImpl i
onDisk : DataBlockEncoding.NONE;
this.inCache = inCache != null ?
inCache : DataBlockEncoding.NONE;
- if (inCache != DataBlockEncoding.NONE) {
- inCacheEncodeCtx =
- this.inCache.getEncoder().newDataBlockEncodingContext(
- Algorithm.NONE, this.inCache, dummyHeader);
- } else {
- // create a default encoding context
- inCacheEncodeCtx =
- new HFileBlockDefaultEncodingContext(Algorithm.NONE,
- this.inCache, dummyHeader);
- }
+ this.dummyHeader = dummyHeader;
Preconditions.checkArgument(onDisk == DataBlockEncoding.NONE ||
onDisk == inCache, "on-disk encoding (" + onDisk + ") must be " +
@@ -166,7 +157,7 @@ public class HFileDataBlockEncoderImpl i
}
// Encode the unencoded block with the in-cache encoding.
return encodeDataBlock(block, inCache, block.doesIncludeMemstoreTS(),
- inCacheEncodeCtx);
+ createInCacheEncodingContext());
}
if (block.getBlockType() == BlockType.ENCODED_DATA) {
@@ -256,6 +247,22 @@ public class HFileDataBlockEncoderImpl i
return encodedBlock;
}
+ /**
+ * Returns a new encoding context given the inCache encoding scheme provided in the constructor.
+ * This used to be kept around but HFileBlockDefaultEncodingContext isn't thread-safe.
+ * See HBASE-8732
+ * @return a new in cache encoding context
+ */
+ private HFileBlockEncodingContext createInCacheEncodingContext() {
+ return (inCache != DataBlockEncoding.NONE) ?
+ this.inCache.getEncoder().newDataBlockEncodingContext(
+ Algorithm.NONE, this.inCache, dummyHeader)
+ :
+ // create a default encoding context
+ new HFileBlockDefaultEncodingContext(Algorithm.NONE,
+ this.inCache, dummyHeader);
+ }
+
@Override
public String toString() {
return getClass().getSimpleName() + "(onDisk=" + onDisk + ", inCache=" +
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java?rev=1504300&r1=1504299&r2=1504300&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java Wed Jul 17 22:05:56 2013
@@ -88,6 +88,7 @@ public class TestReplicationBase {
conf1.setBoolean("dfs.support.append", true);
conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
conf1.setInt("replication.stats.thread.period.seconds", 5);
+ conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
utility1 = new HBaseTestingUtility(conf1);
utility1.startMiniZKCluster();
@@ -105,6 +106,7 @@ public class TestReplicationBase {
conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
conf2.setBoolean("dfs.support.append", true);
+ conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
utility2 = new HBaseTestingUtility(conf2);
utility2.setZkCluster(miniZK);
@@ -127,7 +129,7 @@ public class TestReplicationBase {
HBaseAdmin admin1 = new HBaseAdmin(conf1);
HBaseAdmin admin2 = new HBaseAdmin(conf2);
admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
- admin2.createTable(table);
+ admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
htable1 = new HTable(conf1, tableName);
htable1.setWriteBufferSize(1024);
htable2 = new HTable(conf2, tableName);
Added: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java?rev=1504300&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java (added)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java Wed Jul 17 22:05:56 2013
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Runs the TestReplicationKillRS test and selects the RS to kill in the master cluster
+ * Do not add other tests in this class.
+ */
+@Category(LargeTests.class)
+public class TestReplicationKillMasterRS extends TestReplicationKillRS {
+
+ @Test(timeout=300000)
+ public void killOneMasterRS() throws Exception {
+ loadTableAndKillRS(utility1);
+ }
+
+}
Added: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java?rev=1504300&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java (added)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java Wed Jul 17 22:05:56 2013
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Run the same test as TestReplicationKillMasterRS but with HLog compression enabled
+ * Do not add other tests in this class.
+ */
+@Category(LargeTests.class)
+public class TestReplicationKillMasterRSCompressed extends TestReplicationKillMasterRS {
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);
+ TestReplicationBase.setUpBeforeClass();
+ }
+}
Added: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java?rev=1504300&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java (added)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java Wed Jul 17 22:05:56 2013
@@ -0,0 +1,128 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.exceptions.UnknownScannerException;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.fail;
+
+@Category(LargeTests.class)
+public class TestReplicationKillRS extends TestReplicationBase {
+
+ private static final Log LOG = LogFactory.getLog(TestReplicationKillRS.class);
+
+ /**
+ * Load up 1 tables over 2 region servers and kill a source during
+ * the upload. The failover happens internally.
+ *
+ * WARNING this test sometimes fails because of HBASE-3515
+ *
+ * @throws Exception
+ */
+ public void loadTableAndKillRS(HBaseTestingUtility util) throws Exception {
+ // killing the RS with .META. can result into failed puts until we solve
+ // IO fencing
+ int rsToKill1 =
+ util.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0;
+
+ // Takes about 20 secs to run the full loading, kill around the middle
+ Thread killer = killARegionServer(util, 5000, rsToKill1);
+
+ LOG.info("Start loading table");
+ int initialCount = utility1.loadTable(htable1, famName);
+ LOG.info("Done loading table");
+ killer.join(5000);
+ LOG.info("Done waiting for threads");
+
+ Result[] res;
+ while (true) {
+ try {
+ Scan scan = new Scan();
+ ResultScanner scanner = htable1.getScanner(scan);
+ res = scanner.next(initialCount);
+ scanner.close();
+ break;
+ } catch (UnknownScannerException ex) {
+ LOG.info("Cluster wasn't ready yet, restarting scanner");
+ }
+ }
+ // Test we actually have all the rows, we may miss some because we
+ // don't have IO fencing.
+ if (res.length != initialCount) {
+ LOG.warn("We lost some rows on the master cluster!");
+ // We don't really expect the other cluster to have more rows
+ initialCount = res.length;
+ }
+
+ int lastCount = 0;
+
+ final long start = System.currentTimeMillis();
+ int i = 0;
+ while (true) {
+ if (i==NB_RETRIES-1) {
+ fail("Waited too much time for queueFailover replication. " +
+ "Waited "+(System.currentTimeMillis() - start)+"ms.");
+ }
+ Scan scan2 = new Scan();
+ ResultScanner scanner2 = htable2.getScanner(scan2);
+ Result[] res2 = scanner2.next(initialCount * 2);
+ scanner2.close();
+ if (res2.length < initialCount) {
+ if (lastCount < res2.length) {
+ i--; // Don't increment timeout if we make progress
+ } else {
+ i++;
+ }
+ lastCount = res2.length;
+ LOG.info("Only got " + lastCount + " rows instead of " +
+ initialCount + " current i=" + i);
+ Thread.sleep(SLEEP_TIME*2);
+ } else {
+ break;
+ }
+ }
+ }
+
+ private static Thread killARegionServer(final HBaseTestingUtility utility,
+ final long timeout, final int rs) {
+ Thread killer = new Thread() {
+ public void run() {
+ try {
+ Thread.sleep(timeout);
+ utility.expireRegionServerSession(rs);
+ } catch (Exception e) {
+ LOG.error("Couldn't kill a region server", e);
+ }
+ }
+ };
+ killer.setDaemon(true);
+ killer.start();
+ return killer;
+ }
+}
Added: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java?rev=1504300&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java (added)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java Wed Jul 17 22:05:56 2013
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Runs the TestReplicationKillRS test and selects the RS to kill in the slave cluster
+ * Do not add other tests in this class.
+ */
+@Category(LargeTests.class)
+public class TestReplicationKillSlaveRS extends TestReplicationKillRS {
+
+ @Test(timeout=300000)
+ public void killOneSlaveRS() throws Exception {
+ loadTableAndKillRS(utility2);
+ }
+}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java?rev=1504300&r1=1504299&r2=1504300&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java Wed Jul 17 22:05:56 2013
@@ -240,6 +240,7 @@ public class TestReplicationSmallTests e
assertEquals(NB_ROWS_IN_BATCH, res1.length);
for (int i = 0; i < NB_RETRIES; i++) {
+ scan = new Scan();
if (i==NB_RETRIES-1) {
fail("Waited too much time for normal batch replication");
}
@@ -378,10 +379,10 @@ public class TestReplicationSmallTests e
assertEquals(NB_ROWS_IN_BIG_BATCH, res.length);
- scan = new Scan();
long start = System.currentTimeMillis();
for (int i = 0; i < NB_RETRIES; i++) {
+ scan = new Scan();
scanner = htable2.getScanner(scan);
res = scanner.next(NB_ROWS_IN_BIG_BATCH);