You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2018/02/10 03:09:45 UTC
hbase git commit: HBASE-19966 The WriteEntry for WALKey maybe null if
we failed to call WAL.append
Repository: hbase
Updated Branches:
refs/heads/master 2b63af376 -> 32f235abe
HBASE-19966 The WriteEntry for WALKey maybe null if we failed to call WAL.append
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/32f235ab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/32f235ab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/32f235ab
Branch: refs/heads/master
Commit: 32f235abee5bd04aa2c788cca822e4d514697dac
Parents: 2b63af3
Author: zhangduo <zh...@apache.org>
Authored: Fri Feb 9 20:51:55 2018 +0800
Committer: Michael Stack <st...@apache.org>
Committed: Fri Feb 9 19:09:40 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/wal/WALUtil.java | 4 +-
.../org/apache/hadoop/hbase/wal/WALKeyImpl.java | 25 +++---
.../regionserver/wal/AbstractTestFSWAL.java | 88 +++++++++++++-------
.../hbase/regionserver/wal/TestFSHLog.java | 30 +++----
4 files changed, 87 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/32f235ab/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
index f226c80..19b2ab1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
@@ -156,7 +156,9 @@ public class WALUtil {
// Call complete only here because these are markers only. They are not for clients to read.
mvcc.complete(walKey.getWriteEntry());
} catch (IOException ioe) {
- mvcc.complete(walKey.getWriteEntry());
+ if (walKey.getWriteEntry() != null) {
+ mvcc.complete(walKey.getWriteEntry());
+ }
throw ioe;
}
return walKey;
http://git-wip-us.apache.org/repos/asf/hbase/blob/32f235ab/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
index 7abd4a5..c1a77ee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,35 +17,32 @@
*/
package org.apache.hadoop.hbase.wal;
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.regionserver.SequenceId;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
import java.io.IOException;
-import java.io.InterruptedIOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.UUID;
-
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
-// imports for things that haven't moved from regionserver.wal yet.
+import org.apache.hadoop.hbase.regionserver.SequenceId;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FamilyScope;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.ScopeType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
/**
* Default implementation of Key for an Entry in the WAL.
@@ -78,8 +74,7 @@ public class WALKeyImpl implements WALKey {
* @return A WriteEntry gotten from local WAL subsystem.
* @see #setWriteEntry(MultiVersionConcurrencyControl.WriteEntry)
*/
- public MultiVersionConcurrencyControl.WriteEntry getWriteEntry() throws InterruptedIOException {
- assert this.writeEntry != null;
+ public MultiVersionConcurrencyControl.WriteEntry getWriteEntry() {
return this.writeEntry;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/32f235ab/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
index 009cca0..1cdb6e5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
@@ -17,11 +17,13 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.hamcrest.CoreMatchers.*;
import java.io.IOException;
import java.util.ArrayList;
@@ -32,7 +34,6 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -41,16 +42,17 @@ import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -153,7 +155,7 @@ public abstract class AbstractTestFSWAL {
}
}
- protected void addEdits(WAL log, RegionInfo hri, HTableDescriptor htd, int times,
+ protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times,
MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes)
throws IOException {
final byte[] row = Bytes.toBytes("row");
@@ -249,26 +251,20 @@ public abstract class AbstractTestFSWAL {
conf1.setInt("hbase.regionserver.maxlogs", 1);
AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(conf1), DIR.toString(),
HConstants.HREGION_OLDLOGDIR_NAME, conf1, null, true, null, null);
- HTableDescriptor t1 =
- new HTableDescriptor(TableName.valueOf("t1")).addFamily(new HColumnDescriptor("row"));
- HTableDescriptor t2 =
- new HTableDescriptor(TableName.valueOf("t2")).addFamily(new HColumnDescriptor("row"));
- RegionInfo hri1 = RegionInfoBuilder.newBuilder(t1.getTableName())
- .setStartKey(HConstants.EMPTY_START_ROW)
- .setEndKey(HConstants.EMPTY_END_ROW)
- .build();
- RegionInfo hri2 = RegionInfoBuilder.newBuilder(t2.getTableName())
- .setStartKey(HConstants.EMPTY_START_ROW)
- .setEndKey(HConstants.EMPTY_END_ROW)
- .build();
+ TableDescriptor t1 = TableDescriptorBuilder.newBuilder(TableName.valueOf("t1"))
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
+ TableDescriptor t2 = TableDescriptorBuilder.newBuilder(TableName.valueOf("t2"))
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
+ RegionInfo hri1 = RegionInfoBuilder.newBuilder(t1.getTableName()).build();
+ RegionInfo hri2 = RegionInfoBuilder.newBuilder(t2.getTableName()).build();
// add edits and roll the wal
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
NavigableMap<byte[], Integer> scopes1 = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : t1.getFamiliesKeys()) {
+ for (byte[] fam : t1.getColumnFamilyNames()) {
scopes1.put(fam, 0);
}
NavigableMap<byte[], Integer> scopes2 = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : t2.getFamiliesKeys()) {
+ for (byte[] fam : t2.getColumnFamilyNames()) {
scopes2.put(fam, 0);
}
try {
@@ -293,12 +289,12 @@ public abstract class AbstractTestFSWAL {
assertEquals(hri1.getEncodedNameAsBytes(), regionsToFlush[0]);
// flush region 1, and roll the wal file. Only last wal which has entries for region1 should
// remain.
- flushRegion(wal, hri1.getEncodedNameAsBytes(), t1.getFamiliesKeys());
+ flushRegion(wal, hri1.getEncodedNameAsBytes(), t1.getColumnFamilyNames());
wal.rollWriter();
// only one wal should remain now (that is for the second region).
assertEquals(1, wal.getNumRolledLogFiles());
// flush the second region
- flushRegion(wal, hri2.getEncodedNameAsBytes(), t2.getFamiliesKeys());
+ flushRegion(wal, hri2.getEncodedNameAsBytes(), t2.getColumnFamilyNames());
wal.rollWriter(true);
// no wal should remain now.
assertEquals(0, wal.getNumRolledLogFiles());
@@ -315,14 +311,14 @@ public abstract class AbstractTestFSWAL {
regionsToFlush = wal.findRegionsToForceFlush();
assertEquals(2, regionsToFlush.length);
// flush both regions
- flushRegion(wal, hri1.getEncodedNameAsBytes(), t1.getFamiliesKeys());
- flushRegion(wal, hri2.getEncodedNameAsBytes(), t2.getFamiliesKeys());
+ flushRegion(wal, hri1.getEncodedNameAsBytes(), t1.getColumnFamilyNames());
+ flushRegion(wal, hri2.getEncodedNameAsBytes(), t2.getColumnFamilyNames());
wal.rollWriter(true);
assertEquals(0, wal.getNumRolledLogFiles());
// Add an edit to region1, and roll the wal.
addEdits(wal, hri1, t1, 2, mvcc, scopes1);
// tests partial flush: roll on a partial flush, and ensure that wal is not archived.
- wal.startCacheFlush(hri1.getEncodedNameAsBytes(), t1.getFamiliesKeys());
+ wal.startCacheFlush(hri1.getEncodedNameAsBytes(), t1.getColumnFamilyNames());
wal.rollWriter();
wal.completeCacheFlush(hri1.getEncodedNameAsBytes());
assertEquals(1, wal.getNumRolledLogFiles());
@@ -364,15 +360,15 @@ public abstract class AbstractTestFSWAL {
final TableName tableName = TableName.valueOf(testName);
final RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
final byte[] rowName = tableName.getName();
- final HTableDescriptor htd = new HTableDescriptor(tableName);
- htd.addFamily(new HColumnDescriptor("f"));
+ final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, TEST_UTIL.getDefaultRootDirPath(),
TEST_UTIL.getConfiguration(), htd);
HBaseTestingUtility.closeRegionAndWAL(r);
final int countPerFamily = 10;
final AtomicBoolean goslow = new AtomicBoolean(false);
NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htd.getFamiliesKeys()) {
+ for (byte[] fam : htd.getColumnFamilyNames()) {
scopes.put(fam, 0);
}
// subclass and doctor a method.
@@ -392,15 +388,15 @@ public abstract class AbstractTestFSWAL {
EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
try {
List<Put> puts = null;
- for (HColumnDescriptor hcd : htd.getFamilies()) {
+ for (byte[] fam : htd.getColumnFamilyNames()) {
puts =
- TestWALReplay.addRegionEdits(rowName, hcd.getName(), countPerFamily, ee, region, "x");
+ TestWALReplay.addRegionEdits(rowName, fam, countPerFamily, ee, region, "x");
}
// Now assert edits made it in.
final Get g = new Get(rowName);
Result result = region.get(g);
- assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
+ assertEquals(countPerFamily * htd.getColumnFamilyNames().size(), result.size());
// Construct a WALEdit and add it a few times to the WAL.
WALEdit edits = new WALEdit();
@@ -445,4 +441,36 @@ public abstract class AbstractTestFSWAL {
wal.close();
}
}
+
+ @Test
+ public void testWriteEntryCanBeNull() throws IOException {
+ String testName = currentTest.getMethodName();
+ AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName,
+ CONF, null, true, null, null);
+ wal.close();
+ TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table"))
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
+ RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
+ MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
+ NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ for (byte[] fam : td.getColumnFamilyNames()) {
+ scopes.put(fam, 0);
+ }
+ long timestamp = System.currentTimeMillis();
+ byte[] row = Bytes.toBytes("row");
+ WALEdit cols = new WALEdit();
+ cols.add(new KeyValue(row, row, row, timestamp, row));
+ WALKeyImpl key =
+ new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID,
+ timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
+ try {
+ wal.append(ri, key, cols, true);
+ fail("Should fail since the wal has already been closed");
+ } catch (IOException e) {
+ // expected
+ assertThat(e.getMessage(), containsString("log is closed"));
+ // the WriteEntry should be null since we fail before setting it.
+ assertNull(key.getWriteEntry());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/32f235ab/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index 9c789ba..93ea2b8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -32,12 +32,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.ChunkCreator;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.MemStoreLABImpl;
@@ -97,7 +99,7 @@ public class TestFSHLog extends AbstractTestFSWAL {
SecurityException, IllegalArgumentException, IllegalAccessException {
final String name = this.name.getMethodName();
FSHLog log = new FSHLog(FS, FSUtils.getRootDir(CONF), name, HConstants.HREGION_OLDLOGDIR_NAME,
- CONF, null, true, null, null);
+ CONF, null, true, null, null);
try {
Field ringBufferEventHandlerField = FSHLog.class.getDeclaredField("ringBufferEventHandler");
ringBufferEventHandlerField.setAccessible(true);
@@ -107,14 +109,14 @@ public class TestFSHLog extends AbstractTestFSWAL {
FSHLog.RingBufferEventHandler.class.getDeclaredField("syncRunnerIndex");
syncRunnerIndexField.setAccessible(true);
syncRunnerIndexField.set(ringBufferEventHandler, Integer.MAX_VALUE - 1);
- HTableDescriptor htd =
- new HTableDescriptor(TableName.valueOf(this.name.getMethodName())).addFamily(new HColumnDescriptor("row"));
+ TableDescriptor htd =
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName()))
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htd.getFamiliesKeys()) {
+ for (byte[] fam : htd.getColumnFamilyNames()) {
scopes.put(fam, 0);
}
- HRegionInfo hri =
- new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+ RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
for (int i = 0; i < 10; i++) {
addEdits(log, hri, htd, 1, mvcc, scopes);
@@ -127,7 +129,7 @@ public class TestFSHLog extends AbstractTestFSWAL {
/**
* Test case for https://issues.apache.org/jira/browse/HBASE-16721
*/
- @Test (timeout = 30000)
+ @Test
public void testUnflushedSeqIdTracking() throws IOException, InterruptedException {
final String name = this.name.getMethodName();
final byte[] b = Bytes.toBytes("b");
@@ -156,10 +158,10 @@ public class TestFSHLog extends AbstractTestFSWAL {
});
// open a new region which uses this WAL
- HTableDescriptor htd =
- new HTableDescriptor(TableName.valueOf(this.name.getMethodName())).addFamily(new HColumnDescriptor(b));
- HRegionInfo hri =
- new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+ TableDescriptor htd =
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName()))
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of(b)).build();
+ RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
final HRegion region = TEST_UTIL.createLocalHRegion(hri, htd, log);
ExecutorService exec = Executors.newFixedThreadPool(2);