You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by md...@apache.org on 2017/11/08 18:15:02 UTC
hbase git commit: HBASE-19195 error-prone fixes for client, mr,
and server
Repository: hbase
Updated Branches:
refs/heads/branch-2 d20ab8859 -> 0d4f33ca2
HBASE-19195 error-prone fixes for client, mr, and server
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d4f33ca
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d4f33ca
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d4f33ca
Branch: refs/heads/branch-2
Commit: 0d4f33ca27079ebea98e5d25cc3cc99511ca7792
Parents: d20ab88
Author: Mike Drob <md...@apache.org>
Authored: Mon Nov 6 21:16:48 2017 -0600
Committer: Mike Drob <md...@apache.org>
Committed: Wed Nov 8 12:04:46 2017 -0600
----------------------------------------------------------------------
.../hadoop/hbase/filter/FilterListWithAND.java | 96 ++++++++++----------
.../hbase/mapreduce/TableInputFormatBase.java | 5 +-
.../hadoop/hbase/PerformanceEvaluation.java | 10 +-
.../hadoop/hbase/mapred/TestSplitTable.java | 8 +-
.../hadoop/hbase/mapreduce/TestImportTsv.java | 3 +-
.../TsvImporterCustomTestMapperForOprAttr.java | 9 +-
.../hbase/client/TestFromClientSide3.java | 35 +++----
.../hbase/regionserver/TestMajorCompaction.java | 1 +
8 files changed, 92 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
index edb5d3e..9d4e619 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterListWithAND.java
@@ -90,57 +90,59 @@ public class FilterListWithAND extends FilterListBase {
* code of current sub-filter.
*/
private ReturnCode mergeReturnCode(ReturnCode rc, ReturnCode localRC) {
- if (rc == ReturnCode.SEEK_NEXT_USING_HINT || localRC == ReturnCode.SEEK_NEXT_USING_HINT) {
+ if (rc == ReturnCode.SEEK_NEXT_USING_HINT) {
return ReturnCode.SEEK_NEXT_USING_HINT;
}
switch (localRC) {
- case INCLUDE:
- return rc;
- case INCLUDE_AND_NEXT_COL:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL)) {
- return ReturnCode.INCLUDE_AND_NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL)) {
- return ReturnCode.NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case INCLUDE_AND_SEEK_NEXT_ROW:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
- ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
- return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
- }
- if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ case SEEK_NEXT_USING_HINT:
+ return ReturnCode.SEEK_NEXT_USING_HINT;
+ case INCLUDE:
+ return rc;
+ case INCLUDE_AND_NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL)) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case INCLUDE_AND_SEEK_NEXT_ROW:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL,
+ ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW)) {
+ return ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ }
+ if (isInReturnCodes(rc, ReturnCode.SKIP, ReturnCode.NEXT_COL, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case SKIP:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP)) {
+ return ReturnCode.SKIP;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case NEXT_COL:
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
+ ReturnCode.NEXT_COL)) {
+ return ReturnCode.NEXT_COL;
+ }
+ if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
+ return ReturnCode.NEXT_ROW;
+ }
+ break;
+ case NEXT_ROW:
return ReturnCode.NEXT_ROW;
- }
- break;
- case SKIP:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.SKIP)) {
- return ReturnCode.SKIP;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.NEXT_COL)) {
- return ReturnCode.NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case NEXT_COL:
- if (isInReturnCodes(rc, ReturnCode.INCLUDE, ReturnCode.INCLUDE_AND_NEXT_COL, ReturnCode.SKIP,
- ReturnCode.NEXT_COL)) {
- return ReturnCode.NEXT_COL;
- }
- if (isInReturnCodes(rc, ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, ReturnCode.NEXT_ROW)) {
- return ReturnCode.NEXT_ROW;
- }
- break;
- case NEXT_ROW:
- return ReturnCode.NEXT_ROW;
}
throw new IllegalStateException(
"Received code is not valid. rc: " + rc + ", localRC: " + localRC);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
index e7a65e8..fa2e6a2 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
@@ -268,8 +268,9 @@ public abstract class TableInputFormatBase
}
//The default value of "hbase.mapreduce.input.autobalance" is false.
- if (context.getConfiguration().getBoolean(MAPREDUCE_INPUT_AUTOBALANCE, false) != false) {
- long maxAveRegionSize = context.getConfiguration().getInt(MAX_AVERAGE_REGION_SIZE, 8*1073741824);
+ if (context.getConfiguration().getBoolean(MAPREDUCE_INPUT_AUTOBALANCE, false)) {
+ long maxAveRegionSize = context.getConfiguration()
+ .getLong(MAX_AVERAGE_REGION_SIZE, 8L*1073741824); //8GB
return calculateAutoBalancedSplits(splits, maxAveRegionSize);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 2bf94f4..bc36cde 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -1068,9 +1068,13 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
int getValueLength(final Random r) {
- if (this.opts.isValueRandom()) return Math.abs(r.nextInt() % opts.valueSize);
- else if (this.opts.isValueZipf()) return Math.abs(this.zipf.nextInt());
- else return opts.valueSize;
+ if (this.opts.isValueRandom()) {
+ return r.nextInt(opts.valueSize);
+ } else if (this.opts.isValueZipf()) {
+ return Math.abs(this.zipf.nextInt());
+ } else {
+ return opts.valueSize;
+ }
}
void updateValueSize(final Result [] rs) throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapred/TestSplitTable.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapred/TestSplitTable.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapred/TestSplitTable.java
index 2655ac2..2897e7b 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapred/TestSplitTable.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapred/TestSplitTable.java
@@ -38,7 +38,7 @@ public class TestSplitTable {
public TestName name = new TestName();
@Test
- @SuppressWarnings("deprecation")
+ @SuppressWarnings({"deprecation", "SelfComparison"})
public void testSplitTableCompareTo() {
TableSplit aTableSplit = new TableSplit(Bytes.toBytes("tableA"),
Bytes.toBytes("aaa"), Bytes.toBytes("ddd"), "locationA");
@@ -49,9 +49,9 @@ public class TestSplitTable {
TableSplit cTableSplit = new TableSplit(Bytes.toBytes("tableA"),
Bytes.toBytes("lll"), Bytes.toBytes("zzz"), "locationA");
- assertTrue(aTableSplit.compareTo(aTableSplit) == 0);
- assertTrue(bTableSplit.compareTo(bTableSplit) == 0);
- assertTrue(cTableSplit.compareTo(cTableSplit) == 0);
+ assertEquals(0, aTableSplit.compareTo(aTableSplit));
+ assertEquals(0, bTableSplit.compareTo(bTableSplit));
+ assertEquals(0, cTableSplit.compareTo(cTableSplit));
assertTrue(aTableSplit.compareTo(bTableSplit) < 0);
assertTrue(bTableSplit.compareTo(aTableSplit) > 0);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
index 6ad7694..f6fcfa3 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -430,7 +431,7 @@ public class TestImportTsv implements Configurable {
// run the import
Tool tool = new ImportTsv();
- LOG.debug("Running ImportTsv with arguments: " + argsArray);
+ LOG.debug("Running ImportTsv with arguments: " + Arrays.toString(argsArray));
assertEquals(0, ToolRunner.run(conf, tool, argsArray));
// Perform basic validation. If the input args did not include
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java
index a9da98b..850d4ab 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.mapreduce;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
@@ -43,10 +44,10 @@ public class TsvImporterCustomTestMapperForOprAttr extends TsvImporterMapper {
for (String attr : attributes) {
String[] split = attr.split(ImportTsv.DEFAULT_ATTRIBUTES_SEPERATOR);
if (split == null || split.length <= 1) {
- throw new BadTsvLineException("Invalid attributes seperator specified" + attributes);
+ throw new BadTsvLineException(msg(attributes));
} else {
if (split[0].length() <= 0 || split[1].length() <= 0) {
- throw new BadTsvLineException("Invalid attributes seperator specified" + attributes);
+ throw new BadTsvLineException(msg(attributes));
}
put.setAttribute(split[0], Bytes.toBytes(split[1]));
}
@@ -54,4 +55,8 @@ public class TsvImporterCustomTestMapperForOprAttr extends TsvImporterMapper {
}
put.add(kv);
}
+
+ private String msg(Object[] attributes) {
+ return "Invalid attributes separator specified: " + Arrays.toString(attributes);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
index 0a3b75a..5f7622a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -83,13 +83,13 @@ public class TestFromClientSide3 {
private static byte[] FAMILY = Bytes.toBytes("testFamily");
private static Random random = new Random();
private static int SLAVES = 3;
- private static byte [] ROW = Bytes.toBytes("testRow");
+ private static final byte[] ROW = Bytes.toBytes("testRow");
private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
- private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
- private static byte [] VALUE = Bytes.toBytes("testValue");
- private final static byte[] COL_QUAL = Bytes.toBytes("f1");
- private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
- private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
+ private static final byte[] QUALIFIER = Bytes.toBytes("testQualifier");
+ private static final byte[] VALUE = Bytes.toBytes("testValue");
+ private static final byte[] COL_QUAL = Bytes.toBytes("f1");
+ private static final byte[] VAL_BYTES = Bytes.toBytes("v1");
+ private static final byte[] ROW_BYTES = Bytes.toBytes("r1");
@Rule
public TestName name = new TestName();
@@ -361,7 +361,7 @@ public class TestFromClientSide3 {
break;
}
} catch (Exception e) {
- LOG.debug("Waiting for region to come online: " + regionName);
+ LOG.debug("Waiting for region to come online: " + Bytes.toString(regionName));
}
Thread.sleep(40);
}
@@ -478,6 +478,7 @@ public class TestFromClientSide3 {
assertEquals(exist, true);
}
+ @Test
public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
Table table = TEST_UTIL.createTable(TableName.valueOf(
name.getMethodName()), new byte[][] { FAMILY });
@@ -488,13 +489,11 @@ public class TestFromClientSide3 {
List<Get> gets = new ArrayList<>();
gets.add(new Get(ROW));
- gets.add(null);
gets.add(new Get(ANOTHERROW));
- boolean[] results = table.existsAll(gets);
- assertEquals(results[0], true);
- assertEquals(results[1], false);
- assertEquals(results[2], false);
+ boolean[] results = table.exists(gets);
+ assertTrue(results[0]);
+ assertFalse(results[1]);
}
@Test
@@ -749,7 +748,7 @@ public class TestFromClientSide3 {
try (Table table = con.getTable(tableName)) {
table.append(append);
fail("The APPEND should fail because the target lock is blocked by previous put");
- } catch (Throwable ex) {
+ } catch (Exception ex) {
}
});
appendService.shutdown();
@@ -802,6 +801,7 @@ public class TestFromClientSide3 {
});
ExecutorService cpService = Executors.newSingleThreadExecutor();
cpService.execute(() -> {
+ boolean threw;
Put put1 = new Put(row);
Put put2 = new Put(rowLocked);
put1.addColumn(FAMILY, QUALIFIER, value1);
@@ -823,10 +823,13 @@ public class TestFromClientSide3 {
exe.mutateRows(controller, request, rpcCallback);
return rpcCallback.get();
});
- fail("This cp should fail because the target lock is blocked by previous put");
+ threw = false;
} catch (Throwable ex) {
- // TODO!!!! Is this right? It catches everything including the above fail
- // if it happens (which it seems too....)
+ threw = true;
+ }
+ if (!threw) {
+ // Can't call fail() earlier because the catch would eat it.
+ fail("This cp should fail because the target lock is blocked by previous put");
}
});
cpService.shutdown();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d4f33ca/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index 2a556d7..40f5135 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -450,6 +450,7 @@ public class TestMajorCompaction {
* basically works.
* @throws IOException
*/
+ @Test
public void testMajorCompactingToNoOutputWithReverseScan() throws IOException {
createStoreFile(r);
for (int i = 0; i < compactionThreshold; i++) {