You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/01/12 07:11:45 UTC
[01/12] hbase git commit: HBASE-15070
DistributedHBaseCluster#restoreRegionServers() starts new RS process on
master server (Samir Ahmic)
Repository: hbase
Updated Branches:
refs/heads/hbase-12439 5266b0770 -> da932ee38
HBASE-15070 DistributedHBaseCluster#restoreRegionServers() starts new RS process on master server (Samir Ahmic)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3d367793
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3d367793
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3d367793
Branch: refs/heads/hbase-12439
Commit: 3d3677932a4ec98c12121c879ac5e2ea71925ea5
Parents: 5266b07
Author: tedyu <yu...@gmail.com>
Authored: Thu Jan 7 09:14:00 2016 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jan 7 09:14:00 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/DistributedHBaseCluster.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3d367793/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index 62f5c66..6f3baa0 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -433,6 +433,8 @@ public class DistributedHBaseCluster extends HBaseCluster {
Set<ServerName> toKill = new TreeSet<ServerName>(new ServerNameIgnoreStartCodeComparator());
toStart.addAll(initial.getServers());
toKill.addAll(current.getServers());
+
+ ServerName master = initial.getMaster();
for (ServerName server : current.getServers()) {
toStart.remove(server);
@@ -447,7 +449,8 @@ public class DistributedHBaseCluster extends HBaseCluster {
try {
if (!clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER,
sn.getHostname(),
- sn.getPort())) {
+ sn.getPort())
+ && master.getPort() != sn.getPort()) {
LOG.info("Restoring cluster - starting initial region server: " + sn.getHostAndPort());
startRegionServer(sn.getHostname(), sn.getPort());
}
@@ -460,7 +463,8 @@ public class DistributedHBaseCluster extends HBaseCluster {
try {
if (clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER,
sn.getHostname(),
- sn.getPort())) {
+ sn.getPort())
+ && master.getPort() != sn.getPort()){
LOG.info("Restoring cluster - stopping initial region server: " + sn.getHostAndPort());
stopRegionServer(sn);
}
[07/12] hbase git commit: HBASE-14975 Don't color the total RIT line
yellow if it's zero (Pallavi Adusumilli)
Posted by sy...@apache.org.
HBASE-14975 Don't color the total RIT line yellow if it's zero (Pallavi Adusumilli)
Signed-off-by: Elliott Clark <ec...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52280b64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52280b64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52280b64
Branch: refs/heads/hbase-12439
Commit: 52280b64e643584b9b73924a4216442def0501a4
Parents: 6e781a1
Author: Elliott Clark <ec...@apache.org>
Authored: Fri Jan 8 14:29:38 2016 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Fri Jan 8 14:29:44 2016 -0800
----------------------------------------------------------------------
.../hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/52280b64/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
index 30a95ce..2bf034a 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
@@ -91,7 +91,12 @@ if (toRemove > 0) {
entry.getValue(), conf) %></td>
<td><% (currentTime - entry.getValue().getStamp()) %> </td></tr>
</%for>
- <tr BGCOLOR="#D7DF01"> <td>Total number of Regions in Transition for more than <% ritThreshold %> milliseconds</td><td> <% numOfRITOverThreshold %></td><td></td>
+ <%if numOfRITOverThreshold > 0 %>
+ <tr BGCOLOR="#D7DF01" >
+ <%else>
+ <tr>
+ </%if>
+ <td>Total number of Regions in Transition for more than <% ritThreshold %> milliseconds</td><td> <% numOfRITOverThreshold %></td><td></td>
</tr>
<tr> <td> Total number of Regions in Transition</td><td><% totalRITs %> </td><td></td>
</table>
[08/12] hbase git commit: HBASE-15065 Addendum fixes checkstyle
warnings
Posted by sy...@apache.org.
HBASE-15065 Addendum fixes checkstyle warnings
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4a266442
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4a266442
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4a266442
Branch: refs/heads/hbase-12439
Commit: 4a26644245bad6a661442f0eede6850e12d07cc9
Parents: 52280b6
Author: tedyu <yu...@gmail.com>
Authored: Fri Jan 8 14:49:30 2016 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Fri Jan 8 14:49:30 2016 -0800
----------------------------------------------------------------------
.../hbase/master/normalizer/SimpleRegionNormalizer.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a266442/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 7195f7d..7ea6cc7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.normalizer.NormalizationPlan;
import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType;
-import org.apache.hadoop.hbase.util.Triple;
import java.util.ArrayList;
import java.util.Collections;
@@ -87,8 +86,12 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
new Comparator<NormalizationPlan>() {
@Override
public int compare(NormalizationPlan plan, NormalizationPlan plan2) {
- if (plan instanceof SplitNormalizationPlan) return -1;
- if (plan2 instanceof SplitNormalizationPlan) return 1;
+ if (plan instanceof SplitNormalizationPlan) {
+ return -1;
+ }
+ if (plan2 instanceof SplitNormalizationPlan) {
+ return 1;
+ }
return 0;
}
};
[11/12] hbase git commit: HBASE-15052 Use EnvironmentEdgeManager in
ReplicationSource
Posted by sy...@apache.org.
HBASE-15052 Use EnvironmentEdgeManager in ReplicationSource
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ec47a811
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ec47a811
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ec47a811
Branch: refs/heads/hbase-12439
Commit: ec47a811a2e7c0bb3f844d780049bb8a67cc4c35
Parents: 894ce84
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Jan 11 09:37:08 2016 -0800
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon Jan 11 09:37:08 2016 -0800
----------------------------------------------------------------------
.../hadoop/hbase/replication/regionserver/ReplicationSource.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec47a811/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 868ddee..b4975bf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hbase.replication.ReplicationQueues;
import org.apache.hadoop.hbase.replication.SystemTableWALEntryFilter;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.DefaultWALProvider;
@@ -640,7 +641,7 @@ public class ReplicationSource extends Thread
sleepMultiplier = 1;
// if there was nothing to ship and it's not an error
// set "ageOfLastShippedOp" to <now> to indicate that we're current
- metrics.setAgeOfLastShippedOp(System.currentTimeMillis(), walGroupId);
+ metrics.setAgeOfLastShippedOp(EnvironmentEdgeManager.currentTime(), walGroupId);
}
if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
sleepMultiplier++;
[03/12] hbase git commit: HBASE-15079
TestMultiParallel.validateLoadedData AssertionError: null
Posted by sy...@apache.org.
HBASE-15079 TestMultiParallel.validateLoadedData AssertionError: null
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f3ee6df0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f3ee6df0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f3ee6df0
Branch: refs/heads/hbase-12439
Commit: f3ee6df0f2d0955c2b334a9131eb3994c00af0c4
Parents: 5bde960
Author: stack <st...@apache.org>
Authored: Thu Jan 7 15:35:59 2016 -0800
Committer: stack <st...@apache.org>
Committed: Thu Jan 7 15:36:49 2016 -0800
----------------------------------------------------------------------
.../hadoop/hbase/client/TestMultiParallel.java | 39 ++++++++++++++++++--
1 file changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f3ee6df0/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
index b2c5c11..af3a54e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
@@ -683,13 +683,44 @@ public class TestMultiParallel {
private void validateLoadedData(Table table) throws IOException {
// get the data back and validate that it is correct
LOG.info("Validating data on " + table);
+ List<Get> gets = new ArrayList<Get>();
for (byte[] k : KEYS) {
Get get = new Get(k);
get.addColumn(BYTES_FAMILY, QUALIFIER);
- Result r = table.get(get);
- Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
- Assert.assertEquals(0, Bytes.compareTo(VALUE, r
- .getValue(BYTES_FAMILY, QUALIFIER)));
+ gets.add(get);
+ }
+ int retryNum = 10;
+ Result[] results = null;
+ do {
+ results = table.get(gets);
+ boolean finished = true;
+ for (Result result : results) {
+ if (result.isEmpty()) {
+ finished = false;
+ break;
+ }
+ }
+ if (finished) {
+ break;
+ }
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ }
+ retryNum--;
+ } while (retryNum > 0);
+
+ if (retryNum == 0) {
+ fail("Timeout for validate data");
+ } else {
+ if (results != null) {
+ for (Result r : results) {
+ Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
+ Assert.assertEquals(0, Bytes.compareTo(VALUE, r
+ .getValue(BYTES_FAMILY, QUALIFIER)));
+ }
+ LOG.info("Validating data on " + table + " successfully!");
+ }
}
}
[06/12] hbase git commit: HBASE-15065 SimpleRegionNormalizer should
return multiple normalization plans in one run
Posted by sy...@apache.org.
HBASE-15065 SimpleRegionNormalizer should return multiple normalization plans in one run
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6e781a1f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6e781a1f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6e781a1f
Branch: refs/heads/hbase-12439
Commit: 6e781a1f342de069868a8a1f95f6a403a6d4c499
Parents: 164aeb5
Author: tedyu <yu...@gmail.com>
Authored: Fri Jan 8 11:47:34 2016 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Fri Jan 8 11:47:34 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 16 ++--
.../master/normalizer/RegionNormalizer.java | 4 +-
.../normalizer/RegionNormalizerChore.java | 2 +-
.../normalizer/SimpleRegionNormalizer.java | 95 +++++++++-----------
.../normalizer/TestSimpleRegionNormalizer.java | 27 +++---
.../TestSimpleRegionNormalizerOnCluster.java | 16 +++-
6 files changed, 82 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e781a1f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 8ff7ab1..2431681 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1341,12 +1341,16 @@ public class HMaster extends HRegionServer implements MasterServices {
continue;
}
}
- NormalizationPlan plan = this.normalizer.computePlanForTable(table, types);
- plan.execute(clusterConnection.getAdmin());
- if (plan.getType() == PlanType.SPLIT) {
- splitPlanCount++;
- } else if (plan.getType() == PlanType.MERGE) {
- mergePlanCount++;
+ List<NormalizationPlan> plans = this.normalizer.computePlanForTable(table, types);
+ if (plans != null) {
+ for (NormalizationPlan plan : plans) {
+ plan.execute(clusterConnection.getAdmin());
+ if (plan.getType() == PlanType.SPLIT) {
+ splitPlanCount++;
+ } else if (plan.getType() == PlanType.MERGE) {
+ mergePlanCount++;
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e781a1f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
index 616098e..c0083e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
@@ -51,9 +51,9 @@ public interface RegionNormalizer {
* Computes next optimal normalization plan.
* @param table table to normalize
* @param types desired types of NormalizationPlan
- * @return Next (perhaps most urgent) normalization action to perform
+ * @return normalization actions to perform. Null if no action to take
*/
- NormalizationPlan computePlanForTable(TableName table, List<PlanType> types)
+ List<NormalizationPlan> computePlanForTable(TableName table, List<PlanType> types)
throws HBaseIOException;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e781a1f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
index 25118c7..c6baa2e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
@@ -38,7 +38,7 @@ public class RegionNormalizerChore extends ScheduledChore {
public RegionNormalizerChore(HMaster master) {
super(master.getServerName() + "-RegionNormalizerChore", master,
- master.getConfiguration().getInt("hbase.normalizer.period", 1800000));
+ master.getConfiguration().getInt("hbase.normalizer.period", 300000));
this.master = master;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e781a1f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index a035647..7195f7d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -72,22 +72,6 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
this.masterServices = masterServices;
}
- /*
- * This comparator compares the region size.
- * The second element in the triple is region size while the 3rd element
- * is the index of the region in the underlying List
- */
- private Comparator<Triple<HRegionInfo, Long, Integer>> regionSizeComparator =
- new Comparator<Triple<HRegionInfo, Long, Integer>>() {
- @Override
- public int compare(Triple<HRegionInfo, Long, Integer> pair,
- Triple<HRegionInfo, Long, Integer> pair2) {
- long sz = pair.getSecond();
- long sz2 = pair2.getSecond();
- return (sz < sz2) ? -1 : ((sz == sz2) ? 0 : 1);
- }
- };
-
@Override
public void planSkipped(HRegionInfo hri, PlanType type) {
skippedCount[type.ordinal()]++;
@@ -98,6 +82,17 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
return skippedCount[type.ordinal()];
}
+ // Comparator that gives higher priority to region Split plan
+ private Comparator<NormalizationPlan> planComparator =
+ new Comparator<NormalizationPlan>() {
+ @Override
+ public int compare(NormalizationPlan plan, NormalizationPlan plan2) {
+ if (plan instanceof SplitNormalizationPlan) return -1;
+ if (plan2 instanceof SplitNormalizationPlan) return 1;
+ return 0;
+ }
+ };
+
/**
* Computes next most "urgent" normalization action on the table.
* Action may be either a split, or a merge, or no action.
@@ -107,13 +102,14 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
* @return normalization plan to execute
*/
@Override
- public NormalizationPlan computePlanForTable(TableName table, List<PlanType> types)
+ public List<NormalizationPlan> computePlanForTable(TableName table, List<PlanType> types)
throws HBaseIOException {
if (table == null || table.isSystemTable()) {
LOG.debug("Normalization of system table " + table + " isn't allowed");
- return EmptyNormalizationPlan.getInstance();
+ return null;
}
+ List<NormalizationPlan> plans = new ArrayList<NormalizationPlan>();
List<HRegionInfo> tableRegions = masterServices.getAssignmentManager().getRegionStates().
getRegionsOfTable(table);
@@ -122,7 +118,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
int nrRegions = tableRegions == null ? 0 : tableRegions.size();
LOG.debug("Table " + table + " has " + nrRegions + " regions, required min number"
+ " of regions for normalizer to run is " + MIN_REGION_COUNT + ", not running normalizer");
- return EmptyNormalizationPlan.getInstance();
+ return null;
}
LOG.debug("Computing normalization plan for table: " + table +
@@ -130,56 +126,49 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
long totalSizeMb = 0;
- ArrayList<Triple<HRegionInfo, Long, Integer>> regionsWithSize =
- new ArrayList<Triple<HRegionInfo, Long, Integer>>(tableRegions.size());
for (int i = 0; i < tableRegions.size(); i++) {
HRegionInfo hri = tableRegions.get(i);
long regionSize = getRegionSize(hri);
- regionsWithSize.add(new Triple<HRegionInfo, Long, Integer>(hri, regionSize, i));
totalSizeMb += regionSize;
}
- Collections.sort(regionsWithSize, regionSizeComparator);
-
- Triple<HRegionInfo, Long, Integer> largestRegion = regionsWithSize.get(tableRegions.size()-1);
double avgRegionSize = totalSizeMb / (double) tableRegions.size();
LOG.debug("Table " + table + ", total aggregated regions size: " + totalSizeMb);
LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
- // now; if the largest region is >2 times large than average, we split it, split
- // is more high priority normalization action than merge.
- if (types.contains(PlanType.SPLIT) && largestRegion.getSecond() > 2 * avgRegionSize) {
- LOG.debug("Table " + table + ", largest region "
- + largestRegion.getFirst().getRegionNameAsString() + " has size "
- + largestRegion.getSecond() + ", more than 2 times than avg size, splitting");
- return new SplitNormalizationPlan(largestRegion.getFirst(), null);
- }
int candidateIdx = 0;
- // look for two successive entries whose indices are adjacent
- while (candidateIdx < tableRegions.size()-1) {
- if (Math.abs(regionsWithSize.get(candidateIdx).getThird() -
- regionsWithSize.get(candidateIdx + 1).getThird()) == 1) {
- break;
+ while (candidateIdx < tableRegions.size()) {
+ HRegionInfo hri = tableRegions.get(candidateIdx);
+ long regionSize = getRegionSize(hri);
+ // if the region is > 2 times larger than average, we split it, split
+ // is more high priority normalization action than merge.
+ if (types.contains(PlanType.SPLIT) && regionSize > 2 * avgRegionSize) {
+ LOG.debug("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size "
+ + regionSize + ", more than twice avg size, splitting");
+ plans.add(new SplitNormalizationPlan(hri, null));
+ } else {
+ if (candidateIdx == tableRegions.size()-1) {
+ break;
+ }
+ HRegionInfo hri2 = tableRegions.get(candidateIdx+1);
+ long regionSize2 = getRegionSize(hri2);
+ if (types.contains(PlanType.MERGE) && regionSize + regionSize2 < avgRegionSize) {
+ LOG.debug("Table " + table + ", small region size: " + regionSize
+ + " plus its neighbor size: " + regionSize2
+ + ", less than the avg size " + avgRegionSize + ", merging them");
+ plans.add(new MergeNormalizationPlan(hri, hri2));
+ candidateIdx++;
+ }
}
candidateIdx++;
}
- if (candidateIdx == tableRegions.size()-1) {
- LOG.debug("No neighboring regions found for table: " + table);
- return EmptyNormalizationPlan.getInstance();
- }
- Triple<HRegionInfo, Long, Integer> candidateRegion = regionsWithSize.get(candidateIdx);
- Triple<HRegionInfo, Long, Integer> candidateRegion2 = regionsWithSize.get(candidateIdx+1);
- if (types.contains(PlanType.MERGE) &&
- candidateRegion.getSecond() + candidateRegion2.getSecond() < avgRegionSize) {
- LOG.debug("Table " + table + ", smallest region size: " + candidateRegion.getSecond()
- + " and its smallest neighbor size: " + candidateRegion2.getSecond()
- + ", less than the avg size, merging them");
- return new MergeNormalizationPlan(candidateRegion.getFirst(),
- candidateRegion2.getFirst());
+ if (plans.isEmpty()) {
+ LOG.debug("No normalization needed, regions look good for table: " + table);
+ return null;
}
- LOG.debug("No normalization needed, regions look good for table: " + table);
- return EmptyNormalizationPlan.getInstance();
+ Collections.sort(plans, planComparator);
+ return plans;
}
private long getRegionSize(HRegionInfo hri) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e781a1f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index 4395aa3..1f66044 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -83,8 +83,8 @@ public class TestSimpleRegionNormalizer {
Map<byte[], Integer> regionSizes = new HashMap<>();
setupMocksForNormalizer(regionSizes, hris);
- NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
- assertTrue(plan instanceof EmptyNormalizationPlan);
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+ assertTrue(plans == null);
}
@Test
@@ -102,8 +102,8 @@ public class TestSimpleRegionNormalizer {
regionSizes.put(hri2.getRegionName(), 15);
setupMocksForNormalizer(regionSizes, hris);
- NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
- assertTrue((plan instanceof EmptyNormalizationPlan));
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+ assertTrue(plans == null);
}
@Test
@@ -129,8 +129,8 @@ public class TestSimpleRegionNormalizer {
regionSizes.put(hri4.getRegionName(), 10);
setupMocksForNormalizer(regionSizes, hris);
- NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
- assertTrue(plan instanceof EmptyNormalizationPlan);
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+ assertTrue(plans == null);
}
@Test
@@ -165,15 +165,16 @@ public class TestSimpleRegionNormalizer {
regionSizes.put(hri5.getRegionName(), 16);
setupMocksForNormalizer(regionSizes, hris);
- NormalizationPlan plan = normalizer.computePlanForTable(testTable,
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable,
mergeDesired ? bothTypes : splitType);
if (mergeDesired) {
+ NormalizationPlan plan = plans.get(0);
assertTrue(plan instanceof MergeNormalizationPlan);
assertEquals(hri2, ((MergeNormalizationPlan) plan).getFirstRegion());
assertEquals(hri3, ((MergeNormalizationPlan) plan).getSecondRegion());
} else {
- assertTrue(plan instanceof EmptyNormalizationPlan);
+ assertTrue(plans == null);
}
}
@@ -209,7 +210,8 @@ public class TestSimpleRegionNormalizer {
regionSizes.put(hri6.getRegionName(), 2700);
setupMocksForNormalizer(regionSizes, hris);
- NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+ NormalizationPlan plan = plans.get(0);
assertTrue(plan instanceof MergeNormalizationPlan);
assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion());
@@ -243,9 +245,9 @@ public class TestSimpleRegionNormalizer {
regionSizes.put(hri5.getRegionName(), 5);
setupMocksForNormalizer(regionSizes, hris);
- NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
- assertTrue(plan instanceof EmptyNormalizationPlan);
+ assertTrue(plans == null);
}
@Test
@@ -271,7 +273,8 @@ public class TestSimpleRegionNormalizer {
regionSizes.put(hri4.getRegionName(), 30);
setupMocksForNormalizer(regionSizes, hris);
- NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
+ List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+ NormalizationPlan plan = plans.get(0);
assertTrue(plan instanceof SplitNormalizationPlan);
assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo());
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e781a1f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
index 99fb268..801e92d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
@@ -155,11 +155,19 @@ public class TestSimpleRegionNormalizerOnCluster {
} while (skippedSplitcnt == 0L);
assert(skippedSplitcnt > 0);
} else {
- while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6) {
- LOG.info("Waiting for normalization split to complete");
- Thread.sleep(100);
+ while (true) {
+ List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
+ int cnt = 0;
+ for (HRegion region : regions) {
+ String regionName = region.getRegionInfo().getRegionNameAsString();
+ if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) {
+ cnt++;
+ }
+ }
+ if (cnt >= 2) {
+ break;
+ }
}
- assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
}
admin.disableTable(TABLENAME);
[12/12] hbase git commit: HBASE-15077 Support OffheapKV write in
compaction with out copying data on heap.
Posted by sy...@apache.org.
HBASE-15077 Support OffheapKV write in compaction with out copying data on heap.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da932ee3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da932ee3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da932ee3
Branch: refs/heads/hbase-12439
Commit: da932ee38d29a13acb3e3ff6653402d4c8141d04
Parents: ec47a81
Author: anoopsjohn <an...@gmail.com>
Authored: Tue Jan 12 10:02:39 2016 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Tue Jan 12 10:02:39 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/OffheapKeyValue.java | 2 +-
.../hadoop/hbase/io/ByteArrayOutputStream.java | 129 +++++++++++++++++++
.../hadoop/hbase/io/ByteBufferOutputStream.java | 3 +-
.../io/ByteBufferSupportDataOutputStream.java | 44 +++++++
.../hbase/io/ByteBufferSupportOutputStream.java | 51 ++++++++
.../io/encoding/BufferedDataBlockEncoder.java | 4 +-
.../hadoop/hbase/util/ByteBufferUtils.java | 26 +---
.../hadoop/hbase/io/hfile/HFileBlock.java | 23 +---
.../io/encoding/TestDataBlockEncoders.java | 11 +-
.../io/hfile/TestHFileDataBlockEncoder.java | 7 +-
10 files changed, 245 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java
index 0af64cd..ced1595 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java
@@ -254,7 +254,7 @@ public class OffheapKeyValue extends ByteBufferedCell implements HeapSize, Clone
length = keyLen + this.getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
}
ByteBufferUtils.putInt(out, length);
- ByteBufferUtils.writeByteBuffer(out, this.buf, this.offset, length);
+ ByteBufferUtils.copyBufferToStream(out, this.buf, this.offset, length);
return length + Bytes.SIZEOF_INT;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteArrayOutputStream.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteArrayOutputStream.java
new file mode 100644
index 0000000..d951595
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteArrayOutputStream.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Our own implementation of ByteArrayOutputStream where all methods are NOT synchronized and
+ * supports writing ByteBuffer directly to it.
+ */
+@InterfaceAudience.Private
+public class ByteArrayOutputStream extends OutputStream implements ByteBufferSupportOutputStream {
+
+ // Borrowed from openJDK:
+ // http://grepcode.com/file/repository.grepcode.com/java/root/jdk/openjdk/8-b132/java/util/ArrayList.java#221
+ private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
+
+ private byte[] buf;
+ private int pos = 0;
+
+ public ByteArrayOutputStream() {
+ this(32);
+ }
+
+ public ByteArrayOutputStream(int capacity) {
+ this.buf = new byte[capacity];
+ }
+
+ @Override
+ public void write(ByteBuffer b, int off, int len) throws IOException {
+ checkSizeAndGrow(len);
+ ByteBufferUtils.copyFromBufferToArray(this.buf, b, off, this.pos, len);
+ this.pos += len;
+ }
+
+ @Override
+ public void writeInt(int i) throws IOException {
+ checkSizeAndGrow(Bytes.SIZEOF_INT);
+ Bytes.putInt(this.buf, this.pos, i);
+ this.pos += Bytes.SIZEOF_INT;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ checkSizeAndGrow(Bytes.SIZEOF_BYTE);
+ buf[this.pos] = (byte) b;
+ this.pos++;
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ checkSizeAndGrow(len);
+ System.arraycopy(b, off, this.buf, this.pos, len);
+ this.pos += len;
+ }
+
+ private void checkSizeAndGrow(int extra) {
+ long capacityNeeded = this.pos + (long) extra;
+ if (capacityNeeded > this.buf.length) {
+ // guarantee it's possible to fit
+ if (capacityNeeded > MAX_ARRAY_SIZE) {
+ throw new BufferOverflowException();
+ }
+ // double until hit the cap
+ long nextCapacity = Math.min(this.buf.length << 1, MAX_ARRAY_SIZE);
+ // but make sure there is enough if twice the existing capacity is still too small
+ nextCapacity = Math.max(nextCapacity, capacityNeeded);
+ if (nextCapacity > MAX_ARRAY_SIZE) {
+ throw new BufferOverflowException();
+ }
+ byte[] newBuf = new byte[(int) nextCapacity];
+ System.arraycopy(buf, 0, newBuf, 0, buf.length);
+ buf = newBuf;
+ }
+ }
+
+ /**
+ * Resets the <code>pos</code> field of this byte array output stream to zero. The output stream
+ * can be used again.
+ */
+ public void reset() {
+ this.pos = 0;
+ }
+
+ /**
+ * Copies the content of this Stream into a new byte array.
+ * @return the contents of this output stream, as new byte array.
+ */
+ public byte toByteArray()[] {
+ return Arrays.copyOf(buf, pos);
+ }
+
+ /**
+ * @return the underlying array where the data gets accumulated
+ */
+ public byte[] getBuffer() {
+ return this.buf;
+ }
+
+ /**
+ * @return The current size of the buffer.
+ */
+ public int size() {
+ return this.pos;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferOutputStream.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferOutputStream.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferOutputStream.java
index d91513e..d4bda18 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferOutputStream.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferOutputStream.java
@@ -37,7 +37,8 @@ import org.apache.hadoop.hbase.util.Bytes;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public class ByteBufferOutputStream extends OutputStream {
+public class ByteBufferOutputStream extends OutputStream
+ implements ByteBufferSupportOutputStream {
// Borrowed from openJDK:
// http://grepcode.com/file/repository.grepcode.com/java/root/jdk/openjdk/8-b132/java/util/ArrayList.java#221
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportDataOutputStream.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportDataOutputStream.java
new file mode 100644
index 0000000..3a52e63
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportDataOutputStream.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+
+/**
+ * Our extension of DataOutputStream which implements ByteBufferSupportOutputStream
+ */
+@InterfaceAudience.Private
+public class ByteBufferSupportDataOutputStream extends DataOutputStream
+ implements ByteBufferSupportOutputStream {
+
+ public ByteBufferSupportDataOutputStream(OutputStream out) {
+ super(out);
+ }
+
+ @Override
+ public void write(ByteBuffer b, int off, int len) throws IOException {
+ ByteBufferUtils.copyBufferToStream(out, b, off, len);
+ written += len;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportOutputStream.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportOutputStream.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportOutputStream.java
new file mode 100644
index 0000000..ccb5c81
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferSupportOutputStream.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Interface adds support for writing {@link ByteBuffer} into OutputStream.
+ */
+@InterfaceAudience.Private
+public interface ByteBufferSupportOutputStream {
+
+ /**
+ * Writes <code>len</code> bytes from the specified ByteBuffer starting at offset <code>off</code>
+ * to this output stream.
+ *
+ * @param b the data.
+ * @param off the start offset in the data.
+ * @param len the number of bytes to write.
+ * @exception IOException
+ * if an I/O error occurs. In particular, an <code>IOException</code> is thrown if
+ * the output stream is closed.
+ */
+ void write(ByteBuffer b, int off, int len) throws IOException;
+
+ /**
+ * Writes an <code>int</code> to the underlying output stream as four
+ * bytes, high byte first.
+ * @param i the <code>int</code> to write
+ * @throws IOException if an I/O error occurs.
+ */
+ void writeInt(int i) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index 112f258..33e38c7 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -673,14 +673,14 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
// Write key
out.write(keyBuffer.array());
// Write value
- ByteBufferUtils.writeByteBuffer(out, this.valueBuffer, this.valueOffset, this.valueLength);
+ ByteBufferUtils.copyBufferToStream(out, this.valueBuffer, this.valueOffset, this.valueLength);
if (withTags) {
// 2 bytes tags length followed by tags bytes
// tags length is serialized with 2 bytes only(short way) even if the type is int.
// As this is non -ve numbers, we save the sign bit. See HBASE-11437
out.write((byte) (0xff & (this.tagsLength >> 8)));
out.write((byte) (0xff & this.tagsLength));
- ByteBufferUtils.writeByteBuffer(out, this.tagsBuffer, this.tagsOffset, this.tagsLength);
+ ByteBufferUtils.copyBufferToStream(out, this.tagsBuffer, this.tagsOffset, this.tagsLength);
}
return lenToWrite + Bytes.SIZEOF_INT;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
index 6e3fcaa..62173c2 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
@@ -27,7 +27,7 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.io.ByteBufferOutputStream;
+import org.apache.hadoop.hbase.io.ByteBufferSupportOutputStream;
import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableUtils;
@@ -141,8 +141,8 @@ public final class ByteBufferUtils {
// We have writeInt in ByteBufferOutputStream so that it can directly write
// int to underlying
// ByteBuffer in one step.
- if (out instanceof ByteBufferOutputStream) {
- ((ByteBufferOutputStream) out).writeInt(value);
+ if (out instanceof ByteBufferSupportOutputStream) {
+ ((ByteBufferSupportOutputStream) out).writeInt(value);
} else {
StreamUtils.writeInt(out, value);
}
@@ -179,9 +179,10 @@ public final class ByteBufferUtils {
*/
public static void copyBufferToStream(OutputStream out, ByteBuffer in,
int offset, int length) throws IOException {
- if (in.hasArray()) {
- out.write(in.array(), in.arrayOffset() + offset,
- length);
+ if (out instanceof ByteBufferSupportOutputStream) {
+ ((ByteBufferSupportOutputStream) out).write(in, offset, length);
+ } else if (in.hasArray()) {
+ out.write(in.array(), in.arrayOffset() + offset, length);
} else {
for (int i = 0; i < length; ++i) {
out.write(toByte(in, offset + i));
@@ -904,19 +905,6 @@ public final class ByteBufferUtils {
}
}
- public static void writeByteBuffer(OutputStream out, ByteBuffer b, int offset, int length)
- throws IOException {
- // We have write which takes ByteBuffer in ByteBufferOutputStream so that it
- // can directly write
- // bytes from the src ByteBuffer to the destination ByteBuffer. This avoid
- // need for temp array
- // creation and copy
- if (out instanceof ByteBufferOutputStream) {
- ((ByteBufferOutputStream) out).write(b, offset, length);
- } else {
- ByteBufferUtils.copyBufferToStream(out, b, offset, length);
- }
- }
// For testing purpose
public static String toStringBinary(final ByteBuffer b, int off, int len) {
StringBuilder result = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index a68d0a6..6916ba0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hbase.io.hfile;
-import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
@@ -35,7 +34,9 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
+import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.hadoop.hbase.io.ByteBuffInputStream;
+import org.apache.hadoop.hbase.io.ByteBufferSupportDataOutputStream;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
@@ -915,7 +916,7 @@ public class HFileBlock implements Cacheable {
state = State.WRITING;
// We will compress it later in finishBlock()
- userDataStream = new DataOutputStream(baosInMemory);
+ userDataStream = new ByteBufferSupportDataOutputStream(baosInMemory);
if (newBlockType == BlockType.DATA) {
this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, userDataStream);
}
@@ -969,11 +970,8 @@ public class HFileBlock implements Cacheable {
*/
private void finishBlock() throws IOException {
if (blockType == BlockType.DATA) {
- BufferGrabbingByteArrayOutputStream baosInMemoryCopy =
- new BufferGrabbingByteArrayOutputStream();
- baosInMemory.writeTo(baosInMemoryCopy);
this.dataBlockEncoder.endBlockEncoding(dataBlockEncodingCtx, userDataStream,
- baosInMemoryCopy.buf, blockType);
+ baosInMemory.getBuffer(), blockType);
blockType = dataBlockEncodingCtx.getBlockType();
}
userDataStream.flush();
@@ -1011,19 +1009,6 @@ public class HFileBlock implements Cacheable {
onDiskChecksum, 0, fileContext.getChecksumType(), fileContext.getBytesPerChecksum());
}
- public static class BufferGrabbingByteArrayOutputStream extends ByteArrayOutputStream {
- private byte[] buf;
-
- @Override
- public void write(byte[] b, int off, int len) {
- this.buf = b;
- }
-
- public byte[] getBuffer() {
- return this.buf;
- }
- }
-
/**
* Put the header into the given byte array at the given offset.
* @param onDiskSize size of the block on disk header + data + checksum
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
index 00969b2..1ef918c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -45,8 +44,8 @@ import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeSeeker;
+import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.hadoop.hbase.io.compress.Compression;
-import org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer.BufferGrabbingByteArrayOutputStream;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
@@ -256,9 +255,7 @@ public class TestDataBlockEncoders {
for (KeyValue kv : kvs) {
encoder.encode(kv, encodingContext, dos);
}
- BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
- baos.writeTo(stream);
- encoder.endBlockEncoding(encodingContext, dos, stream.getBuffer());
+ encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer());
byte[] encodedData = new byte[baos.size() - ENCODED_DATA_OFFSET];
System.arraycopy(baos.toByteArray(), ENCODED_DATA_OFFSET, encodedData, 0, encodedData.length);
if (useOffheapData) {
@@ -398,9 +395,7 @@ public class TestDataBlockEncoders {
for (KeyValue kv : kvList) {
encoder.encode(kv, encodingContext, dos);
}
- BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
- baos.writeTo(stream);
- encoder.endBlockEncoding(encodingContext, dos, stream.getBuffer());
+ encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer());
byte[] encodedData = baos.toByteArray();
testAlgorithm(encodedData, unencodedDataBuf, encoder);
http://git-wip-us.apache.org/repos/asf/hbase/blob/da932ee3/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
index 2523a8c..6f434bb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
@@ -20,7 +20,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -31,12 +30,12 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
-import org.apache.hadoop.hbase.io.hfile.HFileBlock.Writer.BufferGrabbingByteArrayOutputStream;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.ChecksumType;
@@ -217,9 +216,7 @@ public class TestHFileDataBlockEncoder {
for (KeyValue kv : kvs) {
blockEncoder.encode(kv, context, dos);
}
- BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
- baos.writeTo(stream);
- blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA);
+ blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA);
byte[] encodedBytes = baos.toByteArray();
size = encodedBytes.length - block.getDummyHeaderForVersion().length;
return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
[10/12] hbase git commit: HBASE-15027 - Findbug fix (addendum)
Posted by sy...@apache.org.
HBASE-15027 - Findbug fix (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/894ce84b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/894ce84b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/894ce84b
Branch: refs/heads/hbase-12439
Commit: 894ce84b8330f9e3b1ca5b04e591c7a77cd2ce13
Parents: c2f67a0
Author: ramkrishna <ra...@gmail.com>
Authored: Mon Jan 11 16:50:59 2016 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Mon Jan 11 16:50:59 2016 +0530
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/CompactedHFilesDischarger.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/894ce84b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java
index c4974cf..18cf35d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java
@@ -73,6 +73,9 @@ public class CompactedHFilesDischarger extends ScheduledChore {
@Override
public void chore() {
+ // Noop if rss is null. This will never happen in a normal condition except for cases
+ // when the test case is not spinning up a cluster
+ if (regionServerServices == null) return;
List<Region> onlineRegions = regionServerServices.getOnlineRegions();
if (onlineRegions != null) {
for (Region region : onlineRegions) {
[02/12] hbase git commit: HBASE-15076 Add getScanner(Scan scan,
List additionalScanners) API into Region interface
(Anoop Sam John)
Posted by sy...@apache.org.
HBASE-15076 Add getScanner(Scan scan, List<KeyValueScanner> additionalScanners) API into Region interface (Anoop Sam John)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5bde960b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5bde960b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5bde960b
Branch: refs/heads/hbase-12439
Commit: 5bde960b9525f97d26f8917041d550eeb0e2b781
Parents: 3d36779
Author: stack <st...@apache.org>
Authored: Thu Jan 7 13:45:01 2016 -0800
Committer: stack <st...@apache.org>
Committed: Thu Jan 7 13:45:01 2016 -0800
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/HRegion.java | 26 ++++++++++++--------
.../hadoop/hbase/regionserver/Region.java | 15 +++++++++++
2 files changed, 31 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bde960b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e553fcc..38404c7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1819,27 +1819,27 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* We are trying to remove / relax the region read lock for compaction.
* Let's see what are the potential race conditions among the operations (user scan,
* region split, region close and region bulk load).
- *
+ *
* user scan ---> region read lock
* region split --> region close first --> region write lock
* region close --> region write lock
* region bulk load --> region write lock
- *
+ *
* read lock is compatible with read lock. ---> no problem with user scan/read
* region bulk load does not cause problem for compaction (no consistency problem, store lock
* will help the store file accounting).
* They can run almost concurrently at the region level.
- *
+ *
* The only remaining race condition is between the region close and compaction.
* So we will evaluate, below, how region close intervenes with compaction if compaction does
* not acquire region read lock.
- *
+ *
* Here are the steps for compaction:
* 1. obtain list of StoreFile's
* 2. create StoreFileScanner's based on list from #1
* 3. perform compaction and save resulting files under tmp dir
* 4. swap in compacted files
- *
+ *
* #1 is guarded by store lock. This patch does not change this --> no worse or better
* For #2, we obtain smallest read point (for region) across all the Scanners (for both default
* compactor and stripe compactor).
@@ -1851,7 +1851,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* This will not conflict with compaction.
* For #3, it can be performed in parallel to other operations.
* For #4 bulk load and compaction don't conflict with each other on the region level
- * (for multi-family atomicy).
+ * (for multi-family atomicy).
* Region close and compaction are guarded pretty well by the 'writestate'.
* In HRegion#doClose(), we have :
* synchronized (writestate) {
@@ -2575,6 +2575,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
return getScanner(scan, true);
}
+ @Override
+ public RegionScanner getScanner(Scan scan, List<KeyValueScanner> additionalScanners)
+ throws IOException {
+ return getScanner(scan, additionalScanners, true);
+ }
+
public RegionScanner getScanner(Scan scan, boolean copyCellsFromSharedMem) throws IOException {
RegionScanner scanner = getScanner(scan, null, copyCellsFromSharedMem);
return scanner;
@@ -7073,7 +7079,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
*/
private static List<Tag> carryForwardTags(final Cell cell, final List<Tag> tags) {
if (cell.getTagsLength() <= 0) return tags;
- List<Tag> newTags = tags == null? new ArrayList<Tag>(): /*Append Tags*/tags;
+ List<Tag> newTags = tags == null? new ArrayList<Tag>(): /*Append Tags*/tags;
Iterator<Tag> i = CellUtil.tagsIterator(cell);
while (i.hasNext()) newTags.add(i.next());
return newTags;
@@ -7349,7 +7355,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// They are subtley different in quiet a few ways. This came out only
// after study. I am not sure that many of the differences are intentional.
- // TODO: St.Ack 20150907
+ // TODO: St.Ack 20150907
@Override
public Result increment(Increment mutation, long nonceGroup, long nonce)
@@ -7363,7 +7369,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
boolean writeToWAL = durability != Durability.SKIP_WAL;
WALEdit walEdits = null;
List<Cell> allKVs = new ArrayList<Cell>(mutation.size());
-
+
Map<Store, List<Cell>> tempMemstore = new HashMap<Store, List<Cell>>();
long size = 0;
long txid = 0;
@@ -8166,7 +8172,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
WALKey key = new HLogKey(getRegionInfo().getEncodedNameAsBytes(),
getRegionInfo().getTable(), WALKey.NO_SEQUENCE_ID, 0, null,
HConstants.NO_NONCE, HConstants.NO_NONCE, getMVCC());
-
+
// Call append but with an empty WALEdit. The returned sequence id will not be associated
// with any edit and we can be sure it went in after all outstanding appends.
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bde960b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
index 6d87057..5da8bcb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
@@ -392,6 +392,21 @@ public interface Region extends ConfigurationObserver {
*/
RegionScanner getScanner(Scan scan) throws IOException;
+ /**
+ * Return an iterator that scans over the HRegion, returning the indicated columns and rows
+ * specified by the {@link Scan}. The scanner will also include the additional scanners passed
+ * along with the scanners for the specified Scan instance. Should be careful with the usage to
+ * pass additional scanners only within this Region
+ * <p>
+ * This Iterator must be closed by the caller.
+ *
+ * @param scan configured {@link Scan}
+ * @param additionalScanners Any additional scanners to be used
+ * @return RegionScanner
+ * @throws IOException read exceptions
+ */
+ RegionScanner getScanner(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException;
+
/** The comparator to be used with the region */
CellComparator getCellCompartor();
[04/12] hbase git commit: HBASE-15027 Refactor the way the
CompactedHFileDischarger threads are created (Ram)
Posted by sy...@apache.org.
HBASE-15027 Refactor the way the CompactedHFileDischarger threads are
created (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/28c2b18d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/28c2b18d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/28c2b18d
Branch: refs/heads/hbase-12439
Commit: 28c2b18d30de4ce9564e328e5fdf42188e83fb63
Parents: f3ee6df
Author: ramkrishna <ra...@gmail.com>
Authored: Fri Jan 8 11:18:39 2016 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Fri Jan 8 11:18:39 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/executor/EventType.java | 9 +-
.../hadoop/hbase/executor/ExecutorType.java | 3 +-
.../hadoop/hbase/executor/ExecutorService.java | 4 +-
.../CompactedHFilesDischargeHandler.java | 45 ++++++
.../regionserver/CompactedHFilesDischarger.java | 112 ++++++++++++++
.../hadoop/hbase/regionserver/HRegion.java | 16 --
.../hbase/regionserver/HRegionServer.java | 26 +++-
.../hadoop/hbase/regionserver/HStore.java | 149 ++++++++-----------
.../hbase/regionserver/OnlineRegions.java | 6 +
.../compactions/CompactedHFilesDischarger.java | 74 ---------
.../compactions/CompactionConfiguration.java | 3 +
.../hadoop/hbase/MockRegionServerServices.java | 5 +
.../TestZooKeeperTableArchiveClient.java | 24 ++-
.../hadoop/hbase/master/MockRegionServer.java | 5 +
.../master/cleaner/TestSnapshotFromMaster.java | 19 ++-
.../regionserver/TestHRegionReplayEvents.java | 17 ++-
.../TestRegionMergeTransactionOnCluster.java | 36 +++--
.../hbase/regionserver/TestRegionReplicas.java | 15 +-
.../TestCompactedHFilesDischarger.java | 15 +-
19 files changed, 368 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
index ac76edb..a7759c5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
@@ -265,7 +265,14 @@ public enum EventType {
*
* RS_REGION_REPLICA_FLUSH
*/
- RS_REGION_REPLICA_FLUSH (82, ExecutorType.RS_REGION_REPLICA_FLUSH_OPS);
+ RS_REGION_REPLICA_FLUSH (82, ExecutorType.RS_REGION_REPLICA_FLUSH_OPS),
+
+ /**
+ * RS compacted files discharger <br>
+ *
+ * RS_COMPACTED_FILES_DISCHARGER
+ */
+ RS_COMPACTED_FILES_DISCHARGER (83, ExecutorType.RS_COMPACTED_FILES_DISCHARGER);
private final int code;
private final ExecutorType executor;
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
index d0f6bee..5a16149 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
@@ -46,7 +46,8 @@ public enum ExecutorType {
RS_CLOSE_META (25),
RS_PARALLEL_SEEK (26),
RS_LOG_REPLAY_OPS (27),
- RS_REGION_REPLICA_FLUSH_OPS (28);
+ RS_REGION_REPLICA_FLUSH_OPS (28),
+ RS_COMPACTED_FILES_DISCHARGER (29);
ExecutorType(int value) {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
index 410fb39..335b672 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
@@ -37,6 +37,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.monitoring.ThreadMonitoring;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -78,7 +79,8 @@ public class ExecutorService {
* started with the same name, this throws a RuntimeException.
* @param name Name of the service to start.
*/
- void startExecutorService(String name, int maxThreads) {
+ @VisibleForTesting
+ public void startExecutorService(String name, int maxThreads) {
if (this.executorMap.get(name) != null) {
throw new RuntimeException("An executor service with the name " + name +
" is already running!");
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischargeHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischargeHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischargeHandler.java
new file mode 100644
index 0000000..02160d8
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischargeHandler.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.regionserver.HStore;
+
+/**
+ * Event handler that handles the removal and archival of the compacted hfiles
+ */
+@InterfaceAudience.Private
+public class CompactedHFilesDischargeHandler extends EventHandler {
+
+ private HStore store;
+
+ public CompactedHFilesDischargeHandler(Server server, EventType eventType, HStore store) {
+ super(server, eventType);
+ this.store = store;
+ }
+
+ @Override
+ public void process() throws IOException {
+ this.store.closeAndArchiveCompactedFiles();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java
new file mode 100644
index 0000000..c4974cf
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactedHFilesDischarger.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.ScheduledChore;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.Store;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A chore service that periodically cleans up the compacted files when there are no active readers
+ * using those compacted files and also helps in clearing the block cache with these compacted
+ * file entries
+ */
+@InterfaceAudience.Private
+public class CompactedHFilesDischarger extends ScheduledChore {
+ private static final Log LOG = LogFactory.getLog(CompactedHFilesDischarger.class);
+ private RegionServerServices regionServerServices;
+ // Default is to use executor
+ @VisibleForTesting
+ private boolean useExecutor = true;
+
+ /**
+ * @param period the period of time to sleep between each run
+ * @param stopper the stopper
+ * @param regionServerServices the region server that starts this chore
+ */
+ public CompactedHFilesDischarger(final int period, final Stoppable stopper,
+ final RegionServerServices regionServerServices) {
+ // Need to add the config classes
+ super("CompactedHFilesCleaner", stopper, period);
+ this.regionServerServices = regionServerServices;
+ }
+
+ /**
+ * @param period the period of time to sleep between each run
+ * @param stopper the stopper
+ * @param regionServerServices the region server that starts this chore
+ * @param useExecutor true if to use the region server's executor service, false otherwise
+ */
+ @VisibleForTesting
+ public CompactedHFilesDischarger(final int period, final Stoppable stopper,
+ final RegionServerServices regionServerServices, boolean useExecutor) {
+ // Need to add the config classes
+ this(period, stopper, regionServerServices);
+ this.useExecutor = useExecutor;
+ }
+
+ @Override
+ public void chore() {
+ List<Region> onlineRegions = regionServerServices.getOnlineRegions();
+ if (onlineRegions != null) {
+ for (Region region : onlineRegions) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(
+ "Started the compacted hfiles cleaner for the region " + region.getRegionInfo());
+ }
+ for (Store store : region.getStores()) {
+ try {
+ if (useExecutor && regionServerServices != null) {
+ CompactedHFilesDischargeHandler handler = new CompactedHFilesDischargeHandler(
+ (Server) regionServerServices, EventType.RS_COMPACTED_FILES_DISCHARGER,
+ (HStore) store);
+ regionServerServices.getExecutorService().submit(handler);
+ } else {
+ // call synchronously if the RegionServerServices are not
+ // available
+ store.closeAndArchiveCompactedFiles();
+ }
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Completed archiving the compacted files for the region "
+ + region.getRegionInfo() + " under the store " + store.getColumnFamilyName());
+ }
+ } catch (Exception e) {
+ LOG.error("Exception while trying to close and archive the comapcted store "
+ + "files of the store " + store.getColumnFamilyName() + " in the" + " region "
+ + region.getRegionInfo(), e);
+ }
+ }
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(
+ "Completed the compacted hfiles cleaner for the region " + region.getRegionInfo());
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 38404c7..d059cd0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -74,7 +74,6 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.CompoundConfiguration;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.DroppedSnapshotException;
@@ -151,7 +150,6 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescripto
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.regionserver.ScannerContext.NextState;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputControllerFactory;
@@ -814,20 +812,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Initialize all the HStores
status.setStatus("Initializing all the Stores");
long maxSeqId = initializeStores(reporter, status);
- // Start the CompactedHFilesDischarger here. This chore helps to remove the compacted files
- // that will no longer be used in reads.
- if (this.getRegionServerServices() != null) {
- ChoreService choreService = this.getRegionServerServices().getChoreService();
- if (choreService != null) {
- // Default is 2 mins. The default value for TTLCleaner is 5 mins so we set this to
- // 2 mins so that compacted files can be archived before the TTLCleaner runs
- int cleanerInterval =
- conf.getInt("hbase.hfile.compactions.cleaner.interval", 2 * 60 * 1000);
- this.compactedFileDischarger =
- new CompactedHFilesDischarger(cleanerInterval, this.getRegionServerServices(), this);
- choreService.scheduleChore(compactedFileDischarger);
- }
- }
this.mvcc.advanceTo(maxSeqId);
if (ServerRegionReplicaUtil.shouldReplayRecoveredEdits(this)) {
// Recover any edits if available.
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 00046ba..b2cc78a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -134,6 +134,7 @@ import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.Repor
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
@@ -484,6 +485,8 @@ public class HRegionServer extends HasThread implements RegionServerServices, La
*/
protected final ConfigurationManager configurationManager;
+ private CompactedHFilesDischarger compactedFileDischarger;
+
/**
* Starts a HRegionServer at the default location.
* @param conf
@@ -615,6 +618,16 @@ public class HRegionServer extends HasThread implements RegionServerServices, La
}
});
}
+ // Create the CompactedFileDischarger chore service. This chore helps to
+ // remove the compacted files
+ // that will no longer be used in reads.
+ // Default is 2 mins. The default value for TTLCleaner is 5 mins so we set this to
+ // 2 mins so that compacted files can be archived before the TTLCleaner runs
+ int cleanerInterval =
+ conf.getInt("hbase.hfile.compaction.discharger.interval", 2 * 60 * 1000);
+ this.compactedFileDischarger =
+ new CompactedHFilesDischarger(cleanerInterval, (Stoppable)this, (RegionServerServices)this);
+ choreService.scheduleChore(compactedFileDischarger);
}
protected TableDescriptors getFsTableDescriptors() throws IOException {
@@ -1716,7 +1729,9 @@ public class HRegionServer extends HasThread implements RegionServerServices, La
}
this.service.startExecutorService(ExecutorType.RS_LOG_REPLAY_OPS, conf.getInt(
"hbase.regionserver.wal.max.splitters", SplitLogWorkerCoordination.DEFAULT_MAX_SPLITTERS));
-
+ // Start the threads for compacted files discharger
+ this.service.startExecutorService(ExecutorType.RS_COMPACTED_FILES_DISCHARGER,
+ conf.getInt(CompactionConfiguration.HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT, 10));
if (ServerRegionReplicaUtil.isRegionReplicaWaitForPrimaryFlushEnabled(conf)) {
this.service.startExecutorService(ExecutorType.RS_REGION_REPLICA_FLUSH_OPS,
conf.getInt("hbase.regionserver.region.replica.flusher.threads",
@@ -2725,6 +2740,15 @@ public class HRegionServer extends HasThread implements RegionServerServices, La
return tableRegions;
}
+ @Override
+ public List<Region> getOnlineRegions() {
+ List<Region> allRegions = new ArrayList<Region>();
+ synchronized (this.onlineRegions) {
+ // Return a clone copy of the onlineRegions
+ allRegions.addAll(onlineRegions.values());
+ }
+ return allRegions;
+ }
/**
* Gets the online tables in this RS.
* This method looks at the in-memory onlineRegions.
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 8d66696..9ebdaee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -37,9 +37,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -76,7 +74,6 @@ import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
@@ -91,7 +88,6 @@ import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ReflectionUtils;
-import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
@@ -139,8 +135,6 @@ public class HStore implements Store {
static int closeCheckInterval = 0;
private volatile long storeSize = 0L;
private volatile long totalUncompressedBytes = 0L;
- private ThreadPoolExecutor compactionCleanerthreadPoolExecutor = null;
- private CompletionService<StoreFile> completionService = null;
/**
* RWLock for store operations.
@@ -274,10 +268,6 @@ public class HStore implements Store {
"hbase.hstore.flush.retries.number must be > 0, not "
+ flushRetriesNumber);
}
- compactionCleanerthreadPoolExecutor = getThreadPoolExecutor(
- conf.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 10));
- completionService =
- new ExecutorCompletionService<StoreFile>(compactionCleanerthreadPoolExecutor);
cryptoContext = EncryptionUtil.createEncryptionContext(conf, family);
}
@@ -802,7 +792,9 @@ public class HStore implements Store {
Collection<StoreFile> compactedfiles =
storeEngine.getStoreFileManager().clearCompactedFiles();
// clear the compacted files
- removeCompactedFiles(compactedfiles);
+ if (compactedfiles != null && !compactedfiles.isEmpty()) {
+ removeCompactedfiles(compactedfiles);
+ }
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
ThreadPoolExecutor storeFileCloserThreadPool = this.region
@@ -844,9 +836,6 @@ public class HStore implements Store {
}
if (ioe != null) throw ioe;
}
- if (compactionCleanerthreadPoolExecutor != null) {
- compactionCleanerthreadPoolExecutor.shutdownNow();
- }
LOG.info("Closed " + this);
return result;
} finally {
@@ -2174,7 +2163,7 @@ public class HStore implements Store {
}
public static final long FIXED_OVERHEAD =
- ClassSize.align(ClassSize.OBJECT + (18 * ClassSize.REFERENCE) + (10 * Bytes.SIZEOF_LONG)
+ ClassSize.align(ClassSize.OBJECT + (16 * ClassSize.REFERENCE) + (10 * Bytes.SIZEOF_LONG)
+ (5 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN));
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
@@ -2311,92 +2300,72 @@ public class HStore implements Store {
} finally {
lock.readLock().unlock();
}
- removeCompactedFiles(copyCompactedfiles);
- }
-
- private ThreadPoolExecutor getThreadPoolExecutor(int maxThreads) {
- return Threads.getBoundedCachedThreadPool(maxThreads, maxThreads * 3, TimeUnit.SECONDS,
- new ThreadFactory() {
- private int count = 1;
-
- @Override
- public Thread newThread(Runnable r) {
- return new Thread(r, "CompactedfilesArchiver-" + count++);
- }
- });
+ if (copyCompactedfiles != null && !copyCompactedfiles.isEmpty()) {
+ removeCompactedfiles(copyCompactedfiles);
+ }
}
- private void removeCompactedFiles(Collection<StoreFile> compactedfiles) throws IOException {
- if (compactedfiles != null && !compactedfiles.isEmpty()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removing the compacted store files " + compactedfiles);
- }
- for (final StoreFile file : compactedfiles) {
- completionService.submit(new Callable<StoreFile>() {
- @Override
- public StoreFile call() throws IOException {
- synchronized (file) {
- try {
- StoreFile.Reader r = file.getReader();
- if (r == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("The file " + file + " was closed but still not archived.");
- }
- return file;
- }
- if (r != null && r.isCompactedAway() && !r.isReferencedInReads()) {
- // Even if deleting fails we need not bother as any new scanners won't be
- // able to use the compacted file as the status is already compactedAway
- if (LOG.isTraceEnabled()) {
- LOG.trace("Closing and archiving the file " + file.getPath());
- }
- r.close(true);
- // Just close and return
- return file;
- }
- } catch (Exception e) {
- LOG.error("Exception while trying to close the compacted store file "
- + file.getPath().getName());
- }
+ /**
+ * Archives and removes the compacted files
+ * @param compactedfiles The compacted files in this store that are not active in reads
+ * @throws IOException
+ */
+ private void removeCompactedfiles(Collection<StoreFile> compactedfiles)
+ throws IOException {
+ final List<StoreFile> filesToRemove = new ArrayList<StoreFile>(compactedfiles.size());
+ for (final StoreFile file : compactedfiles) {
+ synchronized (file) {
+ try {
+ StoreFile.Reader r = file.getReader();
+ if (r == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The file " + file + " was closed but still not archived.");
}
- return null;
+ filesToRemove.add(file);
}
- });
- }
- final List<StoreFile> filesToRemove = new ArrayList<StoreFile>(compactedfiles.size());
- try {
- for (final StoreFile file : compactedfiles) {
- Future<StoreFile> future = completionService.take();
- StoreFile closedFile = future.get();
- if (closedFile != null) {
- filesToRemove.add(closedFile);
+ if (r != null && r.isCompactedAway() && !r.isReferencedInReads()) {
+ // Even if deleting fails we need not bother as any new scanners won't be
+ // able to use the compacted file as the status is already compactedAway
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Closing and archiving the file " + file.getPath());
+ }
+ r.close(true);
+ // Just close and return
+ filesToRemove.add(file);
}
+ } catch (Exception e) {
+ LOG.error(
+ "Exception while trying to close the compacted store file " + file.getPath().getName());
}
- } catch (InterruptedException ie) {
- LOG.error("Interrupted exception while closing the compacted files", ie);
- } catch (Exception e) {
- LOG.error("Exception occured while closing the compacted files", e);
}
- if (isPrimaryReplicaStore()) {
- archiveAndRemoveCompactedFiles(filesToRemove);
+ }
+ if (this.isPrimaryReplicaStore()) {
+ // Only the primary region is allowed to move the file to archive.
+ // The secondary region does not move the files to archive. Any active reads from
+ // the secondary region will still work because the file as such has active readers on it.
+ if (!filesToRemove.isEmpty()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Moving the files " + filesToRemove + " to archive");
+ }
+ // Only if this is successful it has to be removed
+ this.fs.removeStoreFiles(this.getFamily().getNameAsString(), filesToRemove);
}
-
+ }
+ if (!filesToRemove.isEmpty()) {
+ // Clear the compactedfiles from the store file manager
+ clearCompactedfiles(filesToRemove);
}
}
- private void archiveAndRemoveCompactedFiles(List<StoreFile> filesToArchive) throws IOException {
- if (!filesToArchive.isEmpty()) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Moving the files " + filesToArchive + " to archive");
- }
- // Only if this is successful it has to be removed
- this.fs.removeStoreFiles(this.getFamily().getNameAsString(), filesToArchive);
- try {
- lock.writeLock().lock();
- this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToArchive);
- } finally {
- lock.writeLock().unlock();
- }
+ private void clearCompactedfiles(final List<StoreFile> filesToRemove) throws IOException {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Clearing the compacted file " + filesToRemove + " from this store");
+ }
+ try {
+ lock.writeLock().lock();
+ this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToRemove);
+ } finally {
+ lock.writeLock().unlock();
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
index 60fc9fb..310108c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OnlineRegions.java
@@ -67,4 +67,10 @@ public interface OnlineRegions extends Server {
* @throws java.io.IOException
*/
List<Region> getOnlineRegions(TableName tableName) throws IOException;
+
+ /**
+ * Get all online regions in this RS.
+ * @return List of online Region
+ */
+ List<Region> getOnlineRegions();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactedHFilesDischarger.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactedHFilesDischarger.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactedHFilesDischarger.java
deleted file mode 100644
index 4cf120d..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactedHFilesDischarger.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver.compactions;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.ScheduledChore;
-import org.apache.hadoop.hbase.Stoppable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.regionserver.Region;
-import org.apache.hadoop.hbase.regionserver.Store;
-
-/**
- * A chore service that periodically cleans up the compacted files when there are no active readers
- * using those compacted files and also helps in clearing the block cache with these compacted
- * file entries
- */
-@InterfaceAudience.Private
-public class CompactedHFilesDischarger extends ScheduledChore {
- private static final Log LOG = LogFactory.getLog(CompactedHFilesDischarger.class);
- private Region region;
-
- /**
- * @param period the period of time to sleep between each run
- * @param stopper the stopper
- * @param region the store to identify the family name
- */
- public CompactedHFilesDischarger(final int period, final Stoppable stopper, final Region region) {
- // Need to add the config classes
- super("CompactedHFilesCleaner", stopper, period);
- this.region = region;
- }
-
- @Override
- public void chore() {
- if (LOG.isTraceEnabled()) {
- LOG.trace(
- "Started the compacted hfiles cleaner for the region " + this.region.getRegionInfo());
- }
- for (Store store : region.getStores()) {
- try {
- store.closeAndArchiveCompactedFiles();
- if (LOG.isTraceEnabled()) {
- LOG.trace("Completed archiving the compacted files for the region "
- + this.region.getRegionInfo() + " under the store " + store.getColumnFamilyName());
- }
- } catch (Exception e) {
- LOG.error(
- "Exception while trying to close and archive the comapcted store files of the store "
- + store.getColumnFamilyName() + " in the region " + this.region.getRegionInfo(),
- e);
- }
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace(
- "Completed the compacted hfiles cleaner for the region " + this.region.getRegionInfo());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
index 62e7c7c..633477e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
@@ -64,6 +64,9 @@ public class CompactionConfiguration {
public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
"hbase.hstore.min.locality.to.skip.major.compact";
+ public static final String HBASE_HFILE_COMPACTION_DISCHARGER_THREAD_COUNT =
+ "hbase.hfile.compaction.discharger.thread.count";
+
Configuration conf;
StoreConfigInformation storeConfigInfo;
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
index a7fc75b..0986ad7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
@@ -110,6 +110,11 @@ public class MockRegionServerServices implements RegionServerServices {
}
@Override
+ public List<Region> getOnlineRegions() {
+ return null;
+ }
+
+ @Override
public void addToOnlineRegions(Region r) {
this.regions.put(r.getRegionInfo().getEncodedName(), r);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java
index 55e43de..64139ee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.backup.example;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
@@ -42,10 +44,11 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -79,6 +82,7 @@ public class TestZooKeeperTableArchiveClient {
private static ZKTableArchiveClient archivingClient;
private final List<Path> toCleanup = new ArrayList<Path>();
private static ClusterConnection CONNECTION;
+ private static RegionServerServices rss;
/**
* Setup the config for the cluster
@@ -93,6 +97,7 @@ public class TestZooKeeperTableArchiveClient {
ZooKeeperWatcher watcher = UTIL.getZooKeeperWatcher();
String archivingZNode = ZKTableArchiveClient.getArchiveZNode(UTIL.getConfiguration(), watcher);
ZKUtil.createWithParents(watcher, archivingZNode);
+ rss = mock(RegionServerServices.class);
}
private static void setupConf(Configuration conf) {
@@ -173,8 +178,11 @@ public class TestZooKeeperTableArchiveClient {
// create the region
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM);
HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd);
+ List<Region> regions = new ArrayList<Region>();
+ regions.add(region);
+ when(rss.getOnlineRegions()).thenReturn(regions);
final CompactedHFilesDischarger compactionCleaner =
- new CompactedHFilesDischarger(100, stop, region);
+ new CompactedHFilesDischarger(100, stop, rss, false);
loadFlushAndCompact(region, TEST_FAM);
compactionCleaner.chore();
// get the current hfiles in the archive directory
@@ -223,15 +231,21 @@ public class TestZooKeeperTableArchiveClient {
// create the region
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM);
HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd);
+ List<Region> regions = new ArrayList<Region>();
+ regions.add(region);
+ when(rss.getOnlineRegions()).thenReturn(regions);
final CompactedHFilesDischarger compactionCleaner =
- new CompactedHFilesDischarger(100, stop, region);
+ new CompactedHFilesDischarger(100, stop, rss, false);
loadFlushAndCompact(region, TEST_FAM);
compactionCleaner.chore();
// create the another table that we don't archive
hcd = new HColumnDescriptor(TEST_FAM);
HRegion otherRegion = UTIL.createTestRegion(otherTable, hcd);
- final CompactedHFilesDischarger compactionCleaner1 =
- new CompactedHFilesDischarger(100, stop, otherRegion);
+ regions = new ArrayList<Region>();
+ regions.add(otherRegion);
+ when(rss.getOnlineRegions()).thenReturn(regions);
+ final CompactedHFilesDischarger compactionCleaner1 = new CompactedHFilesDischarger(100, stop,
+ rss, false);
loadFlushAndCompact(otherRegion, TEST_FAM);
compactionCleaner1.chore();
// get the current hfiles in the archive directory
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index 234ad20..32f644b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -462,6 +462,11 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
}
@Override
+ public List<Region> getOnlineRegions() {
+ return null;
+ }
+
+ @Override
public OpenRegionResponse openRegion(RpcController controller,
OpenRegionRequest request) throws ServiceException {
// TODO Auto-generated method stub
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 60c5473..a6b6e4c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -46,11 +46,10 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnaps
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
+import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactedHFilesDischarger;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
@@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -324,10 +324,17 @@ public class TestSnapshotFromMaster {
region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it.
region.compactStores(); // min is 2 so will compact and archive
}
- for (HRegion region : regions) {
- CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, region);
- cleaner.chore();
+ List<RegionServerThread> regionServerThreads = UTIL.getMiniHBaseCluster()
+ .getRegionServerThreads();
+ HRegionServer hrs = null;
+ for (RegionServerThread rs : regionServerThreads) {
+ if (!rs.getRegionServer().getOnlineRegions(TABLE_NAME).isEmpty()) {
+ hrs = rs.getRegionServer();
+ break;
+ }
}
+ CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, hrs, false);
+ cleaner.chore();
LOG.info("After compaction File-System state");
FSUtils.logFileSystemState(fs, rootDir, LOG);
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index c59d6f7..382193b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -69,7 +70,6 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescripto
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -167,9 +167,17 @@ public class TestHRegionReplayEvents {
when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1));
when(rss.getConfiguration()).thenReturn(CONF);
when(rss.getRegionServerAccounting()).thenReturn(new RegionServerAccounting());
-
+ String string = org.apache.hadoop.hbase.executor.EventType.RS_COMPACTED_FILES_DISCHARGER
+ .toString();
+ ExecutorService es = new ExecutorService(string);
+ es.startExecutorService(
+ string+"-"+string, 1);
+ when(rss.getExecutorService()).thenReturn(es);
primaryRegion = HRegion.createHRegion(primaryHri, rootDir, CONF, htd, walPrimary);
primaryRegion.close();
+ List<Region> regions = new ArrayList<Region>();
+ regions.add(primaryRegion);
+ when(rss.getOnlineRegions()).thenReturn(regions);
primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
secondaryRegion = HRegion.openHRegion(secondaryHri, htd, null, CONF, rss, null);
@@ -1370,7 +1378,10 @@ public class TestHRegionReplayEvents {
// Test case 3: compact primary files
primaryRegion.compactStores();
- CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, primaryRegion);
+ List<Region> regions = new ArrayList<Region>();
+ regions.add(primaryRegion);
+ when(rss.getOnlineRegions()).thenReturn(regions);
+ CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, rss, false);
cleaner.chore();
secondaryRegion.refreshStoreFiles();
assertPathListsEqual(primaryRegion.getStoreFileList(families),
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index e0c1453..44b24ce 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -62,12 +62,12 @@ import org.apache.hadoop.hbase.master.RegionStates;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.util.StringUtils;
@@ -246,25 +246,37 @@ public class TestRegionMergeTransactionOnCluster {
count += hrfs.getStoreFiles(colFamily.getName()).size();
}
admin.compactRegion(mergedRegionInfo.getRegionName());
- // wait until merged region doesn't have reference file
+ // clean up the merged region store files
+ // wait until merged region have reference file
long timeout = System.currentTimeMillis() + waitTime;
+ int newcount = 0;
while (System.currentTimeMillis() < timeout) {
- if (!hrfs.hasReferences(tableDescriptor)) {
+ for(HColumnDescriptor colFamily : columnFamilies) {
+ newcount += hrfs.getStoreFiles(colFamily.getName()).size();
+ }
+ if(newcount > count) {
break;
}
Thread.sleep(50);
}
- int newcount = 0;
- for(HColumnDescriptor colFamily : columnFamilies) {
- newcount += hrfs.getStoreFiles(colFamily.getName()).size();
- }
assertTrue(newcount > count);
- // clean up the merged region store files
- List<HRegion> regions =
- TEST_UTIL.getHBaseCluster().getRegions(tableDescriptor.getName());
- for (HRegion region : regions) {
- CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, region);
+ List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
+ .getRegionServerThreads();
+ for (RegionServerThread rs : regionServerThreads) {
+ CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null,
+ rs.getRegionServer(), false);
cleaner.chore();
+ Thread.sleep(1000);
+ }
+ int newcount1 = 0;
+ while (System.currentTimeMillis() < timeout) {
+ for(HColumnDescriptor colFamily : columnFamilies) {
+ newcount1 += hrfs.getStoreFiles(colFamily.getName()).size();
+ }
+ if(newcount1 <= 1) {
+ break;
+ }
+ Thread.sleep(50);
}
// run CatalogJanitor to clean merge references in hbase:meta and archive the
// files of merging regions
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
index 67258aa..99f5801 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver;
import static org.apache.hadoop.hbase.regionserver.TestRegionServerNoMaster.*;
import java.io.IOException;
+import java.util.List;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -47,9 +48,9 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.util.StringUtils;
import org.junit.AfterClass;
@@ -454,8 +455,18 @@ public class TestRegionReplicas {
LOG.info("Force Major compaction on primary region " + hriPrimary);
primaryRegion.compact(true);
Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());
+ List<RegionServerThread> regionServerThreads = HTU.getMiniHBaseCluster()
+ .getRegionServerThreads();
+ HRegionServer hrs = null;
+ for (RegionServerThread rs : regionServerThreads) {
+ if (rs.getRegionServer()
+ .getOnlineRegion(primaryRegion.getRegionInfo().getRegionName()) != null) {
+ hrs = rs.getRegionServer();
+ break;
+ }
+ }
CompactedHFilesDischarger cleaner =
- new CompactedHFilesDischarger(100, null, (HRegion) primaryRegion);
+ new CompactedHFilesDischarger(100, null, hrs, false);
cleaner.chore();
// scan all the hfiles on the secondary.
// since there are no read on the secondary when we ask locations to
http://git-wip-us.apache.org/repos/asf/hbase/blob/28c2b18d/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java
index 40539c4..c23e794 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.regionserver.compactions;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
@@ -38,10 +40,12 @@ import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -62,6 +66,7 @@ public class TestCompactedHFilesDischarger {
private static CountDownLatch latch = new CountDownLatch(3);
private static AtomicInteger counter = new AtomicInteger(0);
private static AtomicInteger scanCompletedCounter = new AtomicInteger(0);
+ private RegionServerServices rss;
@Before
public void setUp() throws Exception {
@@ -71,6 +76,10 @@ public class TestCompactedHFilesDischarger {
HRegionInfo info = new HRegionInfo(tableName, null, null, false);
Path path = testUtil.getDataTestDir(getClass().getSimpleName());
region = HBaseTestingUtility.createRegionAndWAL(info, path, testUtil.getConfiguration(), htd);
+ rss = mock(RegionServerServices.class);
+ List<Region> regions = new ArrayList<Region>();
+ regions.add(region);
+ when(rss.getOnlineRegions()).thenReturn(regions);
}
@After
@@ -86,7 +95,7 @@ public class TestCompactedHFilesDischarger {
public void testCompactedHFilesCleaner() throws Exception {
// Create the cleaner object
CompactedHFilesDischarger cleaner =
- new CompactedHFilesDischarger(1000, (Stoppable) null, (HRegion) region);
+ new CompactedHFilesDischarger(1000, (Stoppable) null, rss, false);
// Add some data to the region and do some flushes
for (int i = 1; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
@@ -152,7 +161,7 @@ public class TestCompactedHFilesDischarger {
public void testCleanerWithParallelScannersAfterCompaction() throws Exception {
// Create the cleaner object
CompactedHFilesDischarger cleaner =
- new CompactedHFilesDischarger(1000, (Stoppable) null, (HRegion) region);
+ new CompactedHFilesDischarger(1000, (Stoppable) null, rss, false);
// Add some data to the region and do some flushes
for (int i = 1; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
@@ -223,7 +232,7 @@ public class TestCompactedHFilesDischarger {
public void testCleanerWithParallelScanners() throws Exception {
// Create the cleaner object
CompactedHFilesDischarger cleaner =
- new CompactedHFilesDischarger(1000, (Stoppable) null, (HRegion) region);
+ new CompactedHFilesDischarger(1000, (Stoppable) null, rss, false);
// Add some data to the region and do some flushes
for (int i = 1; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
[09/12] hbase git commit: HBASE-15057 local-master-backup.sh doesn't
start HMaster correctly (Samir Ahmic)
Posted by sy...@apache.org.
HBASE-15057 local-master-backup.sh doesn't start HMaster correctly (Samir Ahmic)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c2f67a05
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c2f67a05
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c2f67a05
Branch: refs/heads/hbase-12439
Commit: c2f67a053ebe687773e5e35aa7c060de964dbb68
Parents: 4a26644
Author: tedyu <yu...@gmail.com>
Authored: Fri Jan 8 15:58:01 2016 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Fri Jan 8 15:58:01 2016 -0800
----------------------------------------------------------------------
bin/local-master-backup.sh | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c2f67a05/bin/local-master-backup.sh
----------------------------------------------------------------------
diff --git a/bin/local-master-backup.sh b/bin/local-master-backup.sh
index bd265ab..cc2dc56 100755
--- a/bin/local-master-backup.sh
+++ b/bin/local-master-backup.sh
@@ -40,6 +40,7 @@ run_master () {
DN=$2
export HBASE_IDENT_STRING="$USER-$DN"
HBASE_MASTER_ARGS="\
+ -D hbase.master.port=`expr 16000 + $DN` \
-D hbase.master.info.port=`expr 16010 + $DN` \
-D hbase.regionserver.port=`expr 16020 + $DN` \
-D hbase.regionserver.info.port=`expr 16030 + $DN` \
[05/12] hbase git commit: HBASE-13525 replace test-patch with
customizations needed to use yetus
Posted by sy...@apache.org.
HBASE-13525 replace test-patch with customizations needed to use yetus
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/164aeb53
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/164aeb53
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/164aeb53
Branch: refs/heads/hbase-12439
Commit: 164aeb53992150f0336f17a32ecb2fb733495964
Parents: 28c2b18
Author: Sean Busbey <bu...@apache.org>
Authored: Sun Dec 27 23:26:49 2015 -0600
Committer: Sean Busbey <bu...@cloudera.com>
Committed: Fri Jan 8 07:39:40 2016 -0600
----------------------------------------------------------------------
dev-support/hbase-personality.sh | 345 ++++++++++
dev-support/test-patch.properties | 35 --
dev-support/test-patch.sh | 1070 --------------------------------
pom.xml | 3 +
4 files changed, 348 insertions(+), 1105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/164aeb53/dev-support/hbase-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
new file mode 100755
index 0000000..2d31fd5
--- /dev/null
+++ b/dev-support/hbase-personality.sh
@@ -0,0 +1,345 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# You'll need a local installation of
+# [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
+# to use this personality.
+#
+# Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
+# build from it, or use the convenience binaries provided on that download page.
+#
+# To run against, e.g. HBASE-15074 you'd then do
+# ```bash
+# test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
+# ```
+#
+# If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
+# ```bash
+# test-patch --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
+# ````
+#
+# pass the `--jenkins` flag if you want to allow test-patch to destructively alter local working
+# directory / branch in order to have things match what the issue patch requests.
+
+personality_plugins "all"
+
+function personality_globals
+{
+ #shellcheck disable=SC2034
+ PROJECT_NAME=hbase
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^HBASE-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/hbase"
+
+ # TODO use PATCH_BRANCH to select hadoop versions to use.
+ # All supported Hadoop versions that we want to test the compilation with
+ HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1"
+
+ # TODO use PATCH_BRANCH to select jdk versions to use.
+
+ # Override the maven options
+ MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
+
+}
+
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ extra="-DHBasePatchProcess"
+
+ if [[ ${repostatus} == branch
+ && ${testtype} == mvninstall ]];then
+ personality_enqueue_module . ${extra}
+ return
+ fi
+
+ if [[ ${testtype} = findbugs ]]; then
+ for module in ${CHANGED_MODULES}; do
+ # skip findbugs on hbase-shell
+ if [[ ${module} == hbase-shell ]]; then
+ continue
+ else
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ fi
+ done
+ return
+ fi
+
+ if [[ ${testtype} = unit ]]; then
+ extra="${extra} -PrunAllTests"
+
+ # Inject the jenkins build-id for our surefire invocations
+ # Used by zombie detection stuff, even though we're not including that yet.
+ if [ -n "${BUILD_ID}" ]; then
+ extra="${extra} -Dbuild.id=${BUILD_ID}"
+ fi
+ fi
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
+
+###################################################
+# Below here are our one-off tests specific to hbase.
+# TODO break them into individual files so it's easier to maintain them?
+
+# TODO line length check? could ignore all java files since checkstyle gets them.
+
+###################################################
+
+add_test_type hadoopcheck
+
+function hadoopcheck_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test hadoopcheck
+ fi
+}
+
+function hadoopcheck_rebuild
+{
+ local repostatus=$1
+ local hadoopver
+ local logfile
+ local count
+ local result=0
+
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
+
+ big_console_header "Compiling against various Hadoop versions"
+
+ export MAVEN_OPTS="${MAVEN_OPTS}"
+ for hadoopver in ${HBASE_HADOOP_VERSIONS}; do
+ logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
+ echo_and_redirect "${logfile}" \
+ "${MAVEN}" clean install \
+ -DskipTests -DHBasePatchProcess \
+ -Dhadoop-two.version="${hadoopver}"
+ count=$(${GREP} -c ERROR "${logfile}")
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${hadoopver}."
+ ((result=result+1))
+ fi
+ done
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
+ return 0
+}
+
+######################################
+
+# TODO if we need th protoc check, we probably need to check building all the modules that rely on hbase-protocol
+add_test_type hbaseprotoc
+
+function hbaseprotoc_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.proto$ ]]; then
+ add_test hbaseprotoc
+ fi
+}
+
+function hbaseprotoc_rebuild
+{
+ local i=0
+ local fn
+ local module
+ local logfile
+ local count
+ local result
+
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
+
+ verify_needed_test hbaseprotoc
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "Patch HBase protoc plugin"
+
+ start_clock
+
+
+ personality_modules patch hbaseprotoc
+ modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
+
+ # shellcheck disable=SC2153
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+ module=${MODULE[$i]}
+ fn=$(module_file_fragment "${module}")
+ logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
+
+ count=$(${GREP} -c ERROR "${logfile}")
+
+ if [[ ${count} -gt 0 ]]; then
+ module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
+ "${count} new protoc errors in ${module}."
+ ((result=result+1))
+ fi
+ ((i=i+1))
+ done
+
+ modules_messages patch hbaseprotoc true
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+######################################
+
+add_test_type hbaseanti
+
+function hbaseanti_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test hbaseanti
+ fi
+}
+
+function hbaseanti_patchfile
+{
+ local patchfile=$1
+ local warnings
+ local result
+
+ verify_needed_test hbaseanti
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "Checking for known anti-patterns"
+
+ start_clock
+
+ warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}")
+ if [[ ${warnings} -gt 0 ]]; then
+ add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}."
+ ((result=result+1))
+ fi
+
+ warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}")
+ if [[ ${warnings} -gt 0 ]]; then
+ add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}."
+ ((result=result+1))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
+ return 0
+}
+
+# Work around HBASE-15042
+function mvnsite_filefilter
+{
+ local filename=$1
+
+ if [[ ${BUILDTOOL} = maven ]]; then
+ if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then
+ yetus_debug "tests/mvnsite: ${filename}"
+ add_test mvnsite
+ fi
+ fi
+}
+
+## This is named so that yetus will check us right after running tests.
+## Essentially, we check for normal failures and then we look for zombies.
+#function hbase_unit_logfilter
+#{
+# declare testtype="unit"
+# declare input=$1
+# declare output=$2
+# declare processes
+# declare process_output
+# declare zombies
+# declare zombie_count=0
+# declare zombie_process
+#
+# yetus_debug "in hbase-specific unit logfilter."
+#
+# # pass-through to whatever is counting actual failures
+# if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
+# "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
+# elif declare -f ${testtype}_logfilter >/dev/null; then
+# "${testtype}_logfilter" "${input}" "${output}"
+# fi
+#
+# start_clock
+# if [ -n "${BUILD_ID}" ]; then
+# yetus_debug "Checking for zombie test processes."
+# processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+# if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
+# yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
+# yetus_debug "${processes}"
+# sleep 30
+# #shellcheck disable=SC2016
+# for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
+# # Test our zombie still running (and that it still an hbase build item)
+# process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+# if [[ -n "${process_output}" ]]; then
+# yetus_error "Zombie: ${process_output}"
+# ((zombie_count = zombie_count + 1))
+# zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
+# zombies="${zombies} ${zombie_process}"
+# fi
+# done
+# fi
+# if [ "${zombie_count}" -ne 0 ]; then
+# add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
+# populate_test_table "zombie unit tests" "${zombies}"
+# else
+# yetus_info "Zombie check complete. All test runs exited normally."
+# stop_clock
+# fi
+# else
+# add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
+# fi
+#
+#}
http://git-wip-us.apache.org/repos/asf/hbase/blob/164aeb53/dev-support/test-patch.properties
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.properties b/dev-support/test-patch.properties
deleted file mode 100644
index bc29896..0000000
--- a/dev-support/test-patch.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
-
-# The number of acceptable warning for *all* modules
-# Please update the per-module test-patch.properties if you update this file.
-
-OK_RELEASEAUDIT_WARNINGS=0
-# Allow four warnings. Javadoc complains about sun.misc.Unsafe use.
-# See HBASE-7457, HBASE-13761
-# Allow 2 additional warnings for Scala stub notice about MR. See HBASE-13992
-OK_JAVADOC_WARNINGS=9
-
-MAX_LINE_LENGTH=100
-
-# All supported branches for testing with precommit build
-# branch-1.x should apprear before branch-1 since the latter is a prefix
-BRANCH_NAMES="0.94 0.98 branch-1.0 branch-1.1 branch-1.2 branch-1 master hbase-12439 hbase-11339"
-
-# All supported Hadoop versions that we want to test the compilation with
-HADOOP2_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.0 2.6.1 2.7.0 2.7.1"
-HADOOP3_VERSIONS="3.0.0-SNAPSHOT"
http://git-wip-us.apache.org/repos/asf/hbase/blob/164aeb53/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
deleted file mode 100755
index d0c0346..0000000
--- a/dev-support/test-patch.sh
+++ /dev/null
@@ -1,1070 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-#set -x
-
-### Setup some variables.
-### GIT_COMMIT and BUILD_URL are set by Hudson if it is run by patch process
-### Read variables from properties file
-bindir=$(dirname $0)
-
-# Defaults
-if [ -z "$MAVEN_HOME" ]; then
- MVN=mvn
-else
- MVN=$MAVEN_HOME/bin/mvn
-fi
-
-NEWLINE=$'\n'
-
-PROJECT_NAME=HBase
-JENKINS=false
-MOVE_PATCH_DIR=true
-PATCH_DIR=/tmp
-BASEDIR=$(pwd)
-BRANCH_NAME="master"
-
-. $BASEDIR/dev-support/test-patch.properties
-
-PS=${PS:-ps}
-AWK=${AWK:-awk}
-WGET=${WGET:-wget}
-GREP=${GREP:-grep}
-EGREP=${EGREP:-egrep}
-PATCH=${PATCH:-patch}
-JIRACLI=${JIRA:-jira}
-FINDBUGS_HOME=${FINDBUGS_HOME}
-FORREST_HOME=${FORREST_HOME}
-ECLIPSE_HOME=${ECLIPSE_HOME}
-GIT=${GIT:-git}
-
-###############################################################################
-printUsage() {
- echo "Usage: $0 [options] patch-file | defect-number"
- echo
- echo "Where:"
- echo " patch-file is a local patch file containing the changes to test"
- echo " defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)"
- echo
- echo "Options:"
- echo "--patch-dir=<dir> The directory for working and output files (default '/tmp')"
- echo "--basedir=<dir> The directory to apply the patch to (default current directory)"
- echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')"
- echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')"
- echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
- echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
- echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
- echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
- echo "--forrest-home=<path> Forrest home directory (default FORREST_HOME environment variable)"
- echo "--dirty-workspace Allow the local workspace to have uncommitted changes"
- echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
- echo
- echo "Jenkins-only options:"
- echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
- echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
- echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
- echo "--jira-password=<pw> The password for the 'jira' command"
- echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
-}
-
-###############################################################################
-parseArgs() {
- for i in $*
- do
- case $i in
- --jenkins)
- JENKINS=true
- ;;
- --no-move-patch-dir)
- MOVE_PATCH_DIR=false
- ;;
- --patch-dir=*)
- PATCH_DIR=${i#*=}
- ;;
- --basedir=*)
- BASEDIR=${i#*=}
- ;;
- --mvn-cmd=*)
- MVN=${i#*=}
- ;;
- --ps-cmd=*)
- PS=${i#*=}
- ;;
- --awk-cmd=*)
- AWK=${i#*=}
- ;;
- --wget-cmd=*)
- WGET=${i#*=}
- ;;
- --grep-cmd=*)
- GREP=${i#*=}
- ;;
- --patch-cmd=*)
- PATCH=${i#*=}
- ;;
- --jira-cmd=*)
- JIRACLI=${i#*=}
- ;;
- --jira-password=*)
- JIRA_PASSWD=${i#*=}
- ;;
- --findbugs-home=*)
- FINDBUGS_HOME=${i#*=}
- ;;
- --forrest-home=*)
- FORREST_HOME=${i#*=}
- ;;
- --eclipse-home=*)
- ECLIPSE_HOME=${i#*=}
- ;;
- --dirty-workspace)
- DIRTY_WORKSPACE=true
- ;;
- --git-cmd=*)
- GIT=${i#*=}
- ;;
- *)
- PATCH_OR_DEFECT=$i
- ;;
- esac
- done
- if [ -z "$PATCH_OR_DEFECT" ]; then
- printUsage
- exit 1
- fi
- if [[ $JENKINS == "true" ]] ; then
- echo "Running in Jenkins mode"
- defect=$PATCH_OR_DEFECT
- ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME"
- else
- echo "Running in developer mode"
- JENKINS=false
- ### PATCH_FILE contains the location of the patchfile
- PATCH_FILE=$PATCH_OR_DEFECT
- if [[ ! -e "$PATCH_FILE" ]] ; then
- echo "Unable to locate the patch file $PATCH_FILE"
- cleanupAndExit 0
- fi
- ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
- if [[ ! -e "$PATCH_DIR" ]] ; then
- mkdir "$PATCH_DIR"
- if [[ $? == 0 ]] ; then
- echo "$PATCH_DIR has been created"
- else
- echo "Unable to create $PATCH_DIR"
- cleanupAndExit 0
- fi
- fi
- ### Obtain the patch filename to append it to the version number
- defect=`basename $PATCH_FILE`
- fi
-}
-
-###############################################################################
-checkout () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Testing patch for ${defect}."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- ### When run by a developer, if the workspace contains modifications, do not continue
- ### unless the --dirty-workspace option was set
- if [[ $JENKINS == "false" ]] ; then
- if [[ -z $DIRTY_WORKSPACE ]] ; then
- # Ref http://stackoverflow.com/a/2659808 for details on checking dirty status
- ${GIT} diff-index --quiet HEAD
- if [[ $? -ne 0 ]] ; then
- uncommitted=`${GIT} diff --name-only HEAD`
- uncommitted="You have the following files with uncommitted changes:${NEWLINE}${uncommitted}"
- fi
- untracked="$(${GIT} ls-files --exclude-standard --others)" && test -z "${untracked}"
- if [[ $? -ne 0 ]] ; then
- untracked="You have untracked and unignored files:${NEWLINE}${untracked}"
- fi
- if [[ $uncommitted || $untracked ]] ; then
- echo "ERROR: can't run in a workspace that contains modifications."
- echo "Pass the '--dirty-workspace' flag to bypass."
- echo ""
- echo "${uncommitted}"
- echo ""
- echo "${untracked}"
- cleanupAndExit 1
- fi
- fi
- echo
- fi
- return $?
-}
-
-findBranchNameFromPatchName() {
- local patchName=$1
- for LOCAL_BRANCH_NAME in $BRANCH_NAMES; do
- if [[ $patchName =~ /jira/secure/attachment/[0-9]*/.*$LOCAL_BRANCH_NAME ]]; then
- BRANCH_NAME=$LOCAL_BRANCH_NAME
- break
- fi
- done
- return 0
-}
-
-checkoutBranch() {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Testing patch on branch ${BRANCH_NAME}."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- if [[ $JENKINS == "true" ]] ; then
- if [[ "$BRANCH_NAME" != "master" ]]; then
- echo "origin/${BRANCH_NAME} HEAD is commit `${GIT} rev-list origin/${BRANCH_NAME} -1`"
- echo "${GIT} checkout -f `${GIT} rev-list origin/${BRANCH_NAME} -1`"
- ${GIT} checkout -f `${GIT} rev-list origin/${BRANCH_NAME} -1`
- echo "${GIT} status"
- ${GIT} status
- fi
- fi
-}
-
-###############################################################################
-### Collect findbugs reports
-collectFindbugsReports() {
- name=$1
- basedir=$2
- patch_dir=$3
- for file in $(find $basedir -name findbugsXml.xml)
- do
- relative_file=${file#$basedir/} # strip leading $basedir prefix
- if [ ! $relative_file == "target/findbugsXml.xml" ]; then
- module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
- module_suffix=`basename ${module_suffix}`
- fi
-
- cp $file $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
- $FINDBUGS_HOME/bin/setBugDatabaseInfo -name $name \
- $patch_dir/${name}FindbugsWarnings${module_suffix}.xml \
- $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
- done
- xml_file=$patch_dir/${name}FindbugsWarnings.xml
- html_file=$patch_dir/${name}FindbugsWarnings.html
- $FINDBUGS_HOME/bin/unionBugs -withMessages \
- -output $xml_file $patch_dir/${name}FindbugsWarnings*.xml
- $FINDBUGS_HOME/bin/convertXmlToText -html $xml_file $html_file
- file $xml_file $html_file
-}
-
-###############################################################################
-setup () {
- ### Download latest patch file (ignoring .htm and .html) when run from patch process
- if [[ $JENKINS == "true" ]] ; then
- $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect
- if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then
- echo "$defect is not \"Patch Available\". Exiting."
- cleanupAndExit 0
- fi
- relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $EGREP '(\.txt$|\.patch$|\.diff$)' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
- patchURL="http://issues.apache.org${relativePatchURL}"
- patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
- # ensure attachment has not already been tested
- ATTACHMENT_ID=$(basename $(dirname $patchURL))
- if grep -q "ATTACHMENT ID: $ATTACHMENT_ID" $PATCH_DIR/jira
- then
- echo "Attachment $ATTACHMENT_ID is already tested for $defect"
- exit 1
- fi
- echo "$defect patch is being downloaded at `date` from"
- echo "$patchURL"
- $WGET -q -O $PATCH_DIR/patch $patchURL
- VERSION=${GIT_COMMIT}_${defect}_PATCH-${patchNum}
- findBranchNameFromPatchName ${relativePatchURL}
- checkoutBranch
- JIRA_COMMENT="Here are the results of testing the latest attachment
- $patchURL
- against ${BRANCH_NAME} branch at commit ${GIT_COMMIT}.
- ATTACHMENT ID: ${ATTACHMENT_ID}"
-
- ### Copy the patch file to $PATCH_DIR
- else
- VERSION=PATCH-${defect}
- cp $PATCH_FILE $PATCH_DIR/patch
- if [[ $? == 0 ]] ; then
- echo "Patch file $PATCH_FILE copied to $PATCH_DIR"
- else
- echo "Could not copy $PATCH_FILE to $PATCH_DIR"
- cleanupAndExit 0
- fi
- fi
- ### exit if warnings are NOT defined in the properties file
- if [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]] ; then
- echo "Please define the following properties in test-patch.properties file"
- echo "OK_RELEASEAUDIT_WARNINGS"
- echo "OK_JAVADOC_WARNINGS"
- cleanupAndExit 1
- fi
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Pre-build master to verify stability and javac warnings"
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
- -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- # build core and tests
- $MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
- -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
- if [[ $? != 0 ]] ; then
- echo "mvn exit code was $?"
- ERR=`$GREP -A 5 'Compilation failure' $PATCH_DIR/trunkJavacWarnings.txt`
- if [[ ${#ERR} -ge 1 ]] ; then
- echo "Trunk compilation is broken?
- {code}$ERR{code}"
- cleanupAndExit 1
- fi
- fi
- mv target/checkstyle-result.xml $PATCH_DIR/trunkCheckstyle.xml
- collectFindbugsReports trunk $BASEDIR $PATCH_DIR
-}
-
-###############################################################################
-### Check for @author tags in the patch
-checkAuthor () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking there are no @author tags in the patch."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch`
- echo "There appear to be $authorTags @author tags in the patch."
- if [[ $authorTags != 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Hadoop community has agreed to not allow in code contributions."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 @author{color}. The patch does not contain any @author tags."
- return 0
-}
-
-###############################################################################
-### Check for tests in the patch
-checkTests () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking there are new or changed tests in the patch."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- testReferences=`$GREP -c -i '/test' $PATCH_DIR/patch`
- echo "There appear to be $testReferences test files referenced in the patch."
- if [[ $testReferences == 0 ]] ; then
- if [[ $JENKINS == "true" ]] ; then
- patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira`
- if [[ $patchIsDoc != 0 ]] ; then
- echo "The patch appears to be a documentation patch that doesn't require tests."
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests."
- return 0
- fi
- fi
- srcReferences=`${GREP} "diff --git" "${PATCH_DIR}/patch" | ${GREP} "src/main" | \
- ${GREP} -v "src/main/asciidoc" | ${GREP} -v "src/main/site" -c`
- if [[ $srcReferences == 0 ]] ; then
- echo "The patch doesn't appear to alter any code that requires tests."
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+0 tests included{color}. The patch appears to be a documentation, build,
- or dev-support patch that doesn't require tests."
- return 0
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests.
- Please justify why no new tests are needed for this patch.
- Also please list what manual steps were performed to verify this patch."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified tests."
- return 0
-}
-
-###############################################################################
-### Check there are no compilation errors, passing a file to be parsed.
-checkCompilationErrors() {
- local file=$1
- hadoopVersion=""
- if [ "$#" -ne 1 ]; then
- hadoopVersion="with Hadoop version $2"
- fi
- COMPILATION_ERROR=false
- eval $(awk '/ERROR/ {print "COMPILATION_ERROR=true"}' $file)
- if $COMPILATION_ERROR ; then
- ERRORS=$($AWK '/ERROR/ { print $0 }' $file)
- echo "======================================================================"
- echo "There are compilation errors $hadoopVersion."
- echo "======================================================================"
- echo "$ERRORS"
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 javac{color}. The patch appears to cause mvn compile goal to fail $hadoopVersion.
-
- Compilation errors resume:
- $ERRORS
- "
- submitJiraComment 1
- cleanupAndExit 1
- fi
-}
-
-###############################################################################
-### Check there are no protoc compilation errors, passing a file to be parsed.
-checkProtocCompilationErrors() {
- local file=$1
- COMPILATION_ERROR=false
- eval $(awk '/\[ERROR/ {print "COMPILATION_ERROR=true"}' $file)
- if $COMPILATION_ERROR ; then
- ERRORS=$($AWK '/\[ERROR/ { print $0 }' $file)
- echo "======================================================================"
- echo "There are Protoc compilation errors."
- echo "======================================================================"
- echo "$ERRORS"
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 javac{color}. The patch appears to cause mvn compile-protobuf profile to fail.
-
- Protoc Compilation errors resume:
- $ERRORS
- "
- cleanupAndExit 1
- fi
-}
-
-###############################################################################
-### Attempt to apply the patch
-applyPatch () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Applying patch."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
-
- export PATCH
- $BASEDIR/dev-support/smart-apply-patch.sh $PATCH_DIR/patch
- if [[ $? != 0 ]] ; then
- echo "PATCH APPLICATION FAILED"
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 patch{color}. The patch command could not apply the patch."
- return 1
- fi
- return 0
-}
-
-###############################################################################
-### Check against known anti-patterns
-checkAntiPatterns () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking against known anti-patterns."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- warnings=`$GREP 'new TreeMap<byte.*()' $PATCH_DIR/patch`
- if [[ $warnings != "" ]]; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 Anti-pattern{color}. The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: $warnings."
- return 1
- fi
- return 0
-}
-
-###############################################################################
-### Check that there are no incorrect annotations
-checkInterfaceAudience () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking against hadoop InterfaceAudience."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- warnings=`$GREP 'import org.apache.hadoop.classification' $PATCH_DIR/patch`
- if [[ $warnings != "" ]]; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 InterfaceAudience{color}. The patch appears to contain InterfaceAudience from hadoop rather than hbase: $warnings."
- return 1
- fi
- return 0
-}
-
-###############################################################################
-### Check there are no javadoc warnings
-checkJavadocWarnings () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched javadoc warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN clean package javadoc:javadoc -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- $MVN clean package javadoc:javadoc -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
- javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
- echo ""
- echo ""
- echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."
-
- ### if current warnings greater than OK_JAVADOC_WARNINGS
- if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages."
- # Add javadoc output url
- JIRA_COMMENT_FOOTER="Javadoc warnings: $BUILD_URL/artifact/patchprocess/patchJavadocWarnings.txt
-$JIRA_COMMENT_FOOTER"
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 javadoc{color}. The javadoc tool did not generate any warning messages."
- return 0
-}
-
-checkBuildWithHadoopVersions() {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Building with all supported Hadoop versions ."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- export MAVEN_OPTS="${MAVEN_OPTS}"
- for HADOOP2_VERSION in $HADOOP2_VERSIONS ; do
- echo "$MVN clean install -DskipTests -D${PROJECT_NAME}PatchProcess -Dhadoop-two.version=$HADOOP2_VERSION > $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt 2>&1"
- $MVN clean install -DskipTests -D${PROJECT_NAME}PatchProcess -Dhadoop-two.version=$HADOOP2_VERSION > $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt 2>&1
- checkCompilationErrors $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt $HADOOP2_VERSION
- done
-
- # TODO: add Hadoop3 versions and compilation here when we get the hadoop.profile=3.0 working
-
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 hadoop versions{color}. The patch compiles with all supported hadoop versions ($HADOOP2_VERSIONS)"
- return 0
-}
-
-###############################################################################
-### Check there are no changes in the number of Javac warnings
-checkJavacWarnings () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched javac warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN clean package -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- $MVN clean package -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavacWarnings.txt 2>&1
- checkCompilationErrors $PATCH_DIR/patchJavacWarnings.txt
- ### Compare trunk and patch javac warning numbers
- if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
- trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
- patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
- echo "There appear to be $trunkJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch."
- if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then
- if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the master's current $trunkJavacWarnings warnings)."
- return 1
- fi
- fi
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings."
- return 0
-}
-
-checkCheckstyleErrors() {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched Checkstyle errors."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- if [[ -f $PATCH_DIR/trunkCheckstyle.xml ]] ; then
- $MVN package -DskipTests checkstyle:checkstyle-aggregate > /dev/null 2>&1
- mv target/checkstyle-result.xml $PATCH_DIR/patchCheckstyle.xml
- mv target/site/checkstyle-aggregate.html $PATCH_DIR
- mv target/site/checkstyle.css $PATCH_DIR
- $BASEDIR/dev-support/checkstyle_report.py $PATCH_DIR/trunkCheckstyle.xml $PATCH_DIR/patchCheckstyle.xml
- if [[ $? -eq 1 ]] ; then
- JIRA_COMMENT_FOOTER="Checkstyle Errors: $BUILD_URL/artifact/patchprocess/checkstyle-aggregate.html
-
- $JIRA_COMMENT_FOOTER"
-
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 checkstyle{color}. The applied patch generated new checkstyle errors. Check build console for list of new errors."
- return 1
- fi
- fi
- JIRA_COMMENT_FOOTER="Checkstyle Errors: $BUILD_URL/artifact/patchprocess/checkstyle-aggregate.html
-
- $JIRA_COMMENT_FOOTER"
-
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 checkstyle{color}. The applied patch does not generate new checkstyle errors."
- return 0
-
-}
-###############################################################################
-checkProtocErrors () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining whether there is patched protoc error."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN clean install -DskipTests -Pcompile-protobuf -X -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchProtocErrors.txt 2>&1"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- $MVN clean install -DskipTests -Pcompile-protobuf -X -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchProtocErrors.txt 2>&1
- checkProtocCompilationErrors $PATCH_DIR/patchProtocErrors.txt
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 protoc{color}. The applied patch does not increase the total number of protoc compiler warnings."
- return 0
-}
-
-###############################################################################
-### Check there are no changes in the number of release audit (RAT) warnings
-checkReleaseAuditWarnings () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched release audit warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
- find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt
-
- ### Compare trunk and patch release audit warning numbers
- if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then
- patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt`
- echo ""
- echo ""
- echo "There appear to be $OK_RELEASEAUDIT_WARNINGS release audit warnings before the patch and $patchReleaseAuditWarnings release audit warnings after applying the patch."
- if [[ $patchReleaseAuditWarnings != "" && $OK_RELEASEAUDIT_WARNINGS != "" ]] ; then
- if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the master's current $OK_RELEASEAUDIT_WARNINGS warnings)."
- $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt
- echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt
- JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditWarnings.txt
-$JIRA_COMMENT_FOOTER"
- return 1
- fi
- fi
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings."
- return 0
-}
-
-###############################################################################
-### Check there are no changes in the number of Findbugs warnings
-checkFindbugsWarnings () {
- findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version`
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched Findbugs warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN clean package findbugs:findbugs -D${PROJECT_NAME}PatchProcess"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- $MVN clean package findbugs:findbugs -D${PROJECT_NAME}PatchProcess -DskipTests < /dev/null
-
- if [ $? != 0 ] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
- return 1
- fi
-
- collectFindbugsReports patch $BASEDIR $PATCH_DIR
- #this files are generated by collectFindbugsReports() named with its first argument
- patch_xml=$PATCH_DIR/patchFindbugsWarnings.xml
- trunk_xml=$PATCH_DIR/trunkFindbugsWarnings.xml
- # combine them to one database
- combined_xml=$PATCH_DIR/combinedFindbugsWarnings.xml
- new_xml=$PATCH_DIR/newFindbugsWarnings.xml
- new_html=$PATCH_DIR/newFindbugsWarnings.html
- $FINDBUGS_HOME/bin/computeBugHistory -useAnalysisTimes -withMessages \
- -output $combined_xml $trunk_xml $patch_xml
- findbugsWarnings=$($FINDBUGS_HOME/bin/filterBugs -first patch $combined_xml $new_xml)
- findbugsFixedWarnings=$($FINDBUGS_HOME/bin/filterBugs -fixed patch $combined_xml $new_xml)
- $FINDBUGS_HOME/bin/convertXmlToText -html $new_xml $new_html
- file $new_xml $new_html
- JIRA_COMMENT_FOOTER="Release Findbugs (version ${findbugs_version}) \
- warnings: $BUILD_URL/artifact/patchprocess/newFindbugsWarnings.html
-$JIRA_COMMENT_FOOTER"
- ### if current warnings greater than 0, fail
- if [[ $findbugsWarnings -gt 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings \
- new Findbugs (version ${findbugs_version}) warnings."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 findbugs{color}. The patch does not introduce any \
- new Findbugs (version ${findbugs_version}) warnings."
- return 0
-}
-
-###############################################################################
-### Check line lengths
-checkLineLengths () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking that no line have length > $MAX_LINE_LENGTH"
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- #see http://en.wikipedia.org/wiki/Diff#Unified_format
-
- MAX_LINE_LENGTH_PATCH=`expr $MAX_LINE_LENGTH + 1`
- lines=`cat $PATCH_DIR/patch | grep "^+" | grep -v "^@@" | grep -v "^+++" | grep -v "import" | grep -v "org.apache.thrift." | grep -v "com.google.protobuf." | grep -v "protobuf.generated" | awk -v len="$MAX_LINE_LENGTH_PATCH" 'length ($0) > len' | head -n 10`
- ll=`echo "$lines" | wc -l`
- if [[ "$ll" -gt "1" ]]; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 lineLengths{color}. The patch introduces the following lines longer than $MAX_LINE_LENGTH:
- $lines"
-
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 lineLengths{color}. The patch does not introduce lines longer than $MAX_LINE_LENGTH"
- return 0
-}
-
-###############################################################################
-### Run the tests
-runTests () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Running tests."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- failed_tests=""
- echo "$MVN clean test -Dsurefire.rerunFailingTestsCount=2 -P runAllTests -D${PROJECT_NAME}PatchProcess"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- ulimit -a
- # Need to export this so the zombie subshell picks up current content
- export JIRA_COMMENT
- $MVN clean test -Dsurefire.rerunFailingTestsCount=2 -P runAllTests -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- ### Find and format names of failed tests
- failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
-
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 core tests{color}. The patch failed these unit tests:
- $failed_tests"
- JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
- return 1
- else
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 core tests{color}. The patch passed unit tests in $modules."
- JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
- return $?
- fi
-}
-
-###############################################################################
-### Check docbook site xml
-checkSiteXml () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking Site generation"
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
-
- echo "$MVN package post-site -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchSiteOutput.txt 2>&1"
- export MAVEN_OPTS="${MAVEN_OPTS}"
- $MVN package post-site -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchSiteOutput.txt 2>&1
- if [[ $? != 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 site{color}. The patch appears to cause mvn post-site goal to fail."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 site{color}. The mvn post-site goal succeeds with this patch."
- return 0
-}
-
-###############################################################################
-### Run the inject-system-faults target
-checkInjectSystemFaults () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking the integrity of system test framework code."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
-
- ### Kill any rogue build processes from the last attempt
- $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
- #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
- #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
- echo "NOP"
- return 0
- if [[ $? != 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 system test framework{color}. The patch failed system test framework compile."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 system test framework{color}. The patch passed system test framework compile."
- return 0
-}
-
-###############################################################################
-### Submit a comment to the defect's Jira
-submitJiraComment () {
- local result=$1
- ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer
- if [[ $JENKINS == "false" ]] ; then
- JIRA_COMMENT_FOOTER=""
- fi
- if [[ $result == 0 ]] ; then
- comment="{color:green}+1 overall{color}. $JIRA_COMMENT
-
-$JIRA_COMMENT_FOOTER"
- else
- comment="{color:red}-1 overall{color}. $JIRA_COMMENT
-
-$JIRA_COMMENT_FOOTER"
- fi
- ### Output the test result to the console
- echo "
-
-
-
-$comment"
-
- if [[ $JENKINS == "true" ]] ; then
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Adding comment to Jira."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- ### Update Jira with a comment
- export USER=hudson
- $JIRACLI -s https://issues.apache.org/jira -a addcomment -u hadoopqa -p $JIRA_PASSWD --comment "$comment" --issue $defect
- $JIRACLI -s https://issues.apache.org/jira -a logout -u hadoopqa -p $JIRA_PASSWD
- fi
-}
-
-###############################################################################
-### Cleanup files
-cleanupAndExit () {
- local result=$1
- if [[ ${JENKINS} == "true" && ${MOVE_PATCH_DIR} == "true" ]] ; then
- if [ -e "$PATCH_DIR" ] ; then
- echo "Relocating patch dir into ${BASEDIR}"
- mv $PATCH_DIR $BASEDIR
- fi
- fi
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Finished build."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- exit $result
-}
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-JIRA_COMMENT=""
-JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console
-
-This message is automatically generated."
-
-### Check if arguments to the script have been specified properly or not
-parseArgs $@
-cd $BASEDIR
-
-echo "Version of this script: Wed Oct 14 00:29:04 PDT 2015"
-checkout
-RESULT=$?
-echo "RESULT = " $RESULT
-if [[ $JENKINS == "true" ]] ; then
- if [[ $RESULT != 0 ]] ; then
- exit 100
- fi
-fi
-setup
-checkAuthor
-RESULT=$?
-echo "RESULT = " $RESULT
-checkTests
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-applyPatch
-if [[ $? != 0 ]] ; then
- submitJiraComment 1
- cleanupAndExit 1
-fi
-
-checkAntiPatterns
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkBuildWithHadoopVersions
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkJavacWarnings
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkProtocErrors
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkJavadocWarnings
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkCheckstyleErrors
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkInterfaceAudience
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkFindbugsWarnings
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkReleaseAuditWarnings
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkLineLengths
-(( RESULT = RESULT + $? ))
-echo "RESULT = " $RESULT
-checkSiteXml
-(( RESULT = RESULT + $?))
-echo "RESULT = " $RESULT
-### Do not call these when run by a developer
-if [[ $JENKINS == "true" ]] ; then
- runTests
- (( RESULT = RESULT + $? ))
- echo "RESULT = " $RESULT
-JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
-$JIRA_COMMENT_FOOTER"
-fi
-submitJiraComment $RESULT
-cleanupAndExit $RESULT
http://git-wip-us.apache.org/repos/asf/hbase/blob/164aeb53/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 897b41c..ccc7eb5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1826,6 +1826,9 @@
<name>HBasePatchProcess</name>
</property>
</activation>
+ <properties>
+ <surefire.rerunFailingTestsCount>2</surefire.rerunFailingTestsCount>
+ </properties>
<build>
<plugins>
<plugin>