You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hbase.apache.org by "nkeywal (JIRA)" <ji...@apache.org> on 2012/11/26 19:56:58 UTC
[jira] [Created] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
nkeywal created HBASE-7220:
------------------------------
Summary: Creating a table with 3000 regions on 2 nodes fails after 1 hour
Key: HBASE-7220
URL: https://issues.apache.org/jira/browse/HBASE-7220
Project: HBase
Issue Type: Bug
Components: metrics, Performance, regionserver
Affects Versions: 0.96.0
Environment: small test cluster. not reproducible on a local cluster (needs a network).
Reporter: nkeywal
I'm trying to create a table with 3000 regions on two regions servers, from the shell.
It's ok on trunk a standalone config.
It's ok on 0.94
It's not ok on trunk: it fails after around 1 hour.
If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
I haven't looked at the metric code itself.
Patch used to remove the metrics from HRegion:
{noformat}
index c70e9ab..6677e65 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
private HTableDescriptor htableDescriptor = null;
private RegionSplitPolicy splitPolicy;
- private final MetricsRegion metricsRegion;
+ private final MetricsRegion metricsRegion = null;
/**
* Should only be used for testing purposes
@@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
this.coprocessorHost = null;
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
- this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
+ //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
}
/**
@@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
- this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
+ //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
/*
* timestamp.slop provides a server-side constraint on the timestamp. This
@@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
status.setStatus("Running coprocessor post-close hooks");
this.coprocessorHost.postClose(abort);
}
- this.metricsRegion.close();
+ //this.metricsRegion.close();
status.markComplete("Closed");
LOG.info("Closed " + this);
return result;
@@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
if (noOfPuts > 0) {
// There were some Puts in the batch.
double noOfMutations = noOfPuts + noOfDeletes;
- this.metricsRegion.updatePut();
+ //this.metricsRegion.updatePut();
}
if (noOfDeletes > 0) {
// There were some Deletes in the batch.
- this.metricsRegion.updateDelete();
+ //this.metricsRegion.updateDelete();
}
if (!success) {
for (int i = firstIndex; i < lastIndexExclusive; i++) {
@@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
// do after lock
- this.metricsRegion.updateGet();
+ //this.metricsRegion.updateGet();
return results;
}
@@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
closeRegionOperation();
}
- this.metricsRegion.updateAppend();
+ //this.metricsRegion.updateAppend();
if (flush) {
@@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
mvcc.completeMemstoreInsert(w);
}
closeRegionOperation();
- this.metricsRegion.updateIncrement();
+ //this.metricsRegion.updateIncrement();
}
if (flush) {
{noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Status: Patch Available (was: Open)
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Hadoop QA (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13504098#comment-13504098 ]
Hadoop QA commented on HBASE-7220:
----------------------------------
{color:red}-1 overall{color}. Here are the results of testing the latest attachment
http://issues.apache.org/jira/secure/attachment/12554899/HBASE-7220-0.patch
against trunk revision .
{color:green}+1 @author{color}. The patch does not contain any @author tags.
{color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests.
Please justify why no new tests are needed for this patch.
Also please list what manual steps were performed to verify this patch.
{color:green}+1 hadoop2.0{color}. The patch compiles against the hadoop 2.0 profile.
{color:red}-1 javadoc{color}. The javadoc tool appears to have generated 98 warning messages.
{color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings.
{color:red}-1 findbugs{color}. The patch appears to introduce 26 new Findbugs (version 1.3.9) warnings.
{color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings.
{color:green}+1 core tests{color}. The patch passed unit tests in .
Test results: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//testReport/
Findbugs warnings: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//artifact/trunk/patchprocess/newPatchFindbugsWarningshbase-hadoop2-compat.html
Findbugs warnings: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//artifact/trunk/patchprocess/newPatchFindbugsWarningshbase-examples.html
Findbugs warnings: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//artifact/trunk/patchprocess/newPatchFindbugsWarningshbase-protocol.html
Findbugs warnings: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//artifact/trunk/patchprocess/newPatchFindbugsWarningshbase-server.html
Findbugs warnings: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//artifact/trunk/patchprocess/newPatchFindbugsWarningshbase-hadoop1-compat.html
Findbugs warnings: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//artifact/trunk/patchprocess/newPatchFindbugsWarningshbase-common.html
Findbugs warnings: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//artifact/trunk/patchprocess/newPatchFindbugsWarningshbase-hadoop-compat.html
Console output: https://builds.apache.org/job/PreCommit-HBASE-Build/3404//console
This message is automatically generated.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Environment: (was: small test cluster. not reproducible on a local cluster (needs a network).)
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Hadoop Flags: Reviewed
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Ted Yu (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13507048#comment-13507048 ]
Ted Yu commented on HBASE-7220:
-------------------------------
{code}
if (fut == null || (!fut.isDone() && fut.getDelay(TimeUnit.MILLISECONDS) > 100)) return;
synchronized (lock) {
fut = executor.getExecutor().schedule(new JmxCacheBusterRunnable(), 5, TimeUnit.SECONDS);
}
{code}
Is it possible that fut becomes non-null after the check but before synchronized block runs ?
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Attachment: HBASE-7220-0.patch
This patch for me moved creating a table with 9k regions from 110 seconds to 52 seconds.
* Created a cache buster runnable
* async clears the jmx cache
* merges any clears that happen close together.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Hudson (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13506424#comment-13506424 ]
Hudson commented on HBASE-7220:
-------------------------------
Integrated in HBase-TRUNK-on-Hadoop-2.0.0 #279 (See [https://builds.apache.org/job/HBase-TRUNK-on-Hadoop-2.0.0/279/])
HBASE-7220 Creating a table with 3000 regions on 2 nodes fails after 1 hour (Revision 1415016)
Result = FAILURE
eclark :
Files :
* /hbase/trunk/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java
* /hbase/trunk/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java
* /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "stack (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13505875#comment-13505875 ]
stack commented on HBASE-7220:
------------------------------
+1
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13506113#comment-13506113 ]
Elliott Clark commented on HBASE-7220:
--------------------------------------
Committed revision 1415016.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13506111#comment-13506111 ]
Elliott Clark commented on HBASE-7220:
--------------------------------------
Looks like hadoopqa is stuck today. Ran tests locally and there were no related failures.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "nkeywal (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13505868#comment-13505868 ]
nkeywal commented on HBASE-7220:
--------------------------------
Tested locally, as fast as the version without any metrics. So +1 from me.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Enis Soztutar (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13507042#comment-13507042 ]
Enis Soztutar commented on HBASE-7220:
--------------------------------------
I've created HBASE-7243 for adding a test for this.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Status: Patch Available (was: Open)
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13505710#comment-13505710 ]
Elliott Clark commented on HBASE-7220:
--------------------------------------
Yeah that seems like a good idea. I'll investigate that soon.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Status: Open (was: Patch Available)
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13503993#comment-13503993 ]
Elliott Clark commented on HBASE-7220:
--------------------------------------
Just tried this locally and the reason this is taking so long is that when creating a new MetricsRegion, the whole metrics system is stopped and then started (to clear all jmx caches). This is taking a long time, and there is no need to do this on the main thread. All that matters is that the metrics system is restarted some time after the regions are added or removed.
I'll add a patch soon that moves this off of the main threads and into the metrics executor.
Additionally we can merge several of the reset requests.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Environment: small test cluster. not reproducible on a local cluster (needs a network).
> Reporter: nkeywal
> Assignee: Elliott Clark
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Attachment: HBASE-7220-2.patch
I've got a new patch. Looks like the tests hit metrics when not in the region server. So I had to add a bunch of null checks
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Resolution: Fixed
Status: Resolved (was: Patch Available)
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch, HBASE-7220-2.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "nkeywal (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13504448#comment-13504448 ]
nkeywal commented on HBASE-7220:
--------------------------------
Thanks Elliott!
HRegionobjects are created also in the master before being given to regionservers. Do we need to create the region statistics in this case? There are already the coprocessors that are taking this into account in the constructor:
{code}
if (rsServices != null) {
this.rsAccounting = this.rsServices.getRegionServerAccounting();
// don't initialize coprocessors if not running within a regionserver
// TODO: revisit if coprocessors should load in other cases
this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
}
{code}
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Assigned] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark reassigned HBASE-7220:
------------------------------------
Assignee: Elliott Clark
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Environment: small test cluster. not reproducible on a local cluster (needs a network).
> Reporter: nkeywal
> Assignee: Elliott Clark
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (HBASE-7220) Creating a table with 3000 regions
on 2 nodes fails after 1 hour
Posted by "nkeywal (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13505822#comment-13505822 ]
nkeywal commented on HBASE-7220:
--------------------------------
Thanks, I will try locally as well
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (HBASE-7220) Creating a table with 3000 regions on
2 nodes fails after 1 hour
Posted by "Elliott Clark (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/HBASE-7220?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elliott Clark updated HBASE-7220:
---------------------------------
Attachment: HBASE-7220-1.patch
Added the code so that metricsRegion is null when not in a regionServer.
> Creating a table with 3000 regions on 2 nodes fails after 1 hour
> ----------------------------------------------------------------
>
> Key: HBASE-7220
> URL: https://issues.apache.org/jira/browse/HBASE-7220
> Project: HBase
> Issue Type: Bug
> Components: metrics, Performance, regionserver
> Affects Versions: 0.96.0
> Reporter: nkeywal
> Assignee: Elliott Clark
> Attachments: HBASE-7220-0.patch, HBASE-7220-1.patch
>
>
> I'm trying to create a table with 3000 regions on two regions servers, from the shell.
> It's ok on trunk a standalone config.
> It's ok on 0.94
> It's not ok on trunk: it fails after around 1 hour.
> If I remove all the code related to metrics in HRegion, the 3000 regions are created in 3 minutes (twice faster than the 0.94).
> On trunk, the region server spends its time in "waitForWork", while the master is in the tcp connection related code. It's a 1Gb network.
> I haven't looked at the metric code itself.
> Patch used to remove the metrics from HRegion:
> {noformat}
> index c70e9ab..6677e65 100644
> --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
> @@ -364,7 +364,7 @@ public class HRegion implements HeapSize { // , Writable{
> private HTableDescriptor htableDescriptor = null;
> private RegionSplitPolicy splitPolicy;
>
> - private final MetricsRegion metricsRegion;
> + private final MetricsRegion metricsRegion = null;
>
> /**
> * Should only be used for testing purposes
> @@ -388,7 +388,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.coprocessorHost = null;
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> }
>
> /**
> @@ -451,7 +451,7 @@ public class HRegion implements HeapSize { // , Writable{
> this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
> this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
>
> - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
> + //this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
>
> /*
> * timestamp.slop provides a server-side constraint on the timestamp. This
> @@ -1024,7 +1024,7 @@ public class HRegion implements HeapSize { // , Writable{
> status.setStatus("Running coprocessor post-close hooks");
> this.coprocessorHost.postClose(abort);
> }
> - this.metricsRegion.close();
> + //this.metricsRegion.close();
> status.markComplete("Closed");
> LOG.info("Closed " + this);
> return result;
> @@ -2331,11 +2331,11 @@ public class HRegion implements HeapSize { // , Writable{
> if (noOfPuts > 0) {
> // There were some Puts in the batch.
> double noOfMutations = noOfPuts + noOfDeletes;
> - this.metricsRegion.updatePut();
> + //this.metricsRegion.updatePut();
> }
> if (noOfDeletes > 0) {
> // There were some Deletes in the batch.
> - this.metricsRegion.updateDelete();
> + //this.metricsRegion.updateDelete();
> }
> if (!success) {
> for (int i = firstIndex; i < lastIndexExclusive; i++) {
> @@ -4270,7 +4270,7 @@ public class HRegion implements HeapSize { // , Writable{
>
> // do after lock
>
> - this.metricsRegion.updateGet();
> + //this.metricsRegion.updateGet();
>
> return results;
> }
> @@ -4657,7 +4657,7 @@ public class HRegion implements HeapSize { // , Writable{
> closeRegionOperation();
> }
>
> - this.metricsRegion.updateAppend();
> + //this.metricsRegion.updateAppend();
>
>
> if (flush) {
> @@ -4795,7 +4795,7 @@ public class HRegion implements HeapSize { // , Writable{
> mvcc.completeMemstoreInsert(w);
> }
> closeRegionOperation();
> - this.metricsRegion.updateIncrement();
> + //this.metricsRegion.updateIncrement();
> }
>
> if (flush) {
> {noformat}
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira