You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/12/19 07:37:25 UTC
[1/5] accumulo git commit: ACCUMULO-3440 Bin assingment log messages
Repository: accumulo
Updated Branches:
refs/heads/1.6 27d79c265 -> 59238d046
refs/heads/master 48a857ebe -> 4e8374562
ACCUMULO-3440 Bin assingment log messages
Don't spam log messages about assignments, but
also don't make huge log messages. Nice sized
log messages that are managable.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c558211e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c558211e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c558211e
Branch: refs/heads/1.6
Commit: c558211ea6cfcd38991e9e8a264773e73b0a2af6
Parents: 27d79c2
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 01:18:33 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 01:18:33 2014 -0500
----------------------------------------------------------------------
.../accumulo/master/TabletGroupWatcher.java | 28 ++++++++++++++++++--
1 file changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c558211e/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index f5ed941..72a96bd 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@ -87,7 +87,10 @@ import org.apache.thrift.TException;
import com.google.common.collect.Iterators;
class TabletGroupWatcher extends Daemon {
-
+ // Constants used to make sure assignment logging isn't excessive in quantity or size
+ private static final String ASSIGNMENT_BUFFER_SEPARATOR = ", ";
+ private static final int ASSINGMENT_BUFFER_MAX_LENGTH = 4096;
+
private final Master master;
final TabletStateStore store;
final TabletGroupWatcher dependentWatcher;
@@ -722,6 +725,7 @@ class TabletGroupWatcher extends Daemon {
if (!currentTServers.isEmpty()) {
Map<KeyExtent,TServerInstance> assignedOut = new HashMap<KeyExtent,TServerInstance>();
+ final StringBuilder builder = new StringBuilder(64);
this.master.tabletBalancer.getAssignments(Collections.unmodifiableSortedMap(currentTServers), Collections.unmodifiableMap(unassigned), assignedOut);
for (Entry<KeyExtent,TServerInstance> assignment : assignedOut.entrySet()) {
if (unassigned.containsKey(assignment.getKey())) {
@@ -730,13 +734,33 @@ class TabletGroupWatcher extends Daemon {
Master.log.warn("balancer assigned " + assignment.getKey() + " to a tablet server that is not current " + assignment.getValue() + " ignoring");
continue;
}
- Master.log.debug(store.name() + " assigning tablet " + assignment);
+
+ if (builder.length() > 0) {
+ builder.append(ASSIGNMENT_BUFFER_SEPARATOR);
+ }
+
+ builder.append(assignment);
+
+ // Don't let the log message get too gigantic
+ if (builder.length() > ASSINGMENT_BUFFER_MAX_LENGTH) {
+ builder.append("]");
+ Master.log.debug(store.name() + " assigning tablets: [" + builder.toString());
+ builder.setLength(0);
+ }
+
assignments.add(new Assignment(assignment.getKey(), assignment.getValue()));
}
} else {
Master.log.warn(store.name() + " load balancer assigning tablet that was not nominated for assignment " + assignment.getKey());
}
}
+
+ if (builder.length() > 0) {
+ // Make sure to log any leftover assignments
+ builder.append("]");
+ Master.log.debug(store.name() + " assigning tablets: [" + builder.toString());
+ }
+
if (!unassigned.isEmpty() && assignedOut.isEmpty())
Master.log.warn("Load balancer failed to assign any tablets");
}
[3/5] accumulo git commit: ACCUMULO-3435 Use RawLocalFileSystem
Posted by el...@apache.org.
ACCUMULO-3435 Use RawLocalFileSystem
Restarting the nodes and expecting recovery
to actually working with the ChecksumFileSystem
just ain't gonna happen. Checksum errors prevent recovery
from happening which means the test can never finish.
RawLocalFilesystem ensures that we have the necessary
filesystem semantics for WALs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/59238d04
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/59238d04
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/59238d04
Branch: refs/heads/master
Commit: 59238d046a39a859a28bc1a0ea79b8a93156e554
Parents: c558211
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 01:19:44 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 01:19:44 2014 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/test/functional/MetadataMaxFilesIT.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/59238d04/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
index be03d1e..b0da8b6 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
@@ -37,6 +37,7 @@ import org.apache.accumulo.harness.AccumuloClusterIT;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
import org.junit.After;
import org.junit.Before;
@@ -53,6 +54,7 @@ public class MetadataMaxFilesIT extends AccumuloClusterIT {
siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
siteConfig.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "10");
cfg.setSiteConfig(siteConfig);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
[2/5] accumulo git commit: ACCUMULO-3440 Bin assingment log messages
Posted by el...@apache.org.
ACCUMULO-3440 Bin assingment log messages
Don't spam log messages about assignments, but
also don't make huge log messages. Nice sized
log messages that are managable.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c558211e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c558211e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c558211e
Branch: refs/heads/master
Commit: c558211ea6cfcd38991e9e8a264773e73b0a2af6
Parents: 27d79c2
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 01:18:33 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 01:18:33 2014 -0500
----------------------------------------------------------------------
.../accumulo/master/TabletGroupWatcher.java | 28 ++++++++++++++++++--
1 file changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c558211e/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index f5ed941..72a96bd 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@ -87,7 +87,10 @@ import org.apache.thrift.TException;
import com.google.common.collect.Iterators;
class TabletGroupWatcher extends Daemon {
-
+ // Constants used to make sure assignment logging isn't excessive in quantity or size
+ private static final String ASSIGNMENT_BUFFER_SEPARATOR = ", ";
+ private static final int ASSINGMENT_BUFFER_MAX_LENGTH = 4096;
+
private final Master master;
final TabletStateStore store;
final TabletGroupWatcher dependentWatcher;
@@ -722,6 +725,7 @@ class TabletGroupWatcher extends Daemon {
if (!currentTServers.isEmpty()) {
Map<KeyExtent,TServerInstance> assignedOut = new HashMap<KeyExtent,TServerInstance>();
+ final StringBuilder builder = new StringBuilder(64);
this.master.tabletBalancer.getAssignments(Collections.unmodifiableSortedMap(currentTServers), Collections.unmodifiableMap(unassigned), assignedOut);
for (Entry<KeyExtent,TServerInstance> assignment : assignedOut.entrySet()) {
if (unassigned.containsKey(assignment.getKey())) {
@@ -730,13 +734,33 @@ class TabletGroupWatcher extends Daemon {
Master.log.warn("balancer assigned " + assignment.getKey() + " to a tablet server that is not current " + assignment.getValue() + " ignoring");
continue;
}
- Master.log.debug(store.name() + " assigning tablet " + assignment);
+
+ if (builder.length() > 0) {
+ builder.append(ASSIGNMENT_BUFFER_SEPARATOR);
+ }
+
+ builder.append(assignment);
+
+ // Don't let the log message get too gigantic
+ if (builder.length() > ASSINGMENT_BUFFER_MAX_LENGTH) {
+ builder.append("]");
+ Master.log.debug(store.name() + " assigning tablets: [" + builder.toString());
+ builder.setLength(0);
+ }
+
assignments.add(new Assignment(assignment.getKey(), assignment.getValue()));
}
} else {
Master.log.warn(store.name() + " load balancer assigning tablet that was not nominated for assignment " + assignment.getKey());
}
}
+
+ if (builder.length() > 0) {
+ // Make sure to log any leftover assignments
+ builder.append("]");
+ Master.log.debug(store.name() + " assigning tablets: [" + builder.toString());
+ }
+
if (!unassigned.isEmpty() && assignedOut.isEmpty())
Master.log.warn("Load balancer failed to assign any tablets");
}
[5/5] accumulo git commit: Merge branch '1.6'
Posted by el...@apache.org.
Merge branch '1.6'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4e837456
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4e837456
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4e837456
Branch: refs/heads/master
Commit: 4e83745627c38842935305fb177f85cbec327de8
Parents: 48a857e 59238d0
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 01:33:42 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 01:33:42 2014 -0500
----------------------------------------------------------------------
.../accumulo/master/TabletGroupWatcher.java | 28 ++++++++++++++++++--
.../test/functional/MetadataMaxFilesIT.java | 2 ++
2 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e837456/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e837456/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
index b2adc89,b0da8b6..ed08cad
--- a/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
@@@ -36,24 -32,31 +36,26 @@@ import org.apache.accumulo.core.master.
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.trace.Tracer;
import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.harness.AccumuloClusterIT;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.server.util.Admin;
import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MetadataMaxFilesIT extends AccumuloClusterIT {
- private static final Logger log = LoggerFactory.getLogger(MetadataMaxFilesIT.class);
+public class MetadataMaxFilesIT extends ConfigurableMacIT {
+
@Override
- public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> siteConfig = new HashMap<String,String>();
siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
siteConfig.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "10");
cfg.setSiteConfig(siteConfig);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
-
+
@Override
protected int defaultTimeoutSeconds() {
return 4 * 60;
[4/5] accumulo git commit: ACCUMULO-3435 Use RawLocalFileSystem
Posted by el...@apache.org.
ACCUMULO-3435 Use RawLocalFileSystem
Restarting the nodes and expecting recovery
to actually working with the ChecksumFileSystem
just ain't gonna happen. Checksum errors prevent recovery
from happening which means the test can never finish.
RawLocalFilesystem ensures that we have the necessary
filesystem semantics for WALs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/59238d04
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/59238d04
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/59238d04
Branch: refs/heads/1.6
Commit: 59238d046a39a859a28bc1a0ea79b8a93156e554
Parents: c558211
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 01:19:44 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 01:19:44 2014 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/test/functional/MetadataMaxFilesIT.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/59238d04/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
index be03d1e..b0da8b6 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java
@@ -37,6 +37,7 @@ import org.apache.accumulo.harness.AccumuloClusterIT;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
import org.junit.After;
import org.junit.Before;
@@ -53,6 +54,7 @@ public class MetadataMaxFilesIT extends AccumuloClusterIT {
siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
siteConfig.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "10");
cfg.setSiteConfig(siteConfig);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override