You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/05/09 18:01:33 UTC
svn commit: r1480705 - in /accumulo/trunk/server/src:
main/java/org/apache/accumulo/server/mini/
test/java/org/apache/accumulo/server/mini/
Author: kturner
Date: Thu May 9 16:01:33 2013
New Revision: 1480705
URL: http://svn.apache.org/r1480705
Log:
ACCUMULO-1357 applied patch from Corey J Nolet for MiniAccumulo that enable remote java debugging and allows users to configure memory size
Added:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java
Added: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java?rev=1480705&view=auto
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java (added)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MemoryUnit.java Thu May 9 16:01:33 2013
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+public enum MemoryUnit {
+
+ BYTE(""), KILOBYTE("K"), MEGABYTE("M"), GIGABYTE("G"), TERABYTE("T");
+
+ private final String printVal;
+
+ private MemoryUnit(String printVal) {
+ this.printVal = printVal;
+ }
+
+ public String getPrintVal(long memory) {
+ return memory + printVal;
+ }
+}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloCluster.java Thu May 9 16:01:33 2013
@@ -25,14 +25,19 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
+import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.tabletserver.TabletServer;
import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.start.Main;
import org.apache.zookeeper.server.ZooKeeperServerMain;
@@ -98,6 +103,8 @@ public class MiniAccumuloCluster {
private Process masterProcess;
private Process[] tabletServerProcesses;
+ private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
+
private File zooCfgFile;
private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
@@ -105,6 +112,10 @@ public class MiniAccumuloCluster {
private MiniAccumuloConfig config;
private Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ return exec(clazz, Collections.singletonList("-Xmx" + config.getDefaultMemorySize()), args);
+ }
+
+ private Process exec(Class<? extends Object> clazz, List<String> extraJvmOpts, String... args) throws IOException {
String javaHome = System.getProperty("java.home");
String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
String classpath = System.getProperty("java.class.path");
@@ -114,10 +125,9 @@ public class MiniAccumuloCluster {
String className = clazz.getCanonicalName();
ArrayList<String> argList = new ArrayList<String>();
-
- argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
- Main.class.getName(), className));
-
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath));
+ argList.addAll(extraJvmOpts);
+ argList.addAll(Arrays.asList("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", Main.class.getName(), className));
argList.addAll(Arrays.asList(args));
ProcessBuilder builder = new ProcessBuilder(argList);
@@ -146,6 +156,19 @@ public class MiniAccumuloCluster {
return process;
}
+ private Process exec(Class<? extends Object> clazz, ServerType serverType, String... args) throws IOException {
+
+ List<String> jvmOpts = new ArrayList<String>();
+ jvmOpts.add("-Xmx" + config.getMemoryConfig(serverType));
+
+ if (config.isDebug()) {
+ Integer port = PortUtils.getRandomFreePort();
+ jvmOpts.addAll(buildRemoteDebugParams(port));
+ debugPorts.add(new Pair<ServerType,Integer>(serverType, port));
+ }
+ return exec(clazz, jvmOpts, args);
+ }
+
/**
*
* @param dir
@@ -232,7 +255,7 @@ public class MiniAccumuloCluster {
}
});
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+ zooKeeperProcess = exec(Main.class, ServerType.ZOOKEEPER, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
// sleep a little bit to let zookeeper come up before calling init, seems to work better
UtilWaitThread.sleep(250);
@@ -245,16 +268,28 @@ public class MiniAccumuloCluster {
tabletServerProcesses = new Process[config.getNumTservers()];
for (int i = 0; i < config.getNumTservers(); i++) {
- tabletServerProcesses[i] = exec(TabletServer.class);
+ tabletServerProcesses[i] = exec(TabletServer.class, ServerType.TABLET_SERVER);
}
- masterProcess = exec(Master.class);
+ masterProcess = exec(Master.class, ServerType.MASTER);
+ }
+
+ private List<String> buildRemoteDebugParams(int port) {
+ return Arrays.asList(new String[] {"-Xdebug", String.format("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=%d", port)});
}
/**
- * @return Accumulo instance name
+ * @return generated remote debug ports if in debug mode.
+ *
+ * @since 1.6.0
*/
+ public Set<Pair<ServerType,Integer>> getDebugPorts() {
+ return debugPorts;
+ }
+ /**
+ * @return Accumulo instance name
+ */
public String getInstanceName() {
return config.getInstanceName();
}
@@ -262,7 +297,6 @@ public class MiniAccumuloCluster {
/**
* @return zookeeper connection string
*/
-
public String getZooKeepers() {
return config.getZooKeepers();
}
@@ -290,10 +324,8 @@ public class MiniAccumuloCluster {
}
/**
- *
* @since 1.6.0
*/
-
public MiniAccumuloConfig getConfig() {
return config;
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/MiniAccumuloConfig.java Thu May 9 16:01:33 2013
@@ -36,6 +36,9 @@ public class MiniAccumuloConfig {
private String rootPassword = null;
private Map<String,String> siteConfig = new HashMap<String,String>();
private int numTservers = 2;
+ private Map<ServerType,String> memoryConfig = new HashMap<ServerType,String>();
+
+ private boolean debug = false;
private String instanceName = "miniInstance";
@@ -47,12 +50,13 @@ public class MiniAccumuloConfig {
private File walogDir;
private Integer zooKeeperPort;
+ private String defaultMemorySize = "128M";
private boolean initialized = false;
/**
* @param dir
- * An empty or nonexistant directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava, and
+ * An empty or nonexistant directory that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava, and
* Junit provide methods for creating temporary directories.
* @param rootPassword
* The initial password for the Accumulo root user
@@ -108,9 +112,6 @@ public class MiniAccumuloConfig {
/**
* Set a given key/value on the site config if it doesn't already exist
- *
- * @param key
- * @param value
*/
private void mergeProp(String key, String value) {
if (!siteConfig.containsKey(key)) {
@@ -120,8 +121,6 @@ public class MiniAccumuloConfig {
/**
* Sets a given key with a random port for the value on the site config if it doesn't already exist.
- *
- * @param key
*/
private void mergePropWithRandomPort(String key) {
if (!siteConfig.containsKey(key)) {
@@ -177,6 +176,40 @@ public class MiniAccumuloConfig {
}
/**
+ * Sets the amount of memory to use in the master process. Calling this method is optional. Default memory is 128M
+ *
+ * @param serverType
+ * the type of server to apply the memory settings
+ * @param memory
+ * amount of memory to set
+ *
+ * @param memoryUnit
+ * the units for which to apply with the memory size
+ *
+ * @since 1.6.0
+ */
+ public MiniAccumuloConfig setMemory(ServerType serverType, long memory, MemoryUnit memoryUnit) {
+ this.memoryConfig.put(serverType, memoryUnit.getPrintVal(memory));
+ return this;
+ }
+
+ /**
+ * Sets the default memory size to use. This value is also used when a ServerType has not been configured explicitly. Calling this method is optional. Default
+ * memory is 128M
+ *
+ * @param memory
+ * amount of memory to set
+ *
+ * @memoryUnit the units for which to apply with the memory size
+ *
+ * @since 1.6.0
+ */
+ public MiniAccumuloConfig setDefaultMemorySize(long memory, MemoryUnit memoryUnit) {
+ this.defaultMemorySize = memoryUnit.getPrintVal(memory);
+ return this;
+ }
+
+ /**
* @return a copy of the site config
*/
public Map<String,String> getSiteConfig() {
@@ -225,6 +258,14 @@ public class MiniAccumuloConfig {
return walogDir;
}
+ String getMemoryConfig(ServerType serverType) {
+ return memoryConfig.containsKey(serverType) ? memoryConfig.get(serverType) : defaultMemorySize;
+ }
+
+ String getDefaultMemorySize() {
+ return defaultMemorySize;
+ }
+
/**
* @return zookeeper connection string
*
@@ -234,16 +275,46 @@ public class MiniAccumuloConfig {
return siteConfig.get(Property.INSTANCE_ZK_HOST.getKey());
}
+ /**
+ * @return the base directory of the cluster configuration
+ */
public File getDir() {
return dir;
}
+ /**
+ * @return the root password of this cluster configuration
+ */
public String getRootPassword() {
return rootPassword;
}
+ /**
+ * @return the number of tservers configured for this cluster
+ */
public int getNumTservers() {
return numTservers;
}
+ /**
+ * @return is the current configuration in debug mode?
+ *
+ * @since 1.6.0
+ */
+ public boolean isDebug() {
+ return debug;
+ }
+
+ /**
+ * @param debug
+ * should the processes run remote debug servers?
+ * @return the current instance
+ *
+ * @since 1.6.0
+ */
+ public MiniAccumuloConfig setDebug(boolean debug) {
+ this.debug = debug;
+ return this;
+ }
+
}
Added: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java?rev=1480705&view=auto
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java (added)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/mini/ServerType.java Thu May 9 16:01:33 2013
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.mini;
+
+public enum ServerType {
+ MASTER, ZOOKEEPER, TABLET_SERVER
+}
Modified: accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java (original)
+++ accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloClusterTest.java Thu May 9 16:01:33 2013
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.mini;
import java.io.File;
import java.util.Collections;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.UUID;
import org.apache.accumulo.core.client.BatchWriter;
@@ -37,7 +38,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.server.mini.MiniAccumuloCluster;
+import org.apache.accumulo.core.util.Pair;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -55,15 +56,12 @@ public class MiniAccumuloClusterTest {
@BeforeClass
public static void setupMiniCluster() throws Exception {
-
- folder.create();
-
Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
- accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret");
-
+ folder.create();
+ MiniAccumuloConfig config = new MiniAccumuloConfig(folder.getRoot(), "superSecret").setDebug(true);
+ accumulo = new MiniAccumuloCluster(config);
accumulo.start();
-
}
@Test(timeout = 30000)
@@ -184,10 +182,20 @@ public class MiniAccumuloClusterTest {
conn.tableOperations().delete("table2");
}
+ @Test(timeout = 10000)
+ public void testDebugPorts() {
+
+ Set<Pair<ServerType,Integer>> debugPorts = accumulo.getDebugPorts();
+ Assert.assertEquals(4, debugPorts.size());
+ for (Pair<ServerType,Integer> debugPort : debugPorts) {
+ Assert.assertTrue(debugPort.getSecond() > 0);
+ }
+ }
+
@AfterClass
public static void tearDownMiniCluster() throws Exception {
accumulo.stop();
- // folder.delete();
+ folder.delete();
}
}
Modified: accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java?rev=1480705&r1=1480704&r2=1480705&view=diff
==============================================================================
--- accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java (original)
+++ accumulo/trunk/server/src/test/java/org/apache/accumulo/server/mini/MiniAccumuloConfigTest.java Thu May 9 16:01:33 2013
@@ -60,6 +60,16 @@ public class MiniAccumuloConfigTest {
assertEquals("hdfs://", config.getSiteConfig().get(Property.INSTANCE_DFS_URI.getKey()));
}
+ @Test
+ public void testMemoryConfig() {
+
+ MiniAccumuloConfig config = new MiniAccumuloConfig(tempFolder.getRoot(), "password").initialize();
+ config.setDefaultMemorySize(128, MemoryUnit.MEGABYTE);
+ assertEquals("128M", config.getMemoryConfig(ServerType.MASTER));
+ config.setMemory(ServerType.MASTER, 256, MemoryUnit.MEGABYTE);
+ assertEquals("256M", config.getMemoryConfig(ServerType.MASTER));
+ }
+
@AfterClass
public static void tearDown() {
tempFolder.delete();