You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2014/01/09 22:28:23 UTC
[1/3] git commit: ACCUMULO-2163 made MAC always run GC
Updated Branches:
refs/heads/1.6.0-SNAPSHOT 2d51e329e -> 77b450724
ACCUMULO-2163 made MAC always run GC
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/96bfa8ca
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/96bfa8ca
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/96bfa8ca
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 96bfa8cad75b5f41bbf9dcf4790bc6677c36dce8
Parents: 9abb725
Author: Keith Turner <kt...@apache.org>
Authored: Thu Jan 9 14:36:33 2014 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Thu Jan 9 15:47:33 2014 -0500
----------------------------------------------------------------------
.../minicluster/MiniAccumuloCluster.java | 8 +++---
.../minicluster/MiniAccumuloConfig.java | 17 ------------
.../minicluster/MiniAccumuloClusterGCTest.java | 27 --------------------
3 files changed, 4 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/96bfa8ca/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 7631f27..cce422c 100644
--- a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -244,7 +244,9 @@ public class MiniAccumuloCluster {
appendProp(fileWriter, Property.TRACE_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
appendProp(fileWriter, Property.LOGGER_SORT_BUFFER_SIZE, "50M", siteConfig);
appendProp(fileWriter, Property.LOGGER_PORTSEARCH, "true", siteConfig);
- appendProp(fileWriter, Property.GC_CYCLE_DELAY, "30s", siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_DELAY, "4s", siteConfig);
+ appendProp(fileWriter, Property.GC_CYCLE_START, "0s", siteConfig);
+ appendProp(fileWriter, Property.GC_PORT, "" + PortUtils.getRandomFreePort(), siteConfig);
// since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
@@ -324,9 +326,7 @@ public class MiniAccumuloCluster {
loggerProcess = exec(LogService.class);
masterProcess = exec(Master.class);
- if (config.shouldRunGC()) {
- gcProcess = exec(SimpleGarbageCollector.class);
- }
+ gcProcess = exec(SimpleGarbageCollector.class);
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/96bfa8ca/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 1efd0d4..c9548a6 100644
--- a/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/src/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -32,7 +32,6 @@ public class MiniAccumuloConfig {
private String rootPassword = null;
private Map<String,String> siteConfig = Collections.emptyMap();
private int numTservers = 2;
- private boolean runGC = false;
/**
* @param dir
@@ -87,20 +86,4 @@ public class MiniAccumuloConfig {
this.siteConfig = siteConfig;
return this;
}
-
- /**
- * Whether or not the Accumulo garbage collector proces will run
- */
- public boolean shouldRunGC() {
- return runGC;
- }
-
- /**
- * Sets if the Accumulo garbage collector process should run
- *
- * @param shouldRunGC
- */
- public void runGC(boolean shouldRunGC) {
- runGC = shouldRunGC;
- }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/96bfa8ca/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
index c32e719..50f3f55 100644
--- a/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
+++ b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
@@ -32,39 +32,15 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Ignore;
-import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Files;
/**
*
*/
public class MiniAccumuloClusterGCTest {
- @Test
- public void testGcConfig() throws Exception {
- File f = Files.createTempDir();
- f.deleteOnExit();
- try {
- MiniAccumuloConfig macConfig = new MiniAccumuloConfig(f, passwd);
- macConfig.setNumTservers(1);
-
- Assert.assertEquals(false, macConfig.shouldRunGC());
-
- // Turn on the garbage collector
- macConfig.runGC(true);
-
- Assert.assertEquals(true, macConfig.shouldRunGC());
- } finally {
- if (null != f && f.exists()) {
- f.delete();
- }
- }
- }
-
-
private static TemporaryFolder tmpDir = new TemporaryFolder();
private static MiniAccumuloConfig macConfig;
private static MiniAccumuloCluster accumulo;
@@ -77,9 +53,6 @@ public class MiniAccumuloClusterGCTest {
macConfig = new MiniAccumuloConfig(tmpDir.getRoot(), passwd);
macConfig.setNumTservers(1);
- // Turn on the garbage collector
- macConfig.runGC(true);
-
String gcPort = Integer.toString(PortUtils.getRandomFreePort());
// And tweak the settings to make it run often
[3/3] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by kt...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/77b45072
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/77b45072
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/77b45072
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 77b45072438b2563c7833dba9b68c08766fea0cd
Parents: 2d51e32 d0f626d
Author: Keith Turner <kt...@apache.org>
Authored: Thu Jan 9 16:32:36 2014 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Thu Jan 9 16:32:36 2014 -0500
----------------------------------------------------------------------
.../minicluster/MiniAccumuloCluster.java | 4 +---
.../minicluster/MiniAccumuloConfig.java | 20 ++--------------
.../minicluster/MiniAccumuloClusterGCTest.java | 25 --------------------
.../minicluster/MiniAccumuloClusterTest.java | 4 ++--
4 files changed, 5 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/77b45072/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 2ed2434,7c618be..09af10b
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@@ -414,149 -289,47 +414,147 @@@ public class MiniAccumuloCluster
* @throws IllegalStateException
* if already started
*/
-
public void start() throws IOException, InterruptedException {
- if (zooKeeperProcess != null)
- throw new IllegalStateException("Already started");
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
+
+ if (!initialized) {
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+
+ if (zooKeeperProcess == null) {
+ zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
+ }
+
+ if (!initialized) {
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ long startTime = System.currentTimeMillis();
+ while (true) {
try {
- MiniAccumuloCluster.this.stop();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
+ Socket s = new Socket("localhost", config.getZooKeeperPort());
+ s.getOutputStream().write("ruok\n".getBytes());
+ s.getOutputStream().flush();
+ byte buffer[] = new byte[100];
+ int n = s.getInputStream().read(buffer);
+ if (n == 4 && new String(buffer, 0, n).equals("imok"))
+ break;
+ } catch (Exception e) {
+ if (System.currentTimeMillis() - startTime >= 10000) {
+ throw new RuntimeException("Zookeeper did not start within 10 seconds . Check the logs in " + config.getLogDir() + " for errors.");
+ }
+ UtilWaitThread.sleep(250);
}
}
- });
-
- zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
-
- // sleep a little bit to let zookeeper come up before calling init, seems to work better
- UtilWaitThread.sleep(250);
-
- Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password", config.getRootPassword());
- int ret = initProcess.waitFor();
+ Process initProcess = exec(Initialize.class, "--instance-name", config.getInstanceName(), "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
+ }
+ initialized = true;
+ }
+ synchronized (tabletServerProcesses) {
+ for (int i = tabletServerProcesses.size(); i < config.getNumTservers(); i++) {
+ tabletServerProcesses.add(_exec(TabletServer.class, ServerType.TABLET_SERVER));
+ }
+ }
+ int ret = 0;
+ for (int i = 0; i < 5; i++) {
+ ret = exec(Main.class, SetGoalState.class.getName(), MasterGoalState.NORMAL.toString()).waitFor();
+ if (ret == 0)
+ break;
+ UtilWaitThread.sleep(1000);
+ }
if (ret != 0) {
- throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + logDir + " for errors.");
+ throw new RuntimeException("Could not set master goal state, process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
}
-
- tabletServerProcesses = new Process[config.getNumTservers()];
- for (int i = 0; i < config.getNumTservers(); i++) {
- tabletServerProcesses[i] = exec(TabletServer.class);
+ if (masterProcess == null) {
+ masterProcess = _exec(Master.class, ServerType.MASTER);
}
- if (config.shouldRunGC()) {
- gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
- }
-
- masterProcess = exec(Master.class);
-
- gcProcess = exec(SimpleGarbageCollector.class);
++ gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
}
-
+
+ private List<String> buildRemoteDebugParams(int port) {
+ return Arrays.asList(new String[] {"-Xdebug", String.format("-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=%d", port)});
+ }
+
+ /**
+ * @return generated remote debug ports if in debug mode.
+ * @since 1.6.0
+ */
+ public Set<Pair<ServerType,Integer>> getDebugPorts() {
+ return debugPorts;
+ }
+
+ List<ProcessReference> references(Process... procs) {
+ List<ProcessReference> result = new ArrayList<ProcessReference>();
+ for (Process proc : procs) {
+ result.add(new ProcessReference(proc));
+ }
+ return result;
+ }
+
+ public Map<ServerType,Collection<ProcessReference>> getProcesses() {
+ Map<ServerType,Collection<ProcessReference>> result = new HashMap<ServerType,Collection<ProcessReference>>();
+ result.put(ServerType.MASTER, references(masterProcess));
+ result.put(ServerType.TABLET_SERVER, references(tabletServerProcesses.toArray(new Process[0])));
+ result.put(ServerType.ZOOKEEPER, references(zooKeeperProcess));
+ if (null != gcProcess) {
+ result.put(ServerType.GARBAGE_COLLECTOR, references(gcProcess));
+ }
+ return result;
+ }
+
+ public void killProcess(ServerType type, ProcessReference proc) throws ProcessNotFoundException, InterruptedException {
+ boolean found = false;
+ switch (type) {
+ case MASTER:
+ if (proc.equals(masterProcess)) {
+ masterProcess.destroy();
+ masterProcess = null;
+ found = true;
+ }
+ break;
+ case TABLET_SERVER:
+ synchronized (tabletServerProcesses) {
+ for (Process tserver : tabletServerProcesses) {
+ if (proc.equals(tserver)) {
+ tabletServerProcesses.remove(tserver);
+ tserver.destroy();
+ found = true;
+ break;
+ }
+ }
+ }
+ break;
+ case ZOOKEEPER:
+ if (proc.equals(zooKeeperProcess)) {
+ zooKeeperProcess.destroy();
+ zooKeeperProcess = null;
+ found = true;
+ }
+ break;
+ case GARBAGE_COLLECTOR:
+ if (proc.equals(gcProcess)) {
+ gcProcess.destroy();
+ gcProcess = null;
+ found = true;
+ }
+ break;
+ }
+ if (!found)
+ throw new ProcessNotFoundException();
+ }
+
/**
* @return Accumulo instance name
*/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/77b45072/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 2326f95,17ab03a..7b410e5
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@@ -30,38 -25,13 +30,37 @@@ import org.apache.accumulo.server.util.
*
* @since 1.5.0
*/
-
public class MiniAccumuloConfig {
+ private static final String DEFAULT_INSTANCE_SECRET = "DONTTELL";
+
private File dir = null;
private String rootPassword = null;
- private Map<String,String> siteConfig = Collections.emptyMap();
+ private Map<String,String> siteConfig = new HashMap<String,String>();
private int numTservers = 2;
+ private Map<ServerType,Long> memoryConfig = new HashMap<ServerType,Long>();
+ private boolean jdwpEnabled = false;
+ private Map<String,String> systemProperties = new HashMap<String,String>();
+
+ private String instanceName = "miniInstance";
+
+ private File libDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File logDir;
+ private File walogDir;
+
+ private Integer zooKeeperPort;
+ private long defaultMemorySize = 128 * 1024 * 1024;
+
+ private boolean initialized = false;
+
+ private boolean useMiniDFS = false;
- private boolean runGC = false;
+
+ private String[] classpathItems = null;
+
+ private String[] nativePathItems = null;
/**
* @param dir
@@@ -75,79 -46,16 +74,80 @@@
this.rootPassword = rootPassword;
}
- public File getDir() {
- return dir;
+ /**
+ * Set directories and fully populate site config
+ */
+ MiniAccumuloConfig initialize() {
+
+ // Sanity checks
+ if (this.getDir().exists() && !this.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + this.getDir() + " is a file");
+
+ if (this.getDir().exists() && this.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + this.getDir() + " is not empty");
+
+ if (!initialized) {
+ libDir = new File(dir, "lib");
+ confDir = new File(dir, "conf");
+ accumuloDir = new File(dir, "accumulo");
+ zooKeeperDir = new File(dir, "zookeeper");
+ logDir = new File(dir, "logs");
+ walogDir = new File(dir, "walogs");
+
+ String[] paths = {"$ACCUMULO_HOME/lib/.*.jar", "$ZOOKEEPER_HOME/zookeeper[^.].*.jar", "$HADOOP_PREFIX/[^.].*.jar", "$HADOOP_PREFIX/lib/[^.].*.jar",
+ "$HADOOP_PREFIX/share/hadoop/common/.*.jar", "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar", "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar",
+ "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar"};
+
+ String classpath = StringUtil.join(Arrays.asList(paths), ",");
+
+ mergeProp(Property.INSTANCE_DFS_URI.getKey(), "file:///");
+ mergeProp(Property.INSTANCE_DFS_DIR.getKey(), accumuloDir.getAbsolutePath());
+ mergeProp(Property.INSTANCE_SECRET.getKey(), DEFAULT_INSTANCE_SECRET);
+ mergeProp(Property.TSERV_PORTSEARCH.getKey(), "true");
+ mergeProp(Property.LOGGER_DIR.getKey(), walogDir.getAbsolutePath());
+ mergeProp(Property.TSERV_DATACACHE_SIZE.getKey(), "10M");
+ mergeProp(Property.TSERV_INDEXCACHE_SIZE.getKey(), "10M");
+ mergeProp(Property.TSERV_MAXMEM.getKey(), "50M");
+ mergeProp(Property.TSERV_WALOG_MAX_SIZE.getKey(), "100M");
+ mergeProp(Property.TSERV_NATIVEMAP_ENABLED.getKey(), "false");
+ mergeProp(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + ".password", getRootPassword());
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ mergeProp(Property.TSERV_MAJC_DELAY.getKey(), "3");
+ mergeProp(Property.GENERAL_CLASSPATHS.getKey(), classpath);
+ mergeProp(Property.GENERAL_DYNAMIC_CLASSPATHS.getKey(), libDir.getAbsolutePath() + "/[^.].*[.]jar");
- mergeProp(Property.GC_CYCLE_DELAY.getKey(), "30s");
++ mergeProp(Property.GC_CYCLE_DELAY.getKey(), "4s");
++ mergeProp(Property.GC_CYCLE_START.getKey(), "0s");
+ mergePropWithRandomPort(Property.MASTER_CLIENTPORT.getKey());
+ mergePropWithRandomPort(Property.TRACE_PORT.getKey());
+ mergePropWithRandomPort(Property.TSERV_CLIENTPORT.getKey());
+ mergePropWithRandomPort(Property.MONITOR_PORT.getKey());
+ mergePropWithRandomPort(Property.GC_PORT.getKey());
+
+ // zookeeper port should be set explicitly in this class, not just on the site config
+ if (zooKeeperPort == null)
+ zooKeeperPort = PortUtils.getRandomFreePort();
+ siteConfig.put(Property.INSTANCE_ZK_HOST.getKey(), "localhost:" + zooKeeperPort);
+ initialized = true;
+ }
+ return this;
}
- public String getRootPassword() {
- return rootPassword;
+ /**
+ * Set a given key/value on the site config if it doesn't already exist
+ */
+ private void mergeProp(String key, String value) {
+ if (!siteConfig.containsKey(key)) {
+ siteConfig.put(key, value);
+ }
}
- public int getNumTservers() {
- return numTservers;
+ /**
+ * Sets a given key with a random port for the value on the site config if it doesn't already exist.
+ */
+ private void mergePropWithRandomPort(String key) {
+ if (!siteConfig.containsKey(key)) {
+ siteConfig.put(key, "0");
+ }
}
/**
@@@ -174,284 -77,14 +174,268 @@@
}
/**
- * Calling this method is optional. If not set, it defautls to an empty map.
+ * Calling this method is optional. If not set, it defaults to an empty map.
*
* @param siteConfig
- * key/values that you normally put in accumulo-site.xml can be put here
+ * key/values that you normally put in accumulo-site.xml can be put here.
*/
-
public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
- this.siteConfig = siteConfig;
+ this.siteConfig = new HashMap<String,String>(siteConfig);
+ return this;
+ }
+
+ /**
+ * Calling this method is optional. A random port is generated by default
+ *
+ * @param zooKeeperPort
+ * A valid (and unused) port to use for the zookeeper
+ *
+ * @since 1.6.0
+ */
+ public MiniAccumuloConfig setZooKeeperPort(int zooKeeperPort) {
+ this.zooKeeperPort = zooKeeperPort;
+ return this;
+ }
+
+ /**
+ * Sets the amount of memory to use in the master process. Calling this method is optional. Default memory is 128M
+ *
+ * @param serverType
+ * the type of server to apply the memory settings
+ * @param memory
+ * amount of memory to set
+ *
+ * @param memoryUnit
+ * the units for which to apply with the memory size
+ *
+ * @since 1.6.0
+ */
+ public MiniAccumuloConfig setMemory(ServerType serverType, long memory, MemoryUnit memoryUnit) {
+ this.memoryConfig.put(serverType, memoryUnit.toBytes(memory));
+ return this;
+ }
+
+ /**
+ * Sets the default memory size to use. This value is also used when a ServerType has not been configured explicitly. Calling this method is optional. Default
+ * memory is 128M
+ *
+ * @param memory
+ * amount of memory to set
+ *
+ * @param memoryUnit
+ * the units for which to apply with the memory size
+ *
+ * @since 1.6.0
+ */
+ public MiniAccumuloConfig setDefaultMemory(long memory, MemoryUnit memoryUnit) {
+ this.defaultMemorySize = memoryUnit.toBytes(memory);
+ return this;
+ }
+
+ /**
+ * @return a copy of the site config
+ */
+ public Map<String,String> getSiteConfig() {
+ return new HashMap<String,String>(siteConfig);
+ }
+
+ /**
+ * @return name of configured instance
+ *
+ * @since 1.6.0
+ */
+ public String getInstanceName() {
+ return instanceName;
+ }
+
+ /**
+ * @return The configured zookeeper port
+ *
+ * @since 1.6.0
+ */
+ public int getZooKeeperPort() {
+ return zooKeeperPort;
+ }
+
+ File getLibDir() {
+ return libDir;
+ }
+
+ File getConfDir() {
+ return confDir;
+ }
+
+ File getZooKeeperDir() {
+ return zooKeeperDir;
+ }
+
+ public File getAccumuloDir() {
+ return accumuloDir;
+ }
+
+ public File getLogDir() {
+ return logDir;
+ }
+
+ File getWalogDir() {
+ return walogDir;
+ }
+
+ /**
+ * @param serverType
+ * get configuration for this server type
+ *
+ * @return memory configured in bytes, returns default if this server type is not configured
+ *
+ * @since 1.6.0
+ */
+ public long getMemory(ServerType serverType) {
+ return memoryConfig.containsKey(serverType) ? memoryConfig.get(serverType) : defaultMemorySize;
+ }
+
+ /**
+ * @return memory configured in bytes
+ *
+ * @since 1.6.0
+ */
+ public long getDefaultMemory() {
+ return defaultMemorySize;
+ }
+
+ /**
+ * @return zookeeper connection string
+ *
+ * @since 1.6.0
+ */
+ public String getZooKeepers() {
+ return siteConfig.get(Property.INSTANCE_ZK_HOST.getKey());
+ }
+
+ /**
+ * @return the base directory of the cluster configuration
+ */
+ public File getDir() {
+ return dir;
+ }
+
+ /**
+ * @return the root password of this cluster configuration
+ */
+ public String getRootPassword() {
+ return rootPassword;
+ }
+
+ /**
+ * @return the number of tservers configured for this cluster
+ */
+ public int getNumTservers() {
+ return numTservers;
+ }
+
+ /**
+ * @return is the current configuration in jdwpEnabled mode?
+ *
+ * @since 1.6.0
+ */
+ public boolean isJDWPEnabled() {
+ return jdwpEnabled;
+ }
+
+ /**
+ * @param jdwpEnabled
+ * should the processes run remote jdwpEnabled servers?
+ * @return the current instance
+ *
+ * @since 1.6.0
+ */
+ public MiniAccumuloConfig setJDWPEnabled(boolean jdwpEnabled) {
+ this.jdwpEnabled = jdwpEnabled;
return this;
}
+
+ public boolean useMiniDFS() {
+ return useMiniDFS;
+ }
+
+ public void useMiniDFS(boolean useMiniDFS) {
+ this.useMiniDFS = useMiniDFS;
+ }
+
+ /**
+ * @return location of client conf file containing connection parameters for connecting to this minicluster
+ *
+ * @since 1.6.0
+ */
+ public File getClientConfFile() {
+ return new File(getConfDir(), "client.conf");
+ }
+
+ /**
- * Whether or not the Accumulo garbage collector proces will run
- */
- public boolean shouldRunGC() {
- return runGC;
- }
-
- /**
- * Sets if the Accumulo garbage collector process should run
- *
- * @param shouldRunGC
- */
- public void runGC(boolean shouldRunGC) {
- runGC = shouldRunGC;
- }
-
- /**
+ * sets system properties set for service processes
+ *
+ * @since 1.6.0
+ */
+ public void setSystemProperties(Map<String,String> systemProperties) {
+ this.systemProperties = new HashMap<String,String>(systemProperties);
+ }
+
+ /**
+ * @return a copy of the system properties for service processes
+ *
+ * @since 1.6.0
+ */
+ public Map<String,String> getSystemProperties() {
+ return new HashMap<String,String>(systemProperties);
+ }
+
+ /**
+ * Gets the classpath elements to use when spawning processes.
+ *
+ * @return the classpathItems, if set
+ *
+ * @since 1.6.0
+ */
+ public String[] getClasspathItems() {
+ return classpathItems;
+ }
+
+ /**
+ * Sets the classpath elements to use when spawning processes.
+ *
+ * @param classpathItems
+ * the classpathItems to set
+ * @since 1.6.0
+ */
+ public void setClasspathItems(String... classpathItems) {
+ this.classpathItems = classpathItems;
+ }
+
+ /**
+ * @return the paths to use for loading native libraries
+ *
+ * @since 1.6.0
+ */
+ public String[] getNativeLibPaths() {
+ return this.nativePathItems == null ? new String[0] : this.nativePathItems;
+ }
+
+ /**
+ * Sets the path for processes to use for loading native libraries
+ *
+ * @param nativePathItems
+ * the nativePathItems to set
+ * @since 1.6.0
+ */
+ public void setNativeLibPaths(String... nativePathItems) {
+ this.nativePathItems = nativePathItems;
+ }
+
+ /**
+ * Sets arbitrary configuration properties.
+ *
+ * @since 1.6.0
+ */
+ public void setProperty(Property p, String value) {
+ this.siteConfig.put(p.getKey(), value);
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/77b45072/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
----------------------------------------------------------------------
diff --cc minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
index 86a8950,318f0af..8a01f99
--- a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
+++ b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
@@@ -33,63 -32,35 +33,38 @@@ import org.apache.commons.io.filefilter
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Ignore;
-import org.junit.rules.TemporaryFolder;
+import org.junit.Test;
import com.google.common.collect.ImmutableMap;
- import com.google.common.io.Files;
/**
*
*/
public class MiniAccumuloClusterGCTest {
+ private static final Logger log = Logger.getLogger(MiniAccumuloClusterGCTest.class);
- @Test
- public void testGcConfig() throws Exception {
- File f = Files.createTempDir();
- f.deleteOnExit();
- try {
- MiniAccumuloConfig macConfig = new MiniAccumuloConfig(f, passwd);
- macConfig.setNumTservers(1);
-
- Assert.assertEquals(false, macConfig.shouldRunGC());
-
- // Turn on the garbage collector
- macConfig.runGC(true);
-
- Assert.assertEquals(true, macConfig.shouldRunGC());
- } finally {
- if (f.exists()) {
- f.delete();
- }
- }
- }
-
- private static TemporaryFolder tmpDir = new TemporaryFolder();
+ private static File testDir = new File(System.getProperty("user.dir") + "/target/" + MiniAccumuloClusterGCTest.class.getName());
private static MiniAccumuloConfig macConfig;
private static MiniAccumuloCluster accumulo;
private static final String passwd = "password";
+ @BeforeClass
public static void setupMiniCluster() throws Exception {
- tmpDir.create();
+ FileUtils.deleteQuietly(testDir);
+ testDir.mkdir();
Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
-
- macConfig = new MiniAccumuloConfig(tmpDir.getRoot(), passwd);
+
+ macConfig = new MiniAccumuloConfig(testDir, passwd);
macConfig.setNumTservers(1);
-
- String gcPort = Integer.toString(PortUtils.getRandomFreePort());
-
+
- // Turn on the garbage collector
- macConfig.runGC(true);
-
// And tweak the settings to make it run often
- Map<String,String> config = ImmutableMap.of(Property.GC_CYCLE_DELAY.getKey(), "1s", Property.GC_CYCLE_START.getKey(), "0s", Property.GC_PORT.getKey(), gcPort);
+ Map<String,String> config = ImmutableMap.of(Property.GC_CYCLE_DELAY.getKey(), "1s", Property.GC_CYCLE_START.getKey(), "0s");
macConfig.setSiteConfig(config);
-
+
accumulo = new MiniAccumuloCluster(macConfig);
accumulo.start();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/77b45072/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
----------------------------------------------------------------------
diff --cc minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
index 01b3f54,75ae808..fe6f5ed
--- a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
+++ b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
@@@ -193,40 -173,14 +193,40 @@@ public class MiniAccumuloClusterTest
Assert.assertFalse(entry.getKey().getRowData().toString().toLowerCase().contains("foo"));
count++;
}
-
+
Assert.assertEquals(2, count);
-
+
conn.instanceOperations().removeProperty(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
conn.tableOperations().delete("table2");
-
}
-
+
+ @Test(timeout = 10000)
+ public void testDebugPorts() {
+
+ Set<Pair<ServerType,Integer>> debugPorts = accumulo.getDebugPorts();
- Assert.assertEquals(4, debugPorts.size());
++ Assert.assertEquals(5, debugPorts.size());
+ for (Pair<ServerType,Integer> debugPort : debugPorts) {
+ Assert.assertTrue(debugPort.getSecond() > 0);
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testAccurateProcessListReturned() throws Exception {
+ Map<ServerType,Collection<ProcessReference>> procs = accumulo.getProcesses();
+
- Assert.assertFalse(procs.containsKey(ServerType.GARBAGE_COLLECTOR));
++ Assert.assertTrue(procs.containsKey(ServerType.GARBAGE_COLLECTOR));
+
+ for (ServerType t : new ServerType[] {ServerType.MASTER, ServerType.TABLET_SERVER, ServerType.ZOOKEEPER}) {
+ Assert.assertTrue(procs.containsKey(t));
+ Collection<ProcessReference> procRefs = procs.get(t);
+ Assert.assertTrue(1 <= procRefs.size());
+
+ for (ProcessReference procRef : procRefs) {
+ Assert.assertNotNull(procRef);
+ }
+ }
+ }
+
@AfterClass
public static void tearDownMiniCluster() throws Exception {
accumulo.stop();
[2/3] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by kt...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d0f626d9
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d0f626d9
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d0f626d9
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: d0f626d9670415d9788ae1c0af33d1b931484f73
Parents: 1413ebc 96bfa8c
Author: Keith Turner <kt...@apache.org>
Authored: Thu Jan 9 16:04:40 2014 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Thu Jan 9 16:04:40 2014 -0500
----------------------------------------------------------------------
.../minicluster/MiniAccumuloCluster.java | 7 +++--
.../minicluster/MiniAccumuloConfig.java | 17 ------------
.../minicluster/MiniAccumuloClusterGCTest.java | 27 --------------------
3 files changed, 3 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d0f626d9/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index d12c05a,0000000..7c618be
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@@ -1,375 -1,0 +1,374 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.gc.SimpleGarbageCollector;
+import org.apache.accumulo.server.master.Master;
+import org.apache.accumulo.server.tabletserver.TabletServer;
+import org.apache.accumulo.server.util.Initialize;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.start.Main;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+
+/**
+ * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test
+ * code against a real Accumulo instance. Its much more accurate for testing than MockAccumulo, but much slower than MockAccumulo.
+ *
+ * @since 1.5.0
+ */
+public class MiniAccumuloCluster {
+
+ private static final String INSTANCE_SECRET = "DONTTELL";
+ private static final String INSTANCE_NAME = "miniInstance";
+
+ private static class LogWriter extends Thread {
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ /**
+ * @throws IOException
+ */
+ public LogWriter(InputStream stream, File logFile) throws IOException {
+ this.setDaemon(true);
+ this.in = new BufferedReader(new InputStreamReader(stream));
+ out = new BufferedWriter(new FileWriter(logFile));
+
+ SimpleTimer.getInstance().schedule(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 1000, 1000);
+ }
+
+ public synchronized void flush() throws IOException {
+ if (out != null)
+ out.flush();
+ }
+
+ @Override
+ public void run() {
+ String line;
+
+ try {
+ while ((line = in.readLine()) != null) {
+ out.append(line);
+ out.append("\n");
+ }
+
+ synchronized (this) {
+ out.close();
+ out = null;
+ in.close();
+ }
+
+ } catch (IOException e) {}
+ }
+ }
+
+ private File libDir;
+ private File confDir;
+ private File zooKeeperDir;
+ private File accumuloDir;
+ private File zooCfgFile;
+ private File logDir;
+ private File walogDir;
+
+ private Process zooKeeperProcess;
+ private Process masterProcess;
+ private Process gcProcess;
+
+ private int zooKeeperPort;
+
+ private List<LogWriter> logWriters = new ArrayList<MiniAccumuloCluster.LogWriter>();
+
+ private MiniAccumuloConfig config;
+ private Process[] tabletServerProcesses;
+
+ private Process exec(Class<? extends Object> clazz, String... args) throws IOException {
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+
+ classpath = confDir.getAbsolutePath() + File.pathSeparator + classpath;
+
+ String className = clazz.getCanonicalName();
+
+ ArrayList<String> argList = new ArrayList<String>();
+
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath, "-Xmx128m", "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
+ "-Dapple.awt.UIElement=true", Main.class.getName(), className));
+
+ argList.addAll(Arrays.asList(args));
+
+ ProcessBuilder builder = new ProcessBuilder(argList);
+
+ builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
+ builder.environment().put("ACCUMULO_LOG_DIR", logDir.getAbsolutePath());
+
+ // if we're running under accumulo.start, we forward these env vars
+ String env = System.getenv("HADOOP_PREFIX");
+ if (env != null)
+ builder.environment().put("HADOOP_PREFIX", env);
+ env = System.getenv("ZOOKEEPER_HOME");
+ if (env != null)
+ builder.environment().put("ZOOKEEPER_HOME", env);
+
+ Process process = builder.start();
+
+ LogWriter lw;
+ lw = new LogWriter(process.getErrorStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".err"));
+ logWriters.add(lw);
+ lw.start();
+ lw = new LogWriter(process.getInputStream(), new File(logDir, clazz.getSimpleName() + "_" + process.hashCode() + ".out"));
+ logWriters.add(lw);
+ lw.start();
+
+ return process;
+ }
+
+ private void appendProp(FileWriter fileWriter, Property key, String value, Map<String,String> siteConfig) throws IOException {
+ appendProp(fileWriter, key.getKey(), value, siteConfig);
+ }
+
+ private void appendProp(FileWriter fileWriter, String key, String value, Map<String,String> siteConfig) throws IOException {
+ if (!siteConfig.containsKey(key))
+ fileWriter.append("<property><name>" + key + "</name><value>" + value + "</value></property>\n");
+ }
+
+ /**
+ * Sets a given key with a random port for the value on the site config if it doesn't already exist.
+ */
+ private void mergePropWithRandomPort(Map<String,String> siteConfig, String key) {
+ if (!siteConfig.containsKey(key)) {
+ siteConfig.put(key, "0");
+ }
+ }
+
+ /**
+ *
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * Initial root password for instance.
+ * @throws IOException
+ */
+ public MiniAccumuloCluster(File dir, String rootPassword) throws IOException {
+ this(new MiniAccumuloConfig(dir, rootPassword));
+ }
+
+ /**
+ * @param config
+ * initial configuration
+ * @throws IOException
+ */
+
+ public MiniAccumuloCluster(MiniAccumuloConfig config) throws IOException {
+
+ if (config.getDir().exists() && !config.getDir().isDirectory())
+ throw new IllegalArgumentException("Must pass in directory, " + config.getDir() + " is a file");
+
+ if (config.getDir().exists() && config.getDir().list().length != 0)
+ throw new IllegalArgumentException("Directory " + config.getDir() + " is not empty");
+
+ this.config = config;
+
+ libDir = new File(config.getDir(), "lib");
+ confDir = new File(config.getDir(), "conf");
+ accumuloDir = new File(config.getDir(), "accumulo");
+ zooKeeperDir = new File(config.getDir(), "zookeeper");
+ logDir = new File(config.getDir(), "logs");
+ walogDir = new File(config.getDir(), "walogs");
+
+ confDir.mkdirs();
+ accumuloDir.mkdirs();
+ zooKeeperDir.mkdirs();
+ logDir.mkdirs();
+ walogDir.mkdirs();
+ libDir.mkdirs();
+
+ zooKeeperPort = PortUtils.getRandomFreePort();
+
+ File siteFile = new File(confDir, "accumulo-site.xml");
+
+ FileWriter fileWriter = new FileWriter(siteFile);
+ fileWriter.append("<configuration>\n");
+
+ HashMap<String,String> siteConfig = new HashMap<String,String>(config.getSiteConfig());
+
+ appendProp(fileWriter, Property.INSTANCE_DFS_URI, "file:///", siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_DFS_DIR, accumuloDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_ZK_HOST, "localhost:" + zooKeeperPort, siteConfig);
+ appendProp(fileWriter, Property.INSTANCE_SECRET, INSTANCE_SECRET, siteConfig);
+ appendProp(fileWriter, Property.TSERV_PORTSEARCH, "true", siteConfig);
+ appendProp(fileWriter, Property.LOGGER_DIR, walogDir.getAbsolutePath(), siteConfig);
+ appendProp(fileWriter, Property.TSERV_DATACACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_INDEXCACHE_SIZE, "10M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_MAXMEM, "50M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_WALOG_MAX_SIZE, "100M", siteConfig);
+ appendProp(fileWriter, Property.TSERV_NATIVEMAP_ENABLED, "false", siteConfig);
+ appendProp(fileWriter, Property.TRACE_TOKEN_PROPERTY_PREFIX + ".password", config.getRootPassword(), siteConfig);
- appendProp(fileWriter, Property.GC_CYCLE_DELAY, "30s", siteConfig);
++ appendProp(fileWriter, Property.GC_CYCLE_DELAY, "4s", siteConfig);
++ appendProp(fileWriter, Property.GC_CYCLE_START, "0s", siteConfig);
+ mergePropWithRandomPort(siteConfig, Property.MASTER_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TRACE_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.TSERV_CLIENTPORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.MONITOR_PORT.getKey());
+ mergePropWithRandomPort(siteConfig, Property.GC_PORT.getKey());
+
+ // since there is a small amount of memory, check more frequently for majc... setting may not be needed in 1.5
+ appendProp(fileWriter, Property.TSERV_MAJC_DELAY, "3", siteConfig);
+ String cp = System.getenv("ACCUMULO_HOME") + "/lib/.*.jar," + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar," + "$HADOOP_HOME/[^.].*.jar,"
+ + "$HADOOP_HOME/lib/[^.].*.jar," + "$HADOOP_PREFIX/share/hadoop/common/.*.jar," + "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,"
+ + "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar," + "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar";
+ appendProp(fileWriter, Property.GENERAL_CLASSPATHS, cp, siteConfig);
+ appendProp(fileWriter, Property.GENERAL_DYNAMIC_CLASSPATHS, libDir.getAbsolutePath(), siteConfig);
+
+ for (Entry<String,String> entry : siteConfig.entrySet())
+ fileWriter.append("<property><name>" + entry.getKey() + "</name><value>" + entry.getValue() + "</value></property>\n");
+ fileWriter.append("</configuration>\n");
+ fileWriter.close();
+
+ zooCfgFile = new File(confDir, "zoo.cfg");
+ fileWriter = new FileWriter(zooCfgFile);
+
+ // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
+ Properties zooCfg = new Properties();
+ zooCfg.setProperty("tickTime", "1000");
+ zooCfg.setProperty("initLimit", "10");
+ zooCfg.setProperty("syncLimit", "5");
+ zooCfg.setProperty("clientPort", zooKeeperPort + "");
+ zooCfg.setProperty("maxClientCnxns", "100");
+ zooCfg.setProperty("dataDir", zooKeeperDir.getAbsolutePath());
+ zooCfg.store(fileWriter, null);
+
+ fileWriter.close();
+
+ }
+
+ /**
+ * Starts Accumulo and Zookeeper processes. Can only be called once.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws IllegalStateException
+ * if already started
+ */
+
+ public void start() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ throw new IllegalStateException("Already started");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloCluster.this.stop();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ zooKeeperProcess = exec(Main.class, ZooKeeperServerMain.class.getName(), zooCfgFile.getAbsolutePath());
+
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ UtilWaitThread.sleep(250);
+
+ Process initProcess = exec(Initialize.class, "--instance-name", INSTANCE_NAME, "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + logDir + " for errors.");
+ }
+
+ tabletServerProcesses = new Process[config.getNumTservers()];
+ for (int i = 0; i < config.getNumTservers(); i++) {
+ tabletServerProcesses[i] = exec(TabletServer.class);
+ }
+
+ masterProcess = exec(Master.class);
+
- if (config.shouldRunGC()) {
- gcProcess = exec(SimpleGarbageCollector.class);
- }
++ gcProcess = exec(SimpleGarbageCollector.class);
+ }
+
+ /**
+ * @return Accumulo instance name
+ */
+
+ public String getInstanceName() {
+ return INSTANCE_NAME;
+ }
+
+ /**
+ * @return zookeeper connection string
+ */
+
+ public String getZooKeepers() {
+ return "localhost:" + zooKeeperPort;
+ }
+
+ /**
+ * Stops Accumulo and Zookeeper processes. If stop is not called, there is a shutdown hook that is setup to kill the processes. Howerver its probably best to
+ * call stop in a finally block as soon as possible.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+
+ public void stop() throws IOException, InterruptedException {
+ if (zooKeeperProcess != null)
+ zooKeeperProcess.destroy();
+ if (masterProcess != null)
+ masterProcess.destroy();
+ if (tabletServerProcesses != null) {
+ for (Process tserver : tabletServerProcesses) {
+ tserver.destroy();
+ }
+ }
+
+ for (LogWriter lw : logWriters)
+ lw.flush();
+
+ if (gcProcess != null)
+ gcProcess.destroy();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d0f626d9/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 94094ed,0000000..17ab03a
mode 100644,000000..100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@@ -1,107 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Holds configuration for {@link MiniAccumuloCluster}. Required configurations must be passed to constructor and all other configurations are optional.
+ *
+ * @since 1.5.0
+ */
+
+public class MiniAccumuloConfig {
+
+ private File dir = null;
+ private String rootPassword = null;
+ private Map<String,String> siteConfig = Collections.emptyMap();
+ private int numTservers = 2;
- private boolean runGC = false;
+
+ /**
+ * @param dir
+ * An empty or nonexistant temp directoy that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava,
+ * and Junit provide methods for creating temporary directories.
+ * @param rootPassword
+ * The initial password for the Accumulo root user
+ */
+
+ public MiniAccumuloConfig(File dir, String rootPassword) {
+ this.dir = dir;
+ this.rootPassword = rootPassword;
+ }
+
+ public File getDir() {
+ return dir;
+ }
+
+ public String getRootPassword() {
+ return rootPassword;
+ }
+
+ public int getNumTservers() {
+ return numTservers;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defaults to two.
+ *
+ * @param numTservers
+ * the number of tablet servers that mini accumulo cluster should start
+ */
+
+ public MiniAccumuloConfig setNumTservers(int numTservers) {
+ if (numTservers < 1)
+ throw new IllegalArgumentException("Must have at least one tablet server");
+ this.numTservers = numTservers;
+ return this;
+ }
+
+ public Map<String,String> getSiteConfig() {
+ return siteConfig;
+ }
+
+ /**
+ * Calling this method is optional. If not set, it defautls to an empty map.
+ *
+ * @param siteConfig
+ * key/values that you normally put in accumulo-site.xml can be put here
+ */
+
+ public MiniAccumuloConfig setSiteConfig(Map<String,String> siteConfig) {
+ this.siteConfig = siteConfig;
+ return this;
+ }
-
- /**
- * Whether or not the Accumulo garbage collector proces will run
- */
- public boolean shouldRunGC() {
- return runGC;
- }
-
- /**
- * Sets if the Accumulo garbage collector process should run
- *
- * @param shouldRunGC
- */
- public void runGC(boolean shouldRunGC) {
- runGC = shouldRunGC;
- }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d0f626d9/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
----------------------------------------------------------------------
diff --cc minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
index a1f58f6,0000000..318f0af
mode 100644,000000..100644
--- a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
+++ b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
@@@ -1,150 -1,0 +1,123 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Ignore;
- import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import com.google.common.collect.ImmutableMap;
- import com.google.common.io.Files;
+
+/**
+ *
+ */
+public class MiniAccumuloClusterGCTest {
+
- @Test
- public void testGcConfig() throws Exception {
- File f = Files.createTempDir();
- f.deleteOnExit();
- try {
- MiniAccumuloConfig macConfig = new MiniAccumuloConfig(f, passwd);
- macConfig.setNumTservers(1);
-
- Assert.assertEquals(false, macConfig.shouldRunGC());
-
- // Turn on the garbage collector
- macConfig.runGC(true);
-
- Assert.assertEquals(true, macConfig.shouldRunGC());
- } finally {
- if (null != f && f.exists()) {
- f.delete();
- }
- }
- }
-
-
+ private static TemporaryFolder tmpDir = new TemporaryFolder();
+ private static MiniAccumuloConfig macConfig;
+ private static MiniAccumuloCluster accumulo;
+ private static final String passwd = "password";
+
+ public static void setupMiniCluster() throws Exception {
+ tmpDir.create();
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+
+ macConfig = new MiniAccumuloConfig(tmpDir.getRoot(), passwd);
+ macConfig.setNumTservers(1);
+
- // Turn on the garbage collector
- macConfig.runGC(true);
-
+ String gcPort = Integer.toString(PortUtils.getRandomFreePort());
+
+ // And tweak the settings to make it run often
+ Map<String,String> config = ImmutableMap.of(Property.GC_CYCLE_DELAY.getKey(), "1s", Property.GC_CYCLE_START.getKey(), "0s", Property.GC_PORT.getKey(), gcPort);
+ macConfig.setSiteConfig(config);
+
+ accumulo = new MiniAccumuloCluster(macConfig);
+ accumulo.start();
+ }
+
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ tmpDir.delete();
+ }
+
+ // This test seems to be a little too unstable for a unit test
+ @Ignore
+ public void test() throws Exception {
+ ZooKeeperInstance inst = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
+ Connector c = inst.getConnector("root", new PasswordToken(passwd));
+
+ final String table = "foobar";
+ c.tableOperations().create(table);
+
+ BatchWriter bw = null;
+
+ // Add some data
+ try {
+ bw = c.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ for (int i = 0; i < 50; i++) {
+ m.put("colf", Integer.toString(i), "");
+ }
+
+ bw.addMutation(m);
+ } finally {
+ if (null != bw) {
+ bw.close();
+ }
+ }
+
+ final boolean flush = true, wait = true;
+
+ // Compact the tables to get some rfiles which we can gc
+ c.tableOperations().compact(table, null, null, flush, wait);
+ c.tableOperations().compact("!METADATA", null, null, flush, wait);
+
+ File accumuloDir = new File(tmpDir.getRoot().getAbsolutePath(), "accumulo");
+ File tables = new File(accumuloDir.getAbsolutePath(), "tables");
+
+ int fileCountAfterCompaction = FileUtils.listFiles(tables, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE).size();
+
+ // Sleep for 4s to let the GC do its thing
+ for (int i = 1; i < 5; i++) {
+ Thread.sleep(1000);
+ int fileCountAfterGCWait = FileUtils.listFiles(tables, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE).size();
+
+ if (fileCountAfterGCWait < fileCountAfterCompaction) {
+ return;
+ }
+ }
+
+ Assert.fail("Expected to find less files after compaction and pause for GC");
+ }
+
+}