You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2014/10/21 22:47:01 UTC
[2/2] git commit: Merge branch '1.6'
Merge branch '1.6'
Conflicts:
minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfiguration.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7aca0326
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7aca0326
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7aca0326
Branch: refs/heads/master
Commit: 7aca0326e71aa46a52bc8e5e1f3c64cbf59dbb23
Parents: 2e012e1 e3a743c
Author: John Vines <vi...@apache.org>
Authored: Tue Oct 21 16:46:50 2014 -0400
Committer: John Vines <vi...@apache.org>
Committed: Tue Oct 21 16:46:50 2014 -0400
----------------------------------------------------------------------
.../core/volume/VolumeConfiguration.java | 12 +-
.../apache/accumulo/core/zookeeper/ZooUtil.java | 17 +-
.../minicluster/MiniAccumuloConfig.java | 19 +-
.../minicluster/MiniAccumuloRunner.java | 1 +
.../impl/MiniAccumuloClusterImpl.java | 206 ++++++++++++-------
.../impl/MiniAccumuloConfigImpl.java | 145 +++++++++++--
.../accumulo/server/fs/VolumeManagerImpl.java | 7 +-
.../accumulo/server/util/AccumuloStatus.java | 74 +++++++
.../org/apache/accumulo/test/ExistingMacIT.java | 152 ++++++++++++++
9 files changed, 536 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7aca0326/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7aca0326/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index 981a7bb,d5dc1e9..0febb58
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@@ -63,9 -65,13 +65,12 @@@ import org.apache.accumulo.core.conf.Pr
import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.master.thrift.MasterGoalState;
import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+ import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
+ import org.apache.accumulo.core.zookeeper.ZooUtil;
+ import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.gc.SimpleGarbageCollector;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.master.state.SetGoalState;
@@@ -83,14 -94,18 +93,20 @@@ import org.apache.commons.io.FileUtils
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.impl.VFSClassLoader;
import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
- import org.apache.log4j.Logger;
import org.apache.thrift.TException;
+ import org.apache.zookeeper.KeeperException;
+ import org.apache.zookeeper.data.Stat;
import org.apache.zookeeper.server.ZooKeeperServerMain;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
++import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.Maps;
@@@ -444,58 -473,98 +472,98 @@@ public class MiniAccumuloClusterImpl im
*/
@Override
public synchronized void start() throws IOException, InterruptedException {
+ if (config.useExistingInstance()) {
+ Configuration acuConf = config.getAccumuloConfiguration();
+ Configuration hadoopConf = config.getHadoopConfiguration();
+
+ ConfigurationCopy cc = new ConfigurationCopy(acuConf);
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get(cc, hadoopConf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
- if (!initialized) {
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, cc, hadoopConf);
+ IZooReaderWriter zrw = new ZooReaderWriterFactory().getZooReaderWriter(cc.get(Property.INSTANCE_ZK_HOST),
+ (int) cc.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), cc.get(Property.INSTANCE_SECRET));
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- MiniAccumuloClusterImpl.this.stop();
- } catch (IOException e) {
- log.error("IOException while attempting to stop the MiniAccumuloCluster.", e);
- } catch (InterruptedException e) {
- log.error("The stopping of MiniAccumuloCluster was interrupted.", e);
+ String rootPath = ZooUtil.getRoot(instanceIdFromFile);
+
+ String instanceName = null;
+ try {
+ for (String name : zrw.getChildren(Constants.ZROOT + Constants.ZINSTANCES)) {
+ String instanceNamePath = Constants.ZROOT + Constants.ZINSTANCES + "/" + name;
+ byte[] bytes = zrw.getData(instanceNamePath, new Stat());
- String iid = new String(bytes, Constants.UTF8);
++ String iid = new String(bytes, Charsets.UTF_8);
+ if (iid.equals(instanceIdFromFile)) {
+ instanceName = name;
}
}
- });
- }
+ } catch (KeeperException e) {
+ throw new RuntimeException("Unable to read instance name from zookeeper.", e);
+ }
+ if (instanceName == null)
+ throw new RuntimeException("Unable to read instance name from zookeeper.");
- if (zooKeeperProcess == null) {
- zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
- }
+ config.setInstanceName(instanceName);
+ if (!AccumuloStatus.isAccumuloOffline(zrw, rootPath))
+ throw new RuntimeException("The Accumulo instance being used is already running. Aborting.");
+ } else {
+ if (!initialized) {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ MiniAccumuloClusterImpl.this.stop();
+ } catch (IOException e) {
- e.printStackTrace();
++ log.error("IOException while attempting to stop the MiniAccumuloCluster.", e);
+ } catch (InterruptedException e) {
- e.printStackTrace();
++ log.error("The stopping of MiniAccumuloCluster was interrupted.", e);
+ }
+ }
+ });
+ }
- if (!initialized) {
- // sleep a little bit to let zookeeper come up before calling init, seems to work better
- long startTime = System.currentTimeMillis();
- while (true) {
- Socket s = null;
- try {
- s = new Socket("localhost", config.getZooKeeperPort());
- s.getOutputStream().write("ruok\n".getBytes());
- s.getOutputStream().flush();
- byte buffer[] = new byte[100];
- int n = s.getInputStream().read(buffer);
- if (n >= 4 && new String(buffer, 0, 4).equals("imok"))
- break;
- } catch (Exception e) {
- if (System.currentTimeMillis() - startTime >= config.getZooKeeperStartupTime()) {
- throw new ZooKeeperBindException("Zookeeper did not start within " + (config.getZooKeeperStartupTime() / 1000) + " seconds. Check the logs in "
- + config.getLogDir() + " for errors. Last exception: " + e);
+ if (zooKeeperProcess == null) {
+ zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
+ }
+
+ if (!initialized) {
+ // sleep a little bit to let zookeeper come up before calling init, seems to work better
+ long startTime = System.currentTimeMillis();
+ while (true) {
+ Socket s = null;
+ try {
+ s = new Socket("localhost", config.getZooKeeperPort());
+ s.getOutputStream().write("ruok\n".getBytes());
+ s.getOutputStream().flush();
+ byte buffer[] = new byte[100];
+ int n = s.getInputStream().read(buffer);
+ if (n >= 4 && new String(buffer, 0, 4).equals("imok"))
+ break;
+ } catch (Exception e) {
+ if (System.currentTimeMillis() - startTime >= config.getZooKeeperStartupTime()) {
+ throw new ZooKeeperBindException("Zookeeper did not start within " + (config.getZooKeeperStartupTime() / 1000) + " seconds. Check the logs in "
+ + config.getLogDir() + " for errors. Last exception: " + e);
+ }
+ } finally {
+ if (s != null)
+ s.close();
}
- UtilWaitThread.sleep(250);
- } finally {
- if (s != null)
- s.close();
}
+ Process initProcess = exec(Initialize.class, "--instance-name", config.getInstanceName(), "--password", config.getRootPassword());
+ int ret = initProcess.waitFor();
+ if (ret != 0) {
+ throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
+ }
+ initialized = true;
}
- Process initProcess = exec(Initialize.class, "--instance-name", config.getInstanceName(), "--password", config.getRootPassword());
- int ret = initProcess.waitFor();
- if (ret != 0) {
- throw new RuntimeException("Initialize process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
- }
- initialized = true;
}
-
++
+ log.info("Starting MAC against instance {} and zookeeper(s) {}.", config.getInstanceName(), config.getZooKeepers());
-
++
synchronized (tabletServerProcesses) {
for (int i = tabletServerProcesses.size(); i < config.getNumTservers(); i++) {
tabletServerProcesses.add(_exec(TabletServer.class, ServerType.TABLET_SERVER));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7aca0326/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloConfigImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7aca0326/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------