You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2013/12/02 21:46:15 UTC
git commit: ACCUMULO-1556 Clarify initialization error for used fs
Updated Branches:
refs/heads/1.4.5-SNAPSHOT 82477f08a -> 1b3d071a8
ACCUMULO-1556 Clarify initialization error for used fs
The Initialize class now generates clearer and more informative
error messages if an initialized instance is discovered. The
messages vary depending on whether instance.dfs.uri is used. The
values of instance.dfs.uri and instance.dfs.dir are also logged.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1b3d071a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1b3d071a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1b3d071a
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: 1b3d071a8b46617245a39d039a2fe9e31600fc2e
Parents: 82477f0
Author: Bill Havanki <bh...@cloudera.com>
Authored: Tue Nov 5 17:07:28 2013 -0500
Committer: John Vines <jv...@gmail.com>
Committed: Fri Nov 29 14:26:41 2013 -0500
----------------------------------------------------------------------
pom.xml | 6 +
src/server/pom.xml | 4 +
.../apache/accumulo/server/util/Initialize.java | 60 +++++--
.../accumulo/server/util/InitializeTest.java | 155 +++++++++++++++++++
4 files changed, 211 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ac57b9d..d0b3041 100644
--- a/pom.xml
+++ b/pom.xml
@@ -708,6 +708,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0.1</version>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/src/server/pom.xml
----------------------------------------------------------------------
diff --git a/src/server/pom.xml b/src/server/pom.xml
index cc55e90..672a6d3 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -101,6 +101,10 @@
<artifactId>commons-lang</artifactId>
</dependency>
<dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java b/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
index 51576fc..c6bb57a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
@@ -29,6 +29,7 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Value;
@@ -73,12 +74,29 @@ public class Initialize {
private static boolean clearInstanceName = false;
private static ConsoleReader reader = null;
+ private static IZooReaderWriter zoo = ZooReaderWriter.getInstance();
private static ConsoleReader getConsoleReader() throws IOException {
if (reader == null)
reader = new ConsoleReader();
return reader;
}
+ /**
+ * Sets this class's ZooKeeper reader/writer.
+ *
+ * @param izoo reader/writer
+ */
+ static void setZooReaderWriter(IZooReaderWriter izoo) {
+ zoo = izoo;
+ }
+ /**
+ * Gets this class's ZooKeeper reader/writer.
+ *
+ * @return reader/writer
+ */
+ static IZooReaderWriter getZooReaderWriter() {
+ return zoo;
+ }
private static HashMap<String,String> initialMetadataConf = new HashMap<String,String>();
static {
@@ -107,20 +125,18 @@ public class Initialize {
initialMetadataConf.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
}
- public static boolean doInit(Configuration conf, FileSystem fs) throws IOException {
- if (!ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI).equals(""))
- log.info("Hadoop Filesystem is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI));
- else
- log.info("Hadoop Filesystem is " + FileSystem.getDefaultUri(conf));
+ static boolean checkInit(Configuration conf, FileSystem fs, SiteConfiguration sconf) throws IOException {
+ String fsUri = fs.getUri().toString();
+ log.info("Hadoop Filesystem is " + fsUri);
log.info("Accumulo data dir is " + ServerConstants.getBaseDir());
- log.info("Zookeeper server is " + ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST));
+ log.info("Zookeeper server is " + sconf.get(Property.INSTANCE_ZK_HOST));
log.info("Checking if Zookeeper is available. If this hangs, then you need to make sure zookeeper is running");
if (!zookeeperAvailable()) {
log.fatal("Zookeeper needs to be up and running in order to init. Exiting ...");
return false;
}
- if (ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) {
+ if (sconf.get(Property.INSTANCE_SECRET).equals(Property.INSTANCE_SECRET.getDefaultValue())) {
ConsoleReader c = getConsoleReader();
c.beep();
c.printNewline();
@@ -137,16 +153,34 @@ public class Initialize {
c.printNewline();
}
- UUID uuid = UUID.randomUUID();
-
try {
if (isInitialized(fs)) {
- log.fatal("It appears this location was previously initialized, exiting ... ");
+ String instanceDfsDir = sconf.get(Property.INSTANCE_DFS_DIR);
+ log.fatal("It appears the directory " + fsUri + instanceDfsDir + " was previously initialized.");
+ String instanceDfsUri = sconf.get(Property.INSTANCE_DFS_URI);
+ if ("".equals(instanceDfsUri)) {
+ log.fatal("You are using the default URI for the filesystem. Set the property " + Property.INSTANCE_DFS_URI + " to use a different filesystem,");
+ } else {
+ log.fatal("Change the property " + Property.INSTANCE_DFS_URI + " to use a different filesystem,");
+ }
+ log.fatal("or change the property " + Property.INSTANCE_DFS_DIR + " to use a different directory.");
+ log.fatal("The current value of " + Property.INSTANCE_DFS_URI + " is |" + instanceDfsUri + "|");
+ log.fatal("The current value of " + Property.INSTANCE_DFS_DIR + " is |" + instanceDfsDir + "|");
return false;
}
} catch (IOException e) {
- throw new RuntimeException(e);
+ throw new IOException("Failed to check if filesystem already initialized", e);
+ }
+
+ return true;
+ }
+
+ public static boolean doInit(Configuration conf, FileSystem fs) throws IOException {
+ if (!checkInit(conf, fs, ServerConfiguration.getSiteConfiguration())) {
+ return false;
}
+
+ UUID uuid = UUID.randomUUID();
// prompt user for instance name and root password early, in case they
// abort, we don't leave an inconsistent HDFS/ZooKeeper structure
@@ -186,7 +220,6 @@ public class Initialize {
* @return
*/
private static boolean zookeeperAvailable() {
- IZooReaderWriter zoo = ZooReaderWriter.getInstance();
try {
return zoo.exists("/");
} catch (KeeperException e) {
@@ -332,7 +365,6 @@ public class Initialize {
private static void initZooKeeper(String uuid, String instanceNamePath) throws KeeperException, InterruptedException {
// setup basic data in zookeeper
- IZooReaderWriter zoo = ZooReaderWriter.getInstance();
ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
ZooUtil.putPersistentData(zoo.getZooKeeper(), Constants.ZROOT + Constants.ZINSTANCES, new byte[0], -1, NodeExistsPolicy.SKIP, Ids.OPEN_ACL_UNSAFE);
@@ -378,7 +410,7 @@ public class Initialize {
if (clearInstanceName) {
exists = false;
break;
- } else if ((boolean) (exists = ZooReaderWriter.getInstance().exists(instanceNamePath))) {
+ } else if ((boolean) (exists = zoo.exists(instanceNamePath))) {
String decision = getConsoleReader().readLine("Instance name \"" + instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : ");
if (decision == null)
System.exit(0);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b3d071a/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java b/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java
new file mode 100644
index 0000000..138a0ac
--- /dev/null
+++ b/src/server/src/test/java/org/apache/accumulo/server/util/InitializeTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.server.zookeeper.IZooReaderWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.easymock.Capture;
+import static org.easymock.EasyMock.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * This test is not thread-safe.
+ */
+public class InitializeTest {
+ private Configuration conf;
+ private FileSystem fs;
+ private SiteConfiguration sconf;
+ private IZooReaderWriter zooOrig;
+ private IZooReaderWriter zoo;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = createMock(Configuration.class);
+ fs = createMock(FileSystem.class);
+ sconf = createMock(SiteConfiguration.class);
+ zoo = createMock(IZooReaderWriter.class);
+ zooOrig = Initialize.getZooReaderWriter();
+ Initialize.setZooReaderWriter(zoo);
+ }
+
+ @After
+ public void tearDown() {
+ Initialize.setZooReaderWriter(zooOrig);
+ }
+
+ @Test
+ public void testIsInitialized_HasInstanceId() throws Exception {
+ expect(fs.exists(anyObject(Path.class))).andReturn(true);
+ replay(fs);
+ assertTrue(Initialize.isInitialized(fs));
+ }
+
+ @Test
+ public void testIsInitialized_HasDataVersion() throws Exception {
+ expect(fs.exists(anyObject(Path.class))).andReturn(false);
+ expect(fs.exists(anyObject(Path.class))).andReturn(true);
+ replay(fs);
+ assertTrue(Initialize.isInitialized(fs));
+ }
+
+ @Test
+ public void testCheckInit_NoZK() throws Exception {
+ expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo");
+ expectLastCall().anyTimes();
+ expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1");
+ replay(sconf);
+ expect(zoo.exists("/")).andReturn(false);
+ replay(zoo);
+ expect(fs.getUri()).andReturn(new URI("hdfs://foo"));
+ replay(fs);
+
+ assertFalse(Initialize.checkInit(conf, fs, sconf));
+ }
+
+ @Test
+ public void testCheckInit_AlreadyInit() throws Exception {
+ expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo");
+ expectLastCall().anyTimes();
+ expect(sconf.get(Property.INSTANCE_DFS_DIR)).andReturn("/bar");
+ expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue());
+ replay(sconf);
+ expect(zoo.exists("/")).andReturn(true);
+ replay(zoo);
+ expect(fs.getUri()).andReturn(new URI("hdfs://foo"));
+ expect(fs.exists(anyObject(Path.class))).andReturn(true);
+ replay(fs);
+
+ assertFalse(Initialize.checkInit(conf, fs, sconf));
+ }
+
+ @Test
+ public void testCheckInit_AlreadyInit_DefaultUri() throws Exception {
+ expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("");
+ expectLastCall().anyTimes();
+ expect(sconf.get(Property.INSTANCE_DFS_DIR)).andReturn("/bar");
+ expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue());
+ replay(sconf);
+ expect(zoo.exists("/")).andReturn(true);
+ replay(zoo);
+ expect(fs.getUri()).andReturn(new URI("hdfs://default"));
+ expect(fs.exists(anyObject(Path.class))).andReturn(true);
+ replay(fs);
+
+ assertFalse(Initialize.checkInit(conf, fs, sconf));
+ }
+
+ @Test(expected = IOException.class)
+ public void testCheckInit_FSException() throws Exception {
+ expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo");
+ expectLastCall().anyTimes();
+ expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue());
+ replay(sconf);
+ expect(zoo.exists("/")).andReturn(true);
+ replay(zoo);
+ expect(fs.getUri()).andReturn(new URI("hdfs://foo"));
+ expect(fs.exists(anyObject(Path.class))).andThrow(new IOException());
+ replay(fs);
+
+ Initialize.checkInit(conf, fs, sconf);
+ }
+
+ @Test
+ public void testCheckInit_OK() throws Exception {
+ expect(sconf.get(Property.INSTANCE_DFS_URI)).andReturn("hdfs://foo");
+ expectLastCall().anyTimes();
+ expect(sconf.get(Property.INSTANCE_ZK_HOST)).andReturn("zk1");
+ expect(sconf.get(Property.INSTANCE_SECRET)).andReturn(Property.INSTANCE_SECRET.getDefaultValue());
+ replay(sconf);
+ expect(zoo.exists("/")).andReturn(true);
+ replay(zoo);
+ expect(fs.getUri()).andReturn(new URI("hdfs://foo"));
+ expect(fs.exists(anyObject(Path.class))).andReturn(false);
+ expect(fs.exists(anyObject(Path.class))).andReturn(false);
+ replay(fs);
+
+ assertTrue(Initialize.checkInit(conf, fs, sconf));
+ }
+}