You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2013/10/17 19:08:34 UTC
[1/3] git commit: ACCUMULO-1637 Rework the hadoop append/sync checks
trying to match what Hadoop is doing internally by default.
Updated Branches:
refs/heads/master a65fe6a2b -> ba8ad057d
ACCUMULO-1637 Rework the hadoop append/sync checks trying to match what
Hadoop is doing internally by default.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d1243aaf
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d1243aaf
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d1243aaf
Branch: refs/heads/master
Commit: d1243aafc7dfc9d14ccbe7d67f92055e26228221
Parents: 1b05bd8
Author: Josh Elser <el...@apache.org>
Authored: Thu Oct 17 12:25:26 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Oct 17 12:28:02 2013 -0400
----------------------------------------------------------------------
.../server/tabletserver/TabletServer.java | 44 ++++++++++++++------
1 file changed, 31 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d1243aaf/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index b670f13..137d3b1 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@ -24,6 +24,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.UnknownHostException;
@@ -210,6 +211,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Reader;
@@ -3242,21 +3244,37 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
private static void ensureHdfsSyncIsEnabled(FileSystem fs) {
if (fs instanceof DistributedFileSystem) {
- if (!fs.getConf().getBoolean("dfs.durable.sync", false) && !fs.getConf().getBoolean("dfs.support.append", false)) {
- String msg = "Must set dfs.durable.sync OR dfs.support.append to true. Which one needs to be set depends on your version of HDFS. See ACCUMULO-623. \n"
- + "HADOOP RELEASE VERSION SYNC NAME DEFAULT\n"
- + "Apache Hadoop 0.20.205 dfs.support.append false\n"
- + "Apache Hadoop 0.23.x dfs.support.append true\n"
- + "Apache Hadoop 1.0.x dfs.support.append false\n"
- + "Apache Hadoop 1.1.x dfs.durable.sync true\n"
- + "Apache Hadoop 2.0.0-2.0.2 dfs.support.append true\n"
- + "Cloudera CDH 3u0-3u3 ???? true\n"
- + "Cloudera CDH 3u4 dfs.support.append true\n"
- + "Hortonworks HDP `1.0 dfs.support.append false\n"
- + "Hortonworks HDP `1.1 dfs.support.append false";
- log.fatal(msg);
+ final String DFS_DURABLE_SYNC = "dfs.durable.sync", DFS_SUPPORT_APPEND = "dfs.support.append";
+ // Check to make sure that we have proper defaults configured
+ try {
+ // If the default is off (0.20.205.x or 1.0.x)
+ DFSConfigKeys configKeys = new DFSConfigKeys();
+
+ // Can't use the final constant itself as Java will inline it at compile time
+ Field dfsSupportAppendDefaultField = configKeys.getClass().getField("DFS_SUPPORT_APPEND_DEFAULT");
+ boolean dfsSupportAppendDefaultValue = dfsSupportAppendDefaultField.getBoolean(configKeys);
+
+ if (!dfsSupportAppendDefaultValue) {
+ // See if the user did the correct override
+ if (!fs.getConf().getBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, false)) {
+ log.fatal("Accumulo requires that dfs.support.append to true. See ACCUMULO-623 and ACCUMULO-1637 for more details.");
+ System.exit(-1);
+ }
+ }
+ } catch (NoSuchFieldException e) {
+ // If we can't find DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT, the user is running
+ // 1.1.x or 1.2.x. This is ok, though, as, by default, these versions have append/sync enabled.
+ } catch (Exception e) {
+ log.warn("Error while checking for " + DFS_SUPPORT_APPEND + ". The user should ensure that Hadoop is configured to properly supports append and sync. See ACCUMULO-623 and ACCUMULO-1637 for more details.", e);
+ }
+
+ // If either of these parameters are configured to be false, fail.
+ // This is a sign that someone is writing bad configuration.
+ if (!fs.getConf().getBoolean(DFS_SUPPORT_APPEND, true) || !fs.getConf().getBoolean(DFS_DURABLE_SYNC, true)) {
+ log.fatal("Accumulo requires that " + DFS_SUPPORT_APPEND + " and " + DFS_DURABLE_SYNC + " not be configured as false. See ACCUMULO-623 and ACCUMULO-1637 for more details.");
System.exit(-1);
}
+
try {
// if this class exists
Class.forName("org.apache.hadoop.fs.CreateFlag");
[3/3] git commit: Merge branch '1.5.1-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT'
Conflicts:
server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ba8ad057
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ba8ad057
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ba8ad057
Branch: refs/heads/master
Commit: ba8ad057d70d90f13a75911b81650897b3889f29
Parents: a65fe6a 685cc4a
Author: Josh Elser <el...@apache.org>
Authored: Thu Oct 17 13:08:11 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Oct 17 13:08:11 2013 -0400
----------------------------------------------------------------------
.../server/tabletserver/TabletServer.java | 60 ++++++++++++++++++++
1 file changed, 60 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ba8ad057/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index a44b194,ea30694..4e06b6d
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@@ -213,9 -206,15 +215,14 @@@ import org.apache.accumulo.trace.instru
import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
import org.apache.accumulo.trace.thrift.TInfo;
import org.apache.commons.collections.map.LRUMap;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.fs.Trash;
+ import org.apache.hadoop.hdfs.DFSConfigKeys;
+ import org.apache.hadoop.hdfs.DistributedFileSystem;
+ import org.apache.hadoop.io.SequenceFile;
+ import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
@@@ -3596,6 -3230,8 +3603,7 @@@ public class TabletServer extends Abstr
Instance instance = HdfsZooInstance.getInstance();
ServerConfiguration conf = new ServerConfiguration(instance);
Accumulo.init(fs, conf, "tserver");
+ ensureHdfsSyncIsEnabled(fs);
- recoverLocalWriteAheadLogs(fs, conf);
TabletServer server = new TabletServer(conf, fs);
server.config(hostname);
Accumulo.enableTracing(hostname, "tserver");
@@@ -3606,6 -3242,108 +3614,58 @@@
}
}
- private static void ensureHdfsSyncIsEnabled(FileSystem fs) {
- if (fs instanceof DistributedFileSystem) {
- final String DFS_DURABLE_SYNC = "dfs.durable.sync", DFS_SUPPORT_APPEND = "dfs.support.append";
- final String ticketMessage = "See ACCUMULO-623 and ACCUMULO-1637 for more details.";
- // Check to make sure that we have proper defaults configured
- try {
- // If the default is off (0.20.205.x or 1.0.x)
- DFSConfigKeys configKeys = new DFSConfigKeys();
-
- // Can't use the final constant itself as Java will inline it at compile time
- Field dfsSupportAppendDefaultField = configKeys.getClass().getField("DFS_SUPPORT_APPEND_DEFAULT");
- boolean dfsSupportAppendDefaultValue = dfsSupportAppendDefaultField.getBoolean(configKeys);
-
- if (!dfsSupportAppendDefaultValue) {
- // See if the user did the correct override
- if (!fs.getConf().getBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, false)) {
- log.fatal("Accumulo requires that dfs.support.append to true. " + ticketMessage);
- System.exit(-1);
++ private static void ensureHdfsSyncIsEnabled(VolumeManager volumes) {
++ for (Entry<String,? extends FileSystem> entry : volumes.getFileSystems().entrySet()) {
++ final String volumeName = entry.getKey();
++ final FileSystem fs = entry.getValue();
++
++ if (fs instanceof DistributedFileSystem) {
++ final String DFS_DURABLE_SYNC = "dfs.durable.sync", DFS_SUPPORT_APPEND = "dfs.support.append";
++ final String ticketMessage = "See ACCUMULO-623 and ACCUMULO-1637 for more details.";
++ // Check to make sure that we have proper defaults configured
++ try {
++ // If the default is off (0.20.205.x or 1.0.x)
++ DFSConfigKeys configKeys = new DFSConfigKeys();
++
++ // Can't use the final constant itself as Java will inline it at compile time
++ Field dfsSupportAppendDefaultField = configKeys.getClass().getField("DFS_SUPPORT_APPEND_DEFAULT");
++ boolean dfsSupportAppendDefaultValue = dfsSupportAppendDefaultField.getBoolean(configKeys);
++
++ if (!dfsSupportAppendDefaultValue) {
++ // See if the user did the correct override
++ if (!fs.getConf().getBoolean(DFS_SUPPORT_APPEND, false)) {
++ log.fatal("Accumulo requires that dfs.support.append to true on volume " + volumeName + ". " + ticketMessage);
++ System.exit(-1);
++ }
+ }
++ } catch (NoSuchFieldException e) {
++ // If we can't find DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT, the user is running
++ // 1.1.x or 1.2.x. This is ok, though, as, by default, these versions have append/sync enabled.
++ } catch (Exception e) {
++ log.warn("Error while checking for " + DFS_SUPPORT_APPEND + " on volume " + volumeName + ". The user should ensure that Hadoop is configured to properly supports append and sync. " + ticketMessage, e);
+ }
- } catch (NoSuchFieldException e) {
- // If we can't find DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT, the user is running
- // 1.1.x or 1.2.x. This is ok, though, as, by default, these versions have append/sync enabled.
- } catch (Exception e) {
- log.warn("Error while checking for " + DFS_SUPPORT_APPEND + ". The user should ensure that Hadoop is configured to properly supports append and sync. " + ticketMessage, e);
- }
-
- // If either of these parameters are configured to be false, fail.
- // This is a sign that someone is writing bad configuration.
- if (!fs.getConf().getBoolean(DFS_SUPPORT_APPEND, true) || !fs.getConf().getBoolean(DFS_DURABLE_SYNC, true)) {
- log.fatal("Accumulo requires that " + DFS_SUPPORT_APPEND + " and " + DFS_DURABLE_SYNC + " not be configured as false. " + ticketMessage);
- System.exit(-1);
- }
-
- try {
- // if this class exists
- Class.forName("org.apache.hadoop.fs.CreateFlag");
- // we're running hadoop 2.0, 1.1
- if (!fs.getConf().getBoolean("dfs.datanode.synconclose", false)) {
- log.warn("dfs.datanode.synconclose set to false: data loss is possible on system reset or power loss");
++
++ // If either of these parameters are configured to be false, fail.
++ // This is a sign that someone is writing bad configuration.
++ if (!fs.getConf().getBoolean(DFS_SUPPORT_APPEND, true) || !fs.getConf().getBoolean(DFS_DURABLE_SYNC, true)) {
++ log.fatal("Accumulo requires that " + DFS_SUPPORT_APPEND + " and " + DFS_DURABLE_SYNC + " not be configured as false on volume " + volumeName + ". " + ticketMessage);
++ System.exit(-1);
+ }
- } catch (ClassNotFoundException ex) {
- // hadoop 1.0
- }
- }
-
- }
-
- /**
- * Copy local walogs into HDFS on an upgrade
- *
- */
- public static void recoverLocalWriteAheadLogs(FileSystem fs, ServerConfiguration serverConf) throws IOException {
- FileSystem localfs = FileSystem.getLocal(fs.getConf()).getRawFileSystem();
- AccumuloConfiguration conf = serverConf.getConfiguration();
- String localWalDirectories = conf.get(Property.LOGGER_DIR);
- for (String localWalDirectory : localWalDirectories.split(",")) {
- if (!localWalDirectory.startsWith("/")) {
- localWalDirectory = System.getenv("ACCUMULO_HOME") + "/" + localWalDirectory;
- }
-
- FileStatus status = null;
- try {
- status = localfs.getFileStatus(new Path(localWalDirectory));
- } catch (FileNotFoundException fne) {}
-
- if (status == null || !status.isDir()) {
- log.debug("Local walog dir " + localWalDirectory + " not found ");
- continue;
- }
-
- for (FileStatus file : localfs.listStatus(new Path(localWalDirectory))) {
- String name = file.getPath().getName();
++
+ try {
- UUID.fromString(name);
- } catch (IllegalArgumentException ex) {
- log.info("Ignoring non-log file " + name + " in " + localWalDirectory);
- continue;
- }
- LogFileKey key = new LogFileKey();
- LogFileValue value = new LogFileValue();
- log.info("Openning local log " + file.getPath());
- Reader reader = new SequenceFile.Reader(localfs, file.getPath(), localfs.getConf());
- Path tmp = new Path(Constants.getWalDirectory(conf) + "/" + name + ".copy");
- FSDataOutputStream writer = fs.create(tmp);
- while (reader.next(key, value)) {
- try {
- key.write(writer);
- value.write(writer);
- } catch (EOFException ex) {
- break;
++ // if this class exists
++ Class.forName("org.apache.hadoop.fs.CreateFlag");
++ // we're running hadoop 2.0, 1.1
++ if (!fs.getConf().getBoolean("dfs.datanode.synconclose", false)) {
++ log.warn("dfs.datanode.synconclose set to false: data loss is possible on system reset or power loss on volume " + volumeName);
+ }
++ } catch (ClassNotFoundException ex) {
++ // hadoop 1.0
+ }
- writer.close();
- reader.close();
- fs.rename(tmp, new Path(tmp.getParent(), name));
- log.info("Copied local log " + name);
- localfs.delete(new Path(localWalDirectory, name), true);
+ }
+ }
+ }
-
++
public void minorCompactionFinished(CommitSession tablet, String newDatafile, int walogSeq) throws IOException {
totalMinorCompactions++;
logger.minorCompactionFinished(tablet, newDatafile, walogSeq);
[2/3] git commit: ACCUMULO-1637 Lifting the ticket message into its
own string constant
Posted by el...@apache.org.
ACCUMULO-1637 Lifting the ticket message into its own string constant
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/685cc4a7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/685cc4a7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/685cc4a7
Branch: refs/heads/master
Commit: 685cc4a70d87fd7eda56538fb381a8347b4e56e3
Parents: d1243aa
Author: Josh Elser <el...@apache.org>
Authored: Thu Oct 17 12:48:57 2013 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Oct 17 12:48:57 2013 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/server/tabletserver/TabletServer.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/685cc4a7/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
index 137d3b1..ea30694 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
@@ -3245,6 +3245,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
private static void ensureHdfsSyncIsEnabled(FileSystem fs) {
if (fs instanceof DistributedFileSystem) {
final String DFS_DURABLE_SYNC = "dfs.durable.sync", DFS_SUPPORT_APPEND = "dfs.support.append";
+ final String ticketMessage = "See ACCUMULO-623 and ACCUMULO-1637 for more details.";
// Check to make sure that we have proper defaults configured
try {
// If the default is off (0.20.205.x or 1.0.x)
@@ -3257,7 +3258,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
if (!dfsSupportAppendDefaultValue) {
// See if the user did the correct override
if (!fs.getConf().getBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, false)) {
- log.fatal("Accumulo requires that dfs.support.append to true. See ACCUMULO-623 and ACCUMULO-1637 for more details.");
+ log.fatal("Accumulo requires that dfs.support.append to true. " + ticketMessage);
System.exit(-1);
}
}
@@ -3265,13 +3266,13 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// If we can't find DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT, the user is running
// 1.1.x or 1.2.x. This is ok, though, as, by default, these versions have append/sync enabled.
} catch (Exception e) {
- log.warn("Error while checking for " + DFS_SUPPORT_APPEND + ". The user should ensure that Hadoop is configured to properly supports append and sync. See ACCUMULO-623 and ACCUMULO-1637 for more details.", e);
+ log.warn("Error while checking for " + DFS_SUPPORT_APPEND + ". The user should ensure that Hadoop is configured to properly supports append and sync. " + ticketMessage, e);
}
// If either of these parameters are configured to be false, fail.
// This is a sign that someone is writing bad configuration.
if (!fs.getConf().getBoolean(DFS_SUPPORT_APPEND, true) || !fs.getConf().getBoolean(DFS_DURABLE_SYNC, true)) {
- log.fatal("Accumulo requires that " + DFS_SUPPORT_APPEND + " and " + DFS_DURABLE_SYNC + " not be configured as false. See ACCUMULO-623 and ACCUMULO-1637 for more details.");
+ log.fatal("Accumulo requires that " + DFS_SUPPORT_APPEND + " and " + DFS_DURABLE_SYNC + " not be configured as false. " + ticketMessage);
System.exit(-1);
}