You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2016/06/14 13:22:51 UTC
[1/3] accumulo git commit: ACCUMULO-3923: Modify parsing of the HDFS
directory to be less restrictive, provide a better error message,
fail if directory can't be created, handle slf4j jars
Repository: accumulo
Updated Branches:
refs/heads/1.8 a98dbecd1 -> 24289e2c4
ACCUMULO-3923: Modify parsing of the HDFS directory to be less restrictive, provide a better error message, fail if directory can't be created, handle slf4j jars
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cd31dc8d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cd31dc8d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cd31dc8d
Branch: refs/heads/1.8
Commit: cd31dc8de27d9cbd46d129e3c536ee90175f7ce8
Parents: 462dfbc
Author: Dave Marion <dl...@apache.org>
Authored: Mon Jun 13 11:42:59 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Tue Jun 14 08:47:05 2016 -0400
----------------------------------------------------------------------
assemble/bin/bootstrap_hdfs.sh | 21 +++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd31dc8d/assemble/bin/bootstrap_hdfs.sh
----------------------------------------------------------------------
diff --git a/assemble/bin/bootstrap_hdfs.sh b/assemble/bin/bootstrap_hdfs.sh
index 881b875..6378587 100755
--- a/assemble/bin/bootstrap_hdfs.sh
+++ b/assemble/bin/bootstrap_hdfs.sh
@@ -30,15 +30,18 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
#
# Find the system context directory in HDFS
#
-SYSTEM_CONTEXT_HDFS_DIR=$(grep -A1 "general.vfs.classpaths" "$ACCUMULO_CONF_DIR/accumulo-site.xml" | tail -1 | perl -pe 's/\s+<value>//; s/<\/value>//; s|[^/]+$||; print $ARGV[1]')
+SYSTEM_CONTEXT_HDFS_DIR=$(grep -A1 "general.vfs.classpaths" "$ACCUMULO_CONF_DIR/accumulo-site.xml" | tail -1 | perl -pe 's/\s+<value>//; s/<\/value>//; s/,.+$//; s|[^/]+$||; print $ARGV[1]')
if [ -z "$SYSTEM_CONTEXT_HDFS_DIR" ]
then
- echo "Your accumulo-site.xml file is not set up for the HDFS Classloader. Please add the following to your accumulo-site.xml file:"
+ echo "Your accumulo-site.xml file is not set up for the HDFS Classloader. Please add the following to your accumulo-site.xml file where ##CLASSPATH## is one of the following formats:"
+ echo "A single directory: hdfs://host:port/directory/"
+ echo "A single directory with a regex: hdfs://host:port/directory/.*.jar"
+ echo "Multiple directories: hdfs://host:port/directory/.*.jar,hdfs://host:port/directory2/"
echo ""
echo "<property>"
echo " <name>general.vfs.classpaths</name>"
- echo " <value>hdfs://host:port/dir</value>"
+ echo " <value>##CLASSPATH##</value>"
echo " <description>location of the jars for the default (system) context</description>"
echo "</property>"
exit 1
@@ -50,6 +53,10 @@ fi
"$HADOOP_PREFIX/bin/hadoop" fs -ls "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null
if [[ $? != 0 ]]; then
"$HADOOP_PREFIX/bin/hadoop" fs -mkdir "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null
+ if [[ $? != 0 ]]; then
+ echo "Unable to create classpath directory at $SYSTEM_CONTEXT_HDFS_DIR"
+ exit 1
+ fi
fi
#
@@ -69,12 +76,14 @@ REP=$(( NUM_SLAVES / 50 ))
"$HADOOP_PREFIX/bin/hadoop" fs -setrep -R $REP "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null
#
-# We need two of the jars in lib, copy them back out and remove them from the system context dir
+# We need some of the jars in lib, copy them back out and remove them from the system context dir
#
"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" "$ACCUMULO_HOME/lib/." > /dev/null
-"$HADOOP_PREFIX/bin/hadoop" fs -rmr "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" > /dev/null
+"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" > /dev/null
"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" "$ACCUMULO_HOME/lib/." > /dev/null
-"$HADOOP_PREFIX/bin/hadoop" fs -rmr "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" > /dev/null
+"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" > /dev/null
+"$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" "$ACCUMULO_HOME/lib/." > /dev/null
+"$HADOOP_PREFIX/bin/hadoop" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" > /dev/null
for f in $(grep -v '^#' "$ACCUMULO_CONF_DIR/slaves")
do
rsync -ra --delete "$ACCUMULO_HOME" $(dirname "$ACCUMULO_HOME")
[2/3] accumulo git commit: ACCUMULO-4341: Preload classes to avoid
deadlock in the case where classes are in HDFS
Posted by dl...@apache.org.
ACCUMULO-4341: Preload classes to avoid deadlock in the case where classes are in HDFS
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1ca7e2ea
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1ca7e2ea
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1ca7e2ea
Branch: refs/heads/1.8
Commit: 1ca7e2ea638f599beb588b738f0030632b8eb432
Parents: cd31dc8
Author: Dave Marion <dl...@apache.org>
Authored: Tue Jun 14 09:16:55 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Tue Jun 14 09:16:55 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/start/Main.java | 35 +++++++++++++++++---
1 file changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1ca7e2ea/start/src/main/java/org/apache/accumulo/start/Main.java
----------------------------------------------------------------------
diff --git a/start/src/main/java/org/apache/accumulo/start/Main.java b/start/src/main/java/org/apache/accumulo/start/Main.java
index abc0489..90eb5f4 100644
--- a/start/src/main/java/org/apache/accumulo/start/Main.java
+++ b/start/src/main/java/org/apache/accumulo/start/Main.java
@@ -42,6 +42,32 @@ public class Main {
public static void main(final String[] args) {
try {
+ // Preload classes that cause a deadlock between the ServiceLoader and the DFSClient when using
+ // the VFSClassLoader with jars in HDFS.
+ ClassLoader loader = getClassLoader();
+ Class<?> confClass = null;
+ try {
+ confClass = AccumuloClassLoader.getClassLoader().loadClass("org.apache.hadoop.conf.Configuration");
+ } catch (ClassNotFoundException e) {
+ log.error("Unable to find Hadoop Configuration class on classpath, check configuration.", e);
+ System.exit(1);
+ }
+ Object conf = null;
+ try {
+ conf = confClass.newInstance();
+ } catch (Exception e) {
+ log.error("Error creating new instance of Hadoop Configuration", e);
+ System.exit(1);
+ }
+ try {
+ Method getClassByNameOrNullMethod = conf.getClass().getMethod("getClassByNameOrNull", String.class);
+ getClassByNameOrNullMethod.invoke(conf, "org.apache.hadoop.mapred.JobConf");
+ getClassByNameOrNullMethod.invoke(conf, "org.apache.hadoop.mapred.JobConfigurable");
+ } catch (Exception e) {
+ log.error("Error pre-loading JobConf and JobConfigurable classes, VFS classloader with " + "system classes in HDFS may not work correctly", e);
+ System.exit(1);
+ }
+
if (args.length == 0) {
printUsage();
System.exit(1);
@@ -49,7 +75,7 @@ public class Main {
// determine whether a keyword was used or a class name, and execute it with the remaining args
String keywordOrClassName = args[0];
- KeywordExecutable keywordExec = getExecutables(getClassLoader()).get(keywordOrClassName);
+ KeywordExecutable keywordExec = getExecutables(loader).get(keywordOrClassName);
if (keywordExec != null) {
execKeyword(keywordExec, stripArgs(args, 1));
} else {
@@ -62,11 +88,12 @@ public class Main {
}
}
- public static ClassLoader getClassLoader() {
+ public static synchronized ClassLoader getClassLoader() {
if (classLoader == null) {
try {
ClassLoader clTmp = (ClassLoader) getVFSClassLoader().getMethod("getClassLoader").invoke(null);
classLoader = clTmp;
+ Thread.currentThread().setContextClassLoader(classLoader);
} catch (ClassNotFoundException | IOException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
| SecurityException e) {
log.error("Problem initializing the class loader", e);
@@ -76,7 +103,7 @@ public class Main {
return classLoader;
}
- public static Class<?> getVFSClassLoader() throws IOException, ClassNotFoundException {
+ public static synchronized Class<?> getVFSClassLoader() throws IOException, ClassNotFoundException {
if (vfsClassLoader == null) {
Thread.currentThread().setContextClassLoader(AccumuloClassLoader.getClassLoader());
Class<?> vfsClassLoaderTmp = AccumuloClassLoader.getClassLoader().loadClass("org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader");
@@ -183,7 +210,7 @@ public class Main {
System.out.println("accumulo " + kwString + " | <accumulo class> args");
}
- static Map<String,KeywordExecutable> getExecutables(final ClassLoader cl) {
+ static synchronized Map<String,KeywordExecutable> getExecutables(final ClassLoader cl) {
if (servicesMap == null) {
servicesMap = checkDuplicates(ServiceLoader.load(KeywordExecutable.class, cl));
}
[3/3] accumulo git commit: Merge branch '1.7' into 1.8
Posted by dl...@apache.org.
Merge branch '1.7' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/24289e2c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/24289e2c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/24289e2c
Branch: refs/heads/1.8
Commit: 24289e2c4c32850185b8649a5022b495c3205a83
Parents: a98dbec 1ca7e2e
Author: Dave Marion <dl...@apache.org>
Authored: Tue Jun 14 09:21:33 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Tue Jun 14 09:21:33 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/start/Main.java | 35 +++++++++++++++++---
1 file changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/24289e2c/start/src/main/java/org/apache/accumulo/start/Main.java
----------------------------------------------------------------------
diff --cc start/src/main/java/org/apache/accumulo/start/Main.java
index 99e1d5c,90eb5f4..414394a
--- a/start/src/main/java/org/apache/accumulo/start/Main.java
+++ b/start/src/main/java/org/apache/accumulo/start/Main.java
@@@ -183,7 -210,7 +210,7 @@@ public class Main
System.out.println("accumulo " + kwString + " | <accumulo class> args");
}
- public static Map<String,KeywordExecutable> getExecutables(final ClassLoader cl) {
- static synchronized Map<String,KeywordExecutable> getExecutables(final ClassLoader cl) {
++ public static synchronized Map<String,KeywordExecutable> getExecutables(final ClassLoader cl) {
if (servicesMap == null) {
servicesMap = checkDuplicates(ServiceLoader.load(KeywordExecutable.class, cl));
}