You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2012/11/22 01:55:28 UTC
svn commit: r1412398 [1/2] - in /accumulo/trunk: ./ bin/
conf/examples/vfs-classloader/
core/src/main/java/org/apache/accumulo/core/client/mock/
core/src/main/java/org/apache/accumulo/core/conf/
core/src/main/java/org/apache/accumulo/core/file/ core/sr...
Author: dlmarion
Date: Thu Nov 22 00:55:22 2012
New Revision: 1412398
URL: http://svn.apache.org/viewvc?rev=1412398&view=rev
Log:
ACCUMULO-708 initial implementation of VFS class loader
Added:
accumulo/trunk/conf/examples/vfs-classloader/
accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml
accumulo/trunk/core/src/test/resources/disabled/
accumulo/trunk/core/src/test/resources/disabled/conf/
accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/commons/
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileSystemConfigBuilder.java
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsReadOnlyFileContentInfoFactory.java
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsReadOnlyRandomAccessContent.java
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/ReadOnlyHdfsFileProvider.java
accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/ReadOnlyHdfsFileSystem.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/test/
accumulo/trunk/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java
accumulo/trunk/start/src/test/java/org/apache/commons/
accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/
accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/provider/
accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/provider/ReadOnlyHdfsFileProviderTest.java
accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/provider/VfsClassLoaderTest.java
accumulo/trunk/start/src/test/resources/HelloWorld.jar (with props)
accumulo/trunk/start/src/test/resources/application1/
accumulo/trunk/start/src/test/resources/application1/conf/
accumulo/trunk/start/src/test/resources/application1/conf/accumulo-site.xml
accumulo/trunk/start/src/test/resources/default/
accumulo/trunk/start/src/test/resources/default/conf/
accumulo/trunk/start/src/test/resources/default/conf/accumulo-site.xml
accumulo/trunk/start/src/test/resources/disabled/
accumulo/trunk/start/src/test/resources/disabled/conf/
accumulo/trunk/start/src/test/resources/disabled/conf/accumulo-site.xml
accumulo/trunk/test/src/test/resources/
accumulo/trunk/test/src/test/resources/conf/
accumulo/trunk/test/src/test/resources/conf/accumulo-site.xml
accumulo/trunk/test/src/test/resources/lib/
accumulo/trunk/test/src/test/resources/lib/ext/
Modified:
accumulo/trunk/bin/accumulo
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java
accumulo/trunk/pom.xml
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java
accumulo/trunk/start/pom.xml
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java
Modified: accumulo/trunk/bin/accumulo
URL: http://svn.apache.org/viewvc/accumulo/trunk/bin/accumulo?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/bin/accumulo (original)
+++ accumulo/trunk/bin/accumulo Thu Nov 22 00:55:22 2012
@@ -24,6 +24,8 @@ bin=`cd "$bin"; pwd`
START_JAR=$ACCUMULO_HOME/lib/accumulo-start-$ACCUMULO_VERSION.jar
COMMONS_JCI_JARS=$ACCUMULO_HOME/lib/commons-jci-core-1.0.jar:$ACCUMULO_HOME/lib/commons-jci-fam-1.0.jar:$ACCUMULO_HOME/lib/log4j-1.2.16.jar:$ACCUMULO_HOME/lib/commons-logging-1.0.4.jar:$ACCUMULO_HOME/lib/commons-logging-api-1.0.4.jar
START_CLASSES_DIR=$ACCUMULO_HOME/src/start/target/classes
+COMMONS_VFS_JARS=$ACCUMULO_HOME/lib/commons-vfs2-2.0.jar
+HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
locationByProgram()
{
@@ -82,7 +84,7 @@ else
fi
XML_FILES=${ACCUMULO_HOME}/conf
-CLASSPATH=${XML_FILES}:${START_JAR}:${COMMONS_JCI_JARS}
+CLASSPATH=${XML_FILES}:${START_JAR}:${COMMONS_JCI_JARS}:${COMMONS_VFS_JARS}:${HADOOP_CLASSPATH}
if [ -z $JAVA_HOME -o ! -d $JAVA_HOME ]; then
echo "JAVA_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
Added: accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml
URL: http://svn.apache.org/viewvc/accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml?rev=1412398&view=auto
==============================================================================
--- accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml (added)
+++ accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml Thu Nov 22 00:55:22 2012
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!--
+ Put your site-specific accumulo configurations here.
+
+ The available configuration values along with their defaults
+ are documented in docs/config.html
+
+ Unless you are simply testing at your workstation, you will most
+ definitely need to change the three entries below.
+ -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+ Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.password</name>
+ <!--
+ change this to the root user's password, and/or change the user below
+ -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/examples/target/classes/,
+ $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_HOME/conf,
+ $HADOOP_HOME/[^.].*.jar,
+ $HADOOP_HOME/lib/[^.].*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/" values.
+ </description>
+ </property>
+
+ <!-- VFS ClassLoader Settings -->
+ <property>
+ <name>classloader.vfs.enabled</name>
+ <value>true</value>
+ <description>set to true to enable the vfs classloader. If not enabled (default=false), then the old classloader is used</description>
+ </property>
+
+ <property>
+ <name>classloader.vfs.context.classpath.system</name>
+ <value>hdfs://localhost:8020/accumulo/classpath</value>
+ <description>location of the jars for the default (system) context</description>
+ </property>
+
+ <property>
+ <name>classloader.vfs.context.names</name>
+ <value>application1</value>
+ <description>list of context names</description>
+ </property>
+
+ <property>
+ <name>classloader.vfs.context.classpath.application1</name>
+ <value>hdfs://localhost:8020/application1/classpath</value>
+ <description>classpath for the application1 context</description>
+ </property>
+
+
+</configuration>
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java Thu Nov 22 00:55:22 2012
@@ -24,7 +24,7 @@ import org.apache.accumulo.core.client.A
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.admin.ActiveScan;
import org.apache.accumulo.core.client.admin.InstanceOperations;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
/**
*
@@ -107,7 +107,7 @@ public class MockInstanceOperations impl
@Override
public boolean testClassLoad(String className, String asTypeName) throws AccumuloException, AccumuloSecurityException {
try {
- AccumuloClassLoader.loadClass(className, Class.forName(asTypeName));
+ AccumuloVFSClassLoader.loadClass(className, Class.forName(asTypeName));
} catch (ClassNotFoundException e) {
e.printStackTrace();
return false;
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java Thu Nov 22 00:55:22 2012
@@ -286,7 +286,18 @@ public enum Property {
TABLE_FORMATTER_CLASS("table.formatter", DefaultFormatter.class.getName(), PropertyType.STRING,
"The Formatter class to apply on results in the shell"),
TABLE_INTERPRETER_CLASS("table.interepreter", DefaultScanInterpreter.class.getName(), PropertyType.STRING,
- "The ScanInterpreter class to apply on scan arguments in the shell");
+ "The ScanInterpreter class to apply on scan arguments in the shell"),
+
+
+ //VFS ClassLoader properties
+ VFS_CLASSLOADER_PREFIX("classloader.vfs", null, PropertyType.PREFIX, "Properties in this category affect the VFS ClassLoader"),
+ VFS_CLASSLOADER_ENABLED("classloader.vfs.enabled", "false", PropertyType.BOOLEAN, "Enable/disable VFS Classloader"),
+ VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.system", "", PropertyType.STRING,
+ "Classpath for the system context"),
+ VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY("classloader.vfs.context.names", "", PropertyType.STRING,
+ "Comma separated list of context names"),
+ VFS_CONTEXT_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.", null, PropertyType.PREFIX, "Classpath for this context");
+
private String key, defaultValue, description;
private PropertyType type;
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java Thu Nov 22 00:55:22 2012
@@ -51,7 +51,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.LoggingRunnable;
import org.apache.accumulo.core.util.NamingThreadFactory;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
@@ -113,7 +113,7 @@ public class BloomFilterLayer {
* load KeyFunctor
*/
try {
- Class<? extends KeyFunctor> clazz = AccumuloClassLoader.loadClass(acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), KeyFunctor.class);
+ Class<? extends KeyFunctor> clazz = AccumuloVFSClassLoader.loadClass(acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), KeyFunctor.class);
transformer = clazz.newInstance();
} catch (Exception e) {
@@ -207,7 +207,7 @@ public class BloomFilterLayer {
*/
ClassName = in.readUTF();
- Class<? extends KeyFunctor> clazz = AccumuloClassLoader.loadClass(ClassName, KeyFunctor.class);
+ Class<? extends KeyFunctor> clazz = AccumuloVFSClassLoader.loadClass(ClassName, KeyFunctor.class);
transformer = clazz.newInstance();
/**
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java Thu Nov 22 00:55:22 2012
@@ -29,7 +29,7 @@ import org.apache.accumulo.core.data.Ran
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.aggregation.Aggregator;
import org.apache.accumulo.core.iterators.conf.ColumnToClassMapping;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.log4j.Logger;
/**
@@ -193,7 +193,7 @@ public class AggregatingIterator impleme
return false;
Class<? extends Aggregator> clazz;
try {
- clazz = AccumuloClassLoader.loadClass(classname, Aggregator.class);
+ clazz = AccumuloVFSClassLoader.loadClass(classname, Aggregator.class);
clazz.newInstance();
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("class not found: " + classname);
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java Thu Nov 22 00:55:22 2012
@@ -38,7 +38,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
import org.apache.accumulo.core.tabletserver.thrift.TIteratorSetting;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.log4j.Logger;
@@ -224,7 +224,7 @@ public class IteratorUtil {
Class<? extends SortedKeyValueIterator<K,V>> clazz;
if (useAccumuloClassLoader){
- clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloClassLoader.loadClass(iterInfo.className, SortedKeyValueIterator.class);
+ clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloVFSClassLoader.loadClass(iterInfo.className, SortedKeyValueIterator.class);
}else{
clazz = (Class<? extends SortedKeyValueIterator<K,V>>) Class.forName(iterInfo.className).asSubclass(SortedKeyValueIterator.class);
}
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Thu Nov 22 00:55:22 2012
@@ -24,7 +24,7 @@ import java.util.NoSuchElementException;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
/**
* A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value.
@@ -132,7 +132,7 @@ public abstract class TypedValueCombiner
protected void setEncoder(String encoderClass) {
try {
@SuppressWarnings("unchecked")
- Class<? extends Encoder<V>> clazz = (Class<? extends Encoder<V>>) AccumuloClassLoader.loadClass(encoderClass, Encoder.class);
+ Class<? extends Encoder<V>> clazz = (Class<? extends Encoder<V>>) AccumuloVFSClassLoader.loadClass(encoderClass, Encoder.class);
encoder = clazz.newInstance();
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException(e);
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java Thu Nov 22 00:55:22 2012
@@ -24,7 +24,7 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.iterators.conf.ColumnUtil.ColFamHashKey;
import org.apache.accumulo.core.iterators.conf.ColumnUtil.ColHashKey;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.hadoop.io.Text;
public class ColumnToClassMapping<K> {
@@ -50,7 +50,7 @@ public class ColumnToClassMapping<K> {
Pair<Text,Text> pcic = ColumnSet.decodeColumns(column);
- Class<? extends K> clazz = AccumuloClassLoader.loadClass(className, c);
+ Class<? extends K> clazz = AccumuloVFSClassLoader.loadClass(className, c);
if (pcic.getSecond() == null) {
addObject(pcic.getFirst(), clazz.newInstance());
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java Thu Nov 22 00:55:22 2012
@@ -18,13 +18,13 @@ package org.apache.accumulo.core.util.sh
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.commons.cli.CommandLine;
public class ClasspathCommand extends Command {
@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) {
- AccumuloClassLoader.printClassPath();
+ AccumuloVFSClassLoader.printClassPath();
return 0;
}
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java Thu Nov 22 00:55:22 2012
@@ -39,7 +39,7 @@ import org.apache.accumulo.core.util.int
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
import org.apache.accumulo.core.util.shell.Shell.PrintFile;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
@@ -166,9 +166,9 @@ public class ScanCommand extends Command
Class<? extends ScanInterpreter> clazz = null;
try {
if (cl.hasOption(interpreterOpt.getOpt())) {
- clazz = AccumuloClassLoader.loadClass(cl.getOptionValue(interpreterOpt.getOpt()), ScanInterpreter.class);
+ clazz = AccumuloVFSClassLoader.loadClass(cl.getOptionValue(interpreterOpt.getOpt()), ScanInterpreter.class);
} else if (cl.hasOption(formatterInterpeterOpt.getOpt())) {
- clazz = AccumuloClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), ScanInterpreter.class);
+ clazz = AccumuloVFSClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), ScanInterpreter.class);
}
} catch (ClassNotFoundException e) {
shellState.getReader().printString("Interpreter class could not be loaded.\n" + e.getMessage() + "\n");
@@ -187,10 +187,10 @@ public class ScanCommand extends Command
try {
if (cl.hasOption(formatterOpt.getOpt())) {
- return AccumuloClassLoader.loadClass(cl.getOptionValue(formatterOpt.getOpt()), Formatter.class);
+ return AccumuloVFSClassLoader.loadClass(cl.getOptionValue(formatterOpt.getOpt()), Formatter.class);
} else if (cl.hasOption(formatterInterpeterOpt.getOpt())) {
- return AccumuloClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), Formatter.class);
+ return AccumuloVFSClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), Formatter.class);
}
} catch (ClassNotFoundException e) {
shellState.getReader().printString("Formatter class could not be loaded.\n" + e.getMessage() + "\n");
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java Thu Nov 22 00:55:22 2012
@@ -43,7 +43,7 @@ import org.apache.accumulo.core.util.she
import org.apache.accumulo.core.util.shell.Shell.Command;
import org.apache.accumulo.core.util.shell.ShellCommandException;
import org.apache.accumulo.core.util.shell.ShellCommandException.ErrorCode;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionGroup;
@@ -123,7 +123,7 @@ public class SetIterCommand extends Comm
OptionDescriber skvi;
Class<? extends OptionDescriber> clazz;
try {
- clazz = AccumuloClassLoader.loadClass(className, OptionDescriber.class);
+ clazz = AccumuloVFSClassLoader.loadClass(className, OptionDescriber.class);
skvi = clazz.newInstance();
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException(e.getMessage());
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java Thu Nov 22 00:55:22 2012
@@ -25,7 +25,7 @@ import org.apache.accumulo.core.client.T
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionGroup;
@@ -102,7 +102,7 @@ public abstract class ShellPluginConfigu
if (ent.getKey().equals(pluginProp.toString())) {
Class<? extends T> pluginClazz;
try {
- pluginClazz = AccumuloClassLoader.loadClass(ent.getValue(), clazz);
+ pluginClazz = AccumuloVFSClassLoader.loadClass(ent.getValue(), clazz);
} catch (ClassNotFoundException e) {
Logger.getLogger(ShellPluginConfigurationCommand.class).warn("Class not found" + e.getMessage());
return null;
Added: accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml?rev=1412398&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml (added)
+++ accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml Thu Nov 22 00:55:22 2012
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!-- Put your site-specific accumulo configurations here. The available
+ configuration values along with their defaults are documented in docs/config.html
+ Unless you are simply testing at your workstation, you will most definitely
+ need to change the three entries below. -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The directory used to store write-ahead logs on the local
+ filesystem. It is possible to specify a comma-separated list of
+ directories.</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers must
+ know in order to communicate with one another.
+ Change it before initialization. To change it later use ./bin/accumulo
+ org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.password</name>
+ <!-- change this to the root user's password, and/or change the user below -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/examples/target/classes/,
+ $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_HOME/conf,
+ $HADOOP_HOME/[^.].*.jar,
+ $HADOOP_HOME/lib/[^.].*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class
+ files.
+ When using the Security Manager, please remove the ".../target/classes/"
+ values.
+ </description>
+ </property>
+
+ <!-- VFS ClassLoader Settings -->
+ <property>
+ <name>vfs.classloader.enabled</name>
+ <value>true</value>
+ </property>
+
+</configuration>
Modified: accumulo/trunk/pom.xml
URL: http://svn.apache.org/viewvc/accumulo/trunk/pom.xml?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/pom.xml (original)
+++ accumulo/trunk/pom.xml Thu Nov 22 00:55:22 2012
@@ -144,7 +144,7 @@
<configuration>
<outputDirectory>../lib</outputDirectory>
<!-- just grab the non-provided runtime dependencies -->
- <includeArtifactIds>commons-collections,commons-configuration,commons-io,commons-lang,jline,log4j,libthrift,commons-jci-core,commons-jci-fam,commons-logging,commons-logging-api</includeArtifactIds>
+ <includeArtifactIds>commons-collections,commons-configuration,commons-io,commons-lang,jline,log4j,libthrift,commons-jci-core,commons-jci-fam,commons-logging,commons-logging-api,commons-vfs2</includeArtifactIds>
<excludeTransitive>true</excludeTransitive>
</configuration>
</execution>
@@ -197,7 +197,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<environmentVariables>
- <ACCUMULO_HOME>..</ACCUMULO_HOME>
+ <ACCUMULO_HOME>../test/src/test/resources</ACCUMULO_HOME>
</environmentVariables>
</configuration>
</plugin>
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java Thu Nov 22 00:55:22 2012
@@ -47,7 +47,7 @@ import org.apache.accumulo.server.conf.S
import org.apache.accumulo.server.security.Authenticator;
import org.apache.accumulo.server.security.ZKAuthenticator;
import org.apache.accumulo.server.zookeeper.TransactionWatcher;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
@@ -279,7 +279,7 @@ public class ClientServiceHandler implem
Class shouldMatch;
try {
shouldMatch = loader.loadClass(interfaceMatch);
- Class test = AccumuloClassLoader.loadClass(className, shouldMatch);
+ Class test = AccumuloVFSClassLoader.loadClass(className, shouldMatch);
test.newInstance();
return true;
} catch (ClassCastException e) {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java Thu Nov 22 00:55:22 2012
@@ -22,7 +22,7 @@ import java.util.Map.Entry;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.constraints.Constraint;
import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.log4j.Logger;
public class ConstraintLoader {
@@ -35,7 +35,7 @@ public class ConstraintLoader {
for (Entry<String,String> entry : conf) {
if (entry.getKey().startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey())) {
String className = entry.getValue();
- Class<? extends Constraint> clazz = AccumuloClassLoader.loadClass(className, Constraint.class);
+ Class<? extends Constraint> clazz = AccumuloVFSClassLoader.loadClass(className, Constraint.class);
log.debug("Loaded constraint " + clazz.getName() + " for " + conf.getTableId());
cc.addConstraint(clazz.newInstance());
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java Thu Nov 22 00:55:22 2012
@@ -157,7 +157,7 @@ import org.apache.accumulo.server.util.T
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -513,7 +513,7 @@ public class Master implements LiveTServ
T instance = null;
try {
- Class<? extends T> clazz = AccumuloClassLoader.loadClass(clazzName, base);
+ Class<? extends T> clazz = AccumuloVFSClassLoader.loadClass(clazzName, base);
instance = clazz.newInstance();
log.info("Loaded class : " + clazzName);
} catch (Exception e) {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java Thu Nov 22 00:55:22 2012
@@ -35,7 +35,7 @@ import org.apache.accumulo.server.client
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
import org.apache.accumulo.server.security.SecurityConstants;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.log4j.Logger;
public class TableLoadBalancer extends TabletBalancer {
@@ -45,7 +45,7 @@ public class TableLoadBalancer extends T
Map<String,TabletBalancer> perTableBalancers = new HashMap<String,TabletBalancer>();
private TabletBalancer constructNewBalancerForTable(String clazzName, String table) throws Exception {
- Class<? extends TabletBalancer> clazz = AccumuloClassLoader.loadClass(clazzName, TabletBalancer.class);
+ Class<? extends TabletBalancer> clazz = AccumuloVFSClassLoader.loadClass(clazzName, TabletBalancer.class);
Constructor<? extends TabletBalancer> constructor = clazz.getConstructor(String.class);
return constructor.newInstance(table);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java Thu Nov 22 00:55:22 2012
@@ -45,14 +45,14 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.file.blockfile.cache.LruBlockCache;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.LoggingRunnable;
-import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.MetadataTable.DataFileValue;
+import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.tabletserver.FileManager.ScanFileManager;
import org.apache.accumulo.server.tabletserver.Tablet.MajorCompactionReason;
import org.apache.accumulo.server.util.time.SimpleTimer;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
@@ -199,7 +199,7 @@ public class TabletServerResourceManager
fileManager = new FileManager(conf, fs, maxOpenFiles, _dCache, _iCache);
try {
- Class<? extends MemoryManager> clazz = AccumuloClassLoader.loadClass(acuConf.get(Property.TSERV_MEM_MGMT), MemoryManager.class);
+ Class<? extends MemoryManager> clazz = AccumuloVFSClassLoader.loadClass(acuConf.get(Property.TSERV_MEM_MGMT), MemoryManager.class);
memoryManager = clazz.newInstance();
memoryManager.init(conf);
log.debug("Loaded memory manager : " + memoryManager.getClass().getName());
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java Thu Nov 22 00:55:22 2012
@@ -42,7 +42,7 @@ import org.apache.accumulo.core.data.Ran
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
@@ -290,7 +290,7 @@ public abstract class FunctionalTest {
String clazz = remainingArgs[0];
String opt = remainingArgs[1];
- Class<? extends FunctionalTest> testClass = AccumuloClassLoader.loadClass(clazz, FunctionalTest.class);
+ Class<? extends FunctionalTest> testClass = AccumuloVFSClassLoader.loadClass(clazz, FunctionalTest.class);
FunctionalTest fTest = testClass.newInstance();
fTest.setMaster(master);
Modified: accumulo/trunk/start/pom.xml
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/pom.xml?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/start/pom.xml (original)
+++ accumulo/trunk/start/pom.xml Thu Nov 22 00:55:22 2012
@@ -46,6 +46,13 @@
</archive>
</configuration>
</plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkMode>always</forkMode>
+ </configuration>
+ </plugin>
+
</plugins>
</pluginManagement>
</build>
@@ -67,6 +74,53 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-vfs2</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <!-- TEST DEPENDENCIES -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-test</artifactId>
+ <version>0.20.205.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ <version>1.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-nop</artifactId>
+ <version>1.7.2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>1.4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.1</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java?rev=1412398&r1=1412397&r2=1412398&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java (original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java Thu Nov 22 00:55:22 2012
@@ -19,7 +19,7 @@ package org.apache.accumulo.start;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
public class Main {
@@ -36,36 +36,37 @@ public class Main {
Class<?> runTMP = null;
- Thread.currentThread().setContextClassLoader(AccumuloClassLoader.getClassLoader());
+ ClassLoader cl = AccumuloVFSClassLoader.getClassLoader();
+ Thread.currentThread().setContextClassLoader(cl);
if (args[0].equals("master")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.master.Master");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.master.Master");
} else if (args[0].equals("tserver")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.tabletserver.TabletServer");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.tabletserver.TabletServer");
} else if (args[0].equals("shell")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.util.shell.Shell");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.core.util.shell.Shell");
} else if (args[0].equals("init")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.util.Initialize");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.util.Initialize");
} else if (args[0].equals("admin")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.util.Admin");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.util.Admin");
} else if (args[0].equals("gc")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.gc.SimpleGarbageCollector");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.gc.SimpleGarbageCollector");
} else if (args[0].equals("monitor")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.monitor.Monitor");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.monitor.Monitor");
} else if (args[0].equals("tracer")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.trace.TraceServer");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.trace.TraceServer");
} else if (args[0].equals("classpath")) {
- AccumuloClassLoader.printClassPath();
+ AccumuloVFSClassLoader.printClassPath();
return;
} else if (args[0].equals("version")) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.Constants");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.core.Constants");
System.out.println(runTMP.getField("VERSION").get(null));
return;
} else if (args[0].equals("rfile-info") ) {
- runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.file.rfile.PrintInfo");
+ runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.core.file.rfile.PrintInfo");
} else {
try {
- runTMP = AccumuloClassLoader.loadClass(args[0]);
+ runTMP = AccumuloVFSClassLoader.loadClass(args[0]);
} catch (ClassNotFoundException cnfe) {
System.out.println("Classname " + args[0] + " not found. Please make sure you use the wholly qualified package name.");
System.exit(1);
@@ -96,7 +97,8 @@ public class Main {
};
Thread t = new Thread(r, args[0]);
- t.setContextClassLoader(AccumuloClassLoader.getClassLoader());
+ ClassLoader tcl = AccumuloVFSClassLoader.getClassLoader();
+ t.setContextClassLoader(tcl);
t.start();
} catch (Throwable t) {
System.err.println("Uncaught exception: " + t.getMessage());
Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java?rev=1412398&view=auto
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java (added)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java Thu Nov 22 00:55:22 2012
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.security.SecureClassLoader;
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.log4j.Logger;
+
+/**
+ * Classloader that delegates requests to a classloader mapped to a specific context
+ *
+ *
+ */
+public class AccumuloContextClassLoader extends SecureClassLoader {
+
+ private static final Logger log = Logger.getLogger(AccumuloContextClassLoader.class);
+
+ public static final String DEFAULT_CONTEXT = "SYSTEM";
+
+ private Map<String, AccumuloReloadingVFSClassLoader> loaders = new ConcurrentHashMap<String, AccumuloReloadingVFSClassLoader>();
+ private FileSystemManager vfs = null;
+ private ClassLoader parent = null;
+
+ public AccumuloContextClassLoader(FileObject[] defaultClassPath, FileSystemManager vfs, ClassLoader parent) throws FileSystemException {
+ this.vfs = vfs;
+ this.parent = parent;
+ loaders.put(DEFAULT_CONTEXT, new AccumuloReloadingVFSClassLoader(defaultClassPath, vfs, parent));
+ }
+
+ public synchronized void addContext(String context, FileObject[] files) throws FileSystemException {
+ if (!loaders.containsKey(context))
+ loaders.put(context, new AccumuloReloadingVFSClassLoader(files, vfs, parent));
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ return loadClass(DEFAULT_CONTEXT, name);
+ }
+
+ public Class<?> loadClass(String context, String name) throws ClassNotFoundException {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ return loaders.get(context).loadClass(name);
+ }
+
+ @Override
+ public URL getResource(String name) {
+ return getResource(DEFAULT_CONTEXT, name);
+ }
+
+ public URL getResource(String context, String name) {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ return loaders.get(context).getResource(name);
+ }
+
+ @Override
+ public Enumeration<URL> getResources(String name) throws IOException {
+ return getResources(DEFAULT_CONTEXT, name);
+ }
+
+ public Enumeration<URL> getResources(String context, String name) throws IOException {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ return loaders.get(context).getResources(name);
+ }
+
+ @Override
+ public InputStream getResourceAsStream(String name) {
+ return getResourceAsStream(DEFAULT_CONTEXT, name);
+ }
+
+ public InputStream getResourceAsStream(String context, String name) {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ return loaders.get(context).getResourceAsStream(name);
+ }
+
+ @Override
+ public synchronized void setDefaultAssertionStatus(boolean enabled) {
+ setDefaultAssertionStatus(DEFAULT_CONTEXT, enabled);
+ }
+
+ public synchronized void setDefaultAssertionStatus(String context, boolean enabled) {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ loaders.get(context).setDefaultAssertionStatus(enabled);
+ }
+
+ @Override
+ public synchronized void setPackageAssertionStatus(String packageName, boolean enabled) {
+ setPackageAssertionStatus(DEFAULT_CONTEXT, packageName, enabled);
+ }
+
+ public synchronized void setPackageAssertionStatus(String context, String packageName, boolean enabled) {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ loaders.get(context).setPackageAssertionStatus(packageName, enabled);
+ }
+
+ @Override
+ public synchronized void setClassAssertionStatus(String className, boolean enabled) {
+ setClassAssertionStatus(DEFAULT_CONTEXT, className, enabled);
+ }
+
+ public synchronized void setClassAssertionStatus(String context, String className, boolean enabled) {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ loaders.get(context).setClassAssertionStatus(className, enabled);
+ }
+
+ @Override
+ public synchronized void clearAssertionStatus() {
+ clearAssertionStatus(DEFAULT_CONTEXT);
+ }
+
+ public synchronized void clearAssertionStatus(String context) {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ loaders.get(context).clearAssertionStatus();
+ }
+
+ public AccumuloReloadingVFSClassLoader getClassLoader(String context) {
+ return loaders.get(context);
+ }
+
+ public void removeContext(String context) {
+ if (null != context && !(DEFAULT_CONTEXT.equals(context))) {
+ AccumuloReloadingVFSClassLoader cl = loaders.remove(context);
+ cl.close();
+ }
+ }
+
+ public void close() {
+ for (Entry<String, AccumuloReloadingVFSClassLoader> e : loaders.entrySet()) {
+ e.getValue().close();
+ }
+ loaders.clear();
+ }
+
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ return this.loadClass(DEFAULT_CONTEXT, name);
+ }
+
+ @Override
+ protected Class<?> findClass(String name) throws ClassNotFoundException {
+ return findClass(DEFAULT_CONTEXT, name);
+ }
+
+ protected Class<?> findClass(String context, String name) throws ClassNotFoundException {
+ if (null == context)
+ context = DEFAULT_CONTEXT;
+ return loaders.get(context).findClass(name);
+ }
+
+ @Override
+ public String toString() {
+ URLClassLoader ucl = (URLClassLoader) this.parent;
+ URL[] uclPaths = ucl.getURLs();
+ StringBuilder buf = new StringBuilder();
+ buf.append("Parent classpath {\n");
+ for (URL u : uclPaths)
+ buf.append("\t").append(u.toString()).append("\n");
+ buf.append("}\nContext classpaths {\n");
+ for (Entry<String, AccumuloReloadingVFSClassLoader> entry : this.loaders.entrySet()) {
+ buf.append("\tContext: ").append(entry.getKey()).append(" {\n");
+ for (FileObject f : entry.getValue().getFiles()) {
+ try {
+ buf.append("\t\t").append(f.getURL().toString()).append("\n");
+ } catch (FileSystemException e) {
+ log.error("Error getting URL for file", e);
+ }
+ }
+ buf.append("\t}\n");
+ }
+ buf.append("}");
+ return buf.toString();
+ }
+
+}
Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1412398&view=auto
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java (added)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java Thu Nov 22 00:55:22 2012
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.security.SecureClassLoader;
+import java.util.Arrays;
+import java.util.Enumeration;
+
+import org.apache.commons.vfs2.FileChangeEvent;
+import org.apache.commons.vfs2.FileListener;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.impl.DefaultFileMonitor;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+import org.apache.log4j.Logger;
+
+/**
+ * Classloader that delegates operations to a VFSClassLoader object. This class also listens
+ * for changes in any of the files/directories that are in the classpath and will recreate
+ * the delegate object if there is any change in the classpath.
+ *
+ */
+public class AccumuloReloadingVFSClassLoader extends SecureClassLoader implements FileListener {
+
+ private static final Logger log = Logger.getLogger(AccumuloReloadingVFSClassLoader.class);
+
+ /* 5 minute timeout */
+ private static final int DEFAULT_TIMEOUT = 300000;
+
+ private FileObject[] files = null;
+ private FileSystemManager vfs = null;
+ private ClassLoader parent = null;
+ private DefaultFileMonitor monitor = null;
+ private volatile VFSClassLoader cl = null;
+
+ public AccumuloReloadingVFSClassLoader(FileObject[] files, FileSystemManager vfs, ClassLoader parent) throws FileSystemException {
+ this(files, vfs, parent, DEFAULT_TIMEOUT);
+ }
+
+ public AccumuloReloadingVFSClassLoader(FileObject[] files, FileSystemManager vfs, ClassLoader parent, long monitorDelay) throws FileSystemException {
+ this.files = files;
+ this.vfs = vfs;
+ this.parent = parent;
+
+ if (null != parent)
+ cl = new VFSClassLoader(files, vfs, parent);
+ else
+ cl = new VFSClassLoader(files, vfs);
+
+ monitor = new DefaultFileMonitor(this);
+ monitor.setDelay(monitorDelay);
+ monitor.setRecursive(true);
+ for (FileObject file : files)
+ monitor.addFile(file);
+ monitor.start();
+ }
+
+ public FileObject[] getFiles() {
+ return this.files;
+ }
+
+ /**
+ * Should be ok if this is not called because the thread started by DefaultFileMonitor is a daemon thread
+ */
+ public void close() {
+ monitor.stop();
+ }
+
+ public void fileCreated(FileChangeEvent event) throws Exception {
+ if (log.isDebugEnabled())
+ log.debug(event.getFile().getURL().toString() + " created, recreating classloader");
+ cl = new VFSClassLoader(files, vfs, parent);
+ }
+
+ public void fileDeleted(FileChangeEvent event) throws Exception {
+ if (log.isDebugEnabled())
+ log.debug(event.getFile().getURL().toString() + " changed, recreating classloader");
+ cl = new VFSClassLoader(files, vfs, parent);
+ }
+
+ public void fileChanged(FileChangeEvent event) throws Exception {
+ if (log.isDebugEnabled())
+ log.debug(event.getFile().getURL().toString() + " deleted, recreating classloader");
+ cl = new VFSClassLoader(files, vfs, parent);
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ return this.cl.loadClass(name);
+ }
+
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ return this.loadClass(name);
+ }
+
+ @Override
+ protected Class<?> findClass(String name) throws ClassNotFoundException {
+ return this.loadClass(name);
+ }
+
+ @Override
+ public URL getResource(String name) {
+ return this.cl.getResource(name);
+ }
+
+ @Override
+ public Enumeration<URL> getResources(String name) throws IOException {
+ return this.cl.getResources(name);
+ }
+
+ @Override
+ public InputStream getResourceAsStream(String name) {
+ return this.cl.getResourceAsStream(name);
+ }
+
+ @Override
+ public synchronized void setDefaultAssertionStatus(boolean enabled) {
+ this.cl.setDefaultAssertionStatus(enabled);
+ }
+
+ @Override
+ public synchronized void setPackageAssertionStatus(String packageName, boolean enabled) {
+ this.cl.setPackageAssertionStatus(packageName, enabled);
+ }
+
+ @Override
+ public synchronized void setClassAssertionStatus(String className, boolean enabled) {
+ this.cl.setClassAssertionStatus(className, enabled);
+ }
+
+ @Override
+ public synchronized void clearAssertionStatus() {
+ this.cl.clearAssertionStatus();
+ }
+
+ @Override
+ public String toString() {
+ return Arrays.toString(this.files);
+ }
+
+}
Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1412398&view=auto
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java (added)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java Thu Nov 22 00:55:22 2012
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.commons.vfs2.CacheStrategy;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileType;
+import org.apache.commons.vfs2.cache.DefaultFilesCache;
+import org.apache.commons.vfs2.cache.SoftRefFilesCache;
+import org.apache.commons.vfs2.impl.DefaultFileReplicator;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
+import org.apache.commons.vfs2.provider.ReadOnlyHdfsFileProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+
+/**
+ * This class builds a hierarchy of Classloaders in the form of:
+ *
+ * <pre>
+ * SystemClassLoader that loads JVM classes
+ * ^
+ * |
+ * URLClassLoader that references the URLs for HADOOP_HOME, ZOOKEEPER_HOME, ACCUMULO_HOME and their associated directories
+ * ^
+ * |
+ * AccumuloContextClassLoader that contains a map of context names to AccumuloReloadingVFSClassLoaders
+ * </pre>
+ *
+ * This class requires new properties in the site configuration file
+ *
+ * default.context.classpath -> list of URIs for the default system context.
+ * classloader.context.names -> name1, name2, name3
+ * <name>.context.classpath -> list of URIs for this context
+ *
+ */
+public class AccumuloVFSClassLoader {
+
+ public static class AccumuloVFSClassLoaderShutdownThread implements Runnable {
+
+ public void run() {
+ AccumuloVFSClassLoader.close();
+ }
+
+ }
+
+ private static final Logger log = Logger.getLogger(AccumuloVFSClassLoader.class);
+
+ public static final String VFS_CLASSLOADER_ENABLED = "classloader.vfs.enabled";
+
+ public static final String VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY = "classloader.vfs.context.classpath.system";
+
+ public static final String VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY = "classloader.vfs.context.names";
+
+ public static final String VFS_CONTEXT_CLASSPATH_PROPERTY = "classloader.vfs.context.classpath.";
+
+ private static DefaultFileSystemManager vfs = null;
+ private static ClassLoader parent = null;
+ private static volatile AccumuloContextClassLoader loader = null;
+ private static final Object lock = new Object();
+ private static final Configuration ACC_CONF = new Configuration();
+ private static final String SITE_CONF;
+ static {
+ String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
+ if (System.getenv("ACCUMULO_HOME") != null) {
+ // accumulo home should be set
+ SITE_CONF = System.getenv("ACCUMULO_HOME") + "/conf/" + configFile;
+ } else {
+ /*
+ * if not it must be the build-server in which case I use a hack to get unittests working
+ */
+ String userDir = System.getProperty("user.dir");
+ if (userDir == null)
+ throw new RuntimeException("Property user.dir is not set");
+ int index = userDir.indexOf("accumulo/");
+ if (index >= 0) {
+ String acuhome = userDir.substring(0, index + "accumulo/".length());
+ SITE_CONF = acuhome + "/conf/" + configFile;
+ } else {
+ SITE_CONF = "/conf/" + configFile;
+ }
+ }
+ try {
+ ACC_CONF.addResource(new File(SITE_CONF).toURI().toURL());
+ } catch (MalformedURLException e) {
+ throw new RuntimeException("Unable to create configuration from accumulo-site file: " + SITE_CONF, e);
+ }
+
+ //Register the shutdown hook
+ Runtime.getRuntime().addShutdownHook(new Thread(new AccumuloVFSClassLoaderShutdownThread()));
+ }
+
+ /**
+ * Replace environment variables in the classpath string with their actual value
+ *
+ * @param classpath
+ * @param env
+ * @return
+ */
+ private static String replaceEnvVars(String classpath, Map<String,String> env) {
+ Pattern envPat = Pattern.compile("\\$[A-Za-z][a-zA-Z0-9_]*");
+ Matcher envMatcher = envPat.matcher(classpath);
+ while (envMatcher.find(0)) {
+ // name comes after the '$'
+ String varName = envMatcher.group().substring(1);
+ String varValue = env.get(varName);
+ if (varValue == null) {
+ varValue = "";
+ }
+ classpath = (classpath.substring(0, envMatcher.start()) + varValue + classpath.substring(envMatcher.end()));
+ envMatcher.reset(classpath);
+ }
+ return classpath;
+ }
+
+ /**
+ * Populate the list of URLs with the items in the classpath string
+ *
+ * @param classpath
+ * @param urls
+ * @throws MalformedURLException
+ */
+ private static void addUrl(String classpath, ArrayList<URL> urls) throws MalformedURLException {
+ classpath = classpath.trim();
+ if (classpath.length() == 0)
+ return;
+
+ classpath = replaceEnvVars(classpath, System.getenv());
+
+ // Try to make a URI out of the classpath
+ URI uri = null;
+ try {
+ uri = new URI(classpath);
+ } catch (URISyntaxException e) {
+ // Not a valid URI
+ }
+
+ if (null == uri || !uri.isAbsolute() || (null != uri.getScheme() && uri.getScheme().equals("file://"))) {
+ // Then treat this URI as a File.
+ // This checks to see if the url string is a dir if it expand and get all jars in that directory
+ final File extDir = new File(classpath);
+ if (extDir.isDirectory())
+ urls.add(extDir.toURI().toURL());
+ else {
+ if (extDir.getParentFile() != null) {
+ File[] extJars = extDir.getParentFile().listFiles(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.matches("^" + extDir.getName());
+ }
+ });
+ if (extJars != null && extJars.length > 0) {
+ for (File jar : extJars)
+ urls.add(jar.toURI().toURL());
+ }
+ }
+ }
+ } else {
+ urls.add(uri.toURL());
+ }
+
+ }
+
+ private static ArrayList<URL> findAccumuloURLs() throws IOException {
+ String cp = ACC_CONF.get(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, AccumuloClassLoader.ACCUMULO_CLASSPATH_VALUE);
+ if (cp == null)
+ return new ArrayList<URL>();
+ String[] cps = replaceEnvVars(cp, System.getenv()).split(",");
+ ArrayList<URL> urls = new ArrayList<URL>();
+ for (String classpath : cps) {
+ if (!classpath.startsWith("#")) {
+ addUrl(classpath, urls);
+ }
+ }
+ return urls;
+ }
+
+ private static ClassLoader getAccumuloClassLoader() throws IOException {
+ ClassLoader parentClassLoader = ClassLoader.getSystemClassLoader();
+ ArrayList<URL> accumuloURLs = findAccumuloURLs();
+ log.debug("Create 2nd tier ClassLoader using URLs: " + accumuloURLs.toString());
+ URLClassLoader aClassLoader = new URLClassLoader(accumuloURLs.toArray(new URL[accumuloURLs.size()]), parentClassLoader);
+ return aClassLoader;
+ }
+
+ public synchronized static <U> Class<? extends U> loadClass(String classname, Class<U> extension) throws ClassNotFoundException {
+ try {
+ return (Class<? extends U>) getClassLoader().loadClass(classname).asSubclass(extension);
+ } catch (IOException e) {
+ throw new ClassNotFoundException("IO Error loading class " + classname, e);
+ }
+ }
+
+ public static Class<?> loadClass(String classname) throws ClassNotFoundException {
+ return loadClass(classname, Object.class).asSubclass(Object.class);
+ }
+
+ public static ClassLoader getClassLoader() throws IOException {
+ ClassLoader localLoader = loader;
+ while (null == localLoader) {
+ synchronized (lock) {
+ if (null == loader) {
+
+ if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) {
+ localLoader = AccumuloClassLoader.getClassLoader();
+ return localLoader;
+ }
+
+ if (null == vfs) {
+ vfs = new DefaultFileSystemManager();
+ //TODO: Might be able to use a different cache impl or specify cache directory in configuration.
+ vfs.setFilesCache(new DefaultFilesCache());
+ vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
+ vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
+ vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
+ vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
+ vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
+ vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
+ vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
+ vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
+ vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
+ vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
+ vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
+ vfs.addProvider("hdfs", new ReadOnlyHdfsFileProvider());
+ vfs.addExtensionMap("jar", "jar");
+ vfs.addExtensionMap("zip", "zip");
+ vfs.addExtensionMap("gz", "gz");
+ vfs.addExtensionMap("tar", "tar");
+ vfs.addExtensionMap("tbz2", "tar");
+ vfs.addExtensionMap("tgz", "tar");
+ vfs.addExtensionMap("bz2", "bz2");
+ vfs.addMimeTypeMap("application/x-tar", "tar");
+ vfs.addMimeTypeMap("application/x-gzip", "gz");
+ vfs.addMimeTypeMap("application/zip", "zip");
+ vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
+ vfs.setFilesCache(new SoftRefFilesCache());
+ vfs.setReplicator(new DefaultFileReplicator());
+ vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
+ vfs.init();
+ }
+
+ //Set up the 2nd tier class loader
+ if (null == parent)
+ parent = getAccumuloClassLoader();
+
+ //Get the default context classpaths from the configuration
+ String[] defaultPaths = ACC_CONF.getStrings(VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY);
+ if (null == defaultPaths || defaultPaths.length == 0) {
+ log.info("Default context not configured.");
+ localLoader = parent;
+ return localLoader;
+ }
+ ArrayList<FileObject> defaultClassPath = new ArrayList<FileObject>();
+ for (String path : defaultPaths) {
+ FileObject fo = vfs.resolveFile(path);
+ if (fo.getType().equals(FileType.FILE)) {
+ defaultClassPath.add(fo);
+ } else {
+ for (FileObject child : fo.getChildren()) {
+ defaultClassPath.add(child);
+ }
+ }
+ }
+
+ //Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
+ loader = new AccumuloContextClassLoader(defaultClassPath.toArray(new FileObject[defaultClassPath.size()]), vfs, parent);
+
+ //Add the other contexts
+ String[] contexts = ACC_CONF.getStrings(VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY);
+ if (null != contexts) {
+ for (String context : contexts) {
+ String[] contextPaths = ACC_CONF.getStrings(VFS_CONTEXT_CLASSPATH_PROPERTY + context);
+ if (null != contextPaths) {
+ ArrayList<FileObject> contextClassPath = new ArrayList<FileObject>();
+ for (String cp : contextPaths) {
+ FileObject fo = vfs.resolveFile(cp);
+ if (fo.getType().equals(FileType.FILE)) {
+ contextClassPath.add(fo);
+ } else {
+ for (FileObject child : fo.getChildren()) {
+ contextClassPath.add(child);
+ }
+ }
+ }
+ log.debug("Creating Context ClassLoader for context: " + context + " using paths: " + contextClassPath.toString());
+ loader.addContext(context, contextClassPath.toArray(new FileObject[contextClassPath.size()]));
+ }
+ }
+ }
+ }
+ }
+ localLoader = loader;
+ }
+ return localLoader;
+ }
+
+ public static void printClassPath() {
+ try {
+ ClassLoader cl = getClassLoader();
+ if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) {
+ //If using older classloader, then use its printClassPath method
+ AccumuloClassLoader.printClassPath();
+ } else if (cl instanceof URLClassLoader) {
+ //If VFS class loader enabled, but no contexts defined.
+ URLClassLoader ucl = (URLClassLoader) cl;
+ System.out.println("URL classpath items are: \n");
+ for (URL u : ucl.getURLs()) {
+ System.out.println(u.toExternalForm());
+ }
+ } else if (cl instanceof AccumuloContextClassLoader) {
+ //If VFS class loader enabled and contexts are defined
+ System.out.println("VFS classpaths items are:\n" + getClassLoader().toString());
+ } else {
+ System.out.println("Unknown classloader configuration");
+ }
+ } catch (Throwable t) {
+ throw new RuntimeException(t);
+ }
+ }
+
+
+ public static void close() {
+ if (null != vfs)
+ vfs.close();
+ }
+
+}
Added: accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java?rev=1412398&view=auto
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java (added)
+++ accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java Thu Nov 22 00:55:22 2012
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.vfs2.provider;
+
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileType;
+import org.apache.commons.vfs2.RandomAccessContent;
+import org.apache.commons.vfs2.util.RandomAccessMode;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class HdfsFileObject extends AbstractFileObject {
+
+ public static final String LAST_ACCESS_TIME = "LAST_ACCESS_TIME";
+ public static final String BLOCK_SIZE = "BLOCK_SIZE";
+ public static final String GROUP = "GROUP";
+ public static final String OWNER = "OWNER";
+ public static final String PERMISSIONS = "PERMISSIONS";
+ public static final String LENGTH = "LENGTH";
+ public static final String MODIFICATION_TIME = "MODIFICATION_TIME";
+
+ private ReadOnlyHdfsFileSystem fs = null;
+ private FileSystem hdfs = null;
+ private Path path = null;
+ private FileStatus stat = null;
+
+ protected HdfsFileObject(AbstractFileName name, ReadOnlyHdfsFileSystem fs, FileSystem hdfs, Path p) {
+ super(name,fs);
+ this.fs = fs;
+ this.hdfs = hdfs;
+ this.path = p;
+ }
+
+ @Override
+ protected void doAttach() throws Exception {
+ this.stat = this.hdfs.getFileStatus(this.path);
+ }
+
+ @Override
+ protected FileType doGetType() throws Exception {
+ doAttach();
+ if (stat.isDir())
+ return FileType.FOLDER;
+ else
+ return FileType.FILE;
+ }
+
+ @Override
+ public boolean exists() throws FileSystemException {
+ try {
+ doAttach();
+ return this.stat != null;
+ } catch (FileNotFoundException fne) {
+ return false;
+ } catch (Exception e) {
+ throw new FileSystemException("Unable to check existance ", e);
+ }
+ }
+
+ @Override
+ public boolean canRenameTo(FileObject newfile) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected String[] doListChildren() throws Exception {
+ FileStatus[] files = this.hdfs.listStatus(this.path);
+ String[] children = new String[files.length];
+ int i = 0;
+ for (FileStatus status : files) {
+ children[i++] = status.getPath().getName();
+ }
+ return children;
+ }
+
+ @Override
+ protected long doGetContentSize() throws Exception {
+ return stat.getLen();
+ }
+
+ @Override
+ protected InputStream doGetInputStream() throws Exception {
+ return this.hdfs.open(this.path);
+ }
+
+ @Override
+ protected boolean doIsHidden() throws Exception {
+ return false;
+ }
+
+ @Override
+ protected boolean doIsReadable() throws Exception {
+ return true;
+ }
+
+ @Override
+ protected boolean doIsWriteable() throws Exception {
+ return false;
+ }
+
+ @Override
+ protected FileObject[] doListChildrenResolved() throws Exception {
+ String[] children = doListChildren();
+ FileObject[] fo = new FileObject[children.length];
+ for (int i = 0; i < children.length; i++) {
+ Path p = new Path(this.path, children[i]);
+ fo[i] = this.fs.resolveFile(p.toUri().toString());
+ }
+ return fo;
+ }
+
+ @Override
+ protected long doGetLastModifiedTime() throws Exception {
+ if (null != this.stat)
+ return this.stat.getModificationTime();
+ else
+ return -1;
+ }
+
+ @Override
+ protected boolean doSetLastModifiedTime(long modtime) throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected Map<String,Object> doGetAttributes() throws Exception {
+ if (null == this.stat)
+ return super.doGetAttributes();
+ else {
+ Map<String,Object> attrs = new HashMap<String,Object>();
+ attrs.put(LAST_ACCESS_TIME, this.stat.getAccessTime());
+ attrs.put(BLOCK_SIZE, this.stat.getBlockSize());
+ attrs.put(GROUP, this.stat.getGroup());
+ attrs.put(OWNER, this.stat.getOwner());
+ attrs.put(PERMISSIONS, this.stat.getPermission().toString());
+ attrs.put(LENGTH, this.stat.getLen());
+ attrs.put(MODIFICATION_TIME, this.stat.getModificationTime());
+ return attrs;
+ }
+ }
+
+ @Override
+ protected void doSetAttribute(String attrName, Object value) throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void doRemoveAttribute(String attrName) throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected RandomAccessContent doGetRandomAccessContent(RandomAccessMode mode) throws Exception {
+ if (mode.equals(RandomAccessMode.READWRITE))
+ throw new UnsupportedOperationException();
+ return new HdfsReadOnlyRandomAccessContent(this.path, this.hdfs);
+ }
+
+ @Override
+ protected boolean doIsSameFile(FileObject destFile) throws FileSystemException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (null == o) return false;
+ if (o == this) return true;
+ if (o instanceof HdfsFileObject) {
+ HdfsFileObject other = (HdfsFileObject) o;
+ try {
+ if (other.getURL().equals(this.getURL()))
+ return true;
+ } catch (FileSystemException e) {
+ //TODO: log error
+ return false;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return this.path.getName().toString().hashCode();
+ }
+
+}