You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2013/04/24 17:11:46 UTC
svn commit: r1471466 - in /hbase/branches/0.95:
hbase-client/src/test/java/org/apache/hadoop/hbase/client/
hbase-common/src/main/java/org/apache/hadoop/hbase/util/
hbase-common/src/test/java/org/apache/hadoop/hbase/util/
hbase-server/src/main/java/org/...
Author: jxiang
Date: Wed Apr 24 15:11:46 2013
New Revision: 1471466
URL: http://svn.apache.org/r1471466
Log:
HBASE-8327 Consolidate class loaders
Added:
hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java (with props)
hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java (with props)
hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java (with props)
Removed:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
Modified:
hbase/branches/0.95/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
Modified: hbase/branches/0.95/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java?rev=1471466&r1=1471465&r2=1471466&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java (original)
+++ hbase/branches/0.95/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java Wed Apr 24 15:11:46 2013
@@ -147,8 +147,8 @@ public class TestGet {
}
Configuration conf = HBaseConfiguration.create();
- String localPath = conf.get("hbase.local.dir") + File.separator
- + "dynamic" + File.separator + "jars" + File.separator;
+ String localPath = conf.get("hbase.local.dir")
+ + File.separator + "jars" + File.separator;
File jarFile = new File(localPath, "MockFilter.jar");
jarFile.deleteOnExit();
Added: hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java?rev=1471466&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java (added)
+++ hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java Wed Apr 24 15:11:46 2013
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Base class loader that defines couple shared constants used
+ * by sub-classes. It also defined method getClassLoadingLock for parallel
+ * class loading and JDK 1.6 support. This method (getClassLoadingLock)
+ * is similar to the same method in the base class Java ClassLoader
+ * introduced in JDK 1.7, but not in JDK 1.6.
+ */
+@InterfaceAudience.Private
+public class ClassLoaderBase extends URLClassLoader {
+
+ // Maps class name to the corresponding lock object
+ private final ConcurrentHashMap<String, Object> parallelLockMap
+ = new ConcurrentHashMap<String, Object>();
+
+ protected static final String DEFAULT_LOCAL_DIR = "/tmp/hbase-local-dir";
+ protected static final String LOCAL_DIR_KEY = "hbase.local.dir";
+
+ /**
+ * Parent class loader.
+ */
+ protected final ClassLoader parent;
+
+ /**
+ * Creates a DynamicClassLoader that can load classes dynamically
+ * from jar files under a specific folder.
+ *
+ * @param conf the configuration for the cluster.
+ * @param parent the parent ClassLoader to set.
+ */
+ public ClassLoaderBase(final ClassLoader parent) {
+ super(new URL[]{}, parent);
+ Preconditions.checkNotNull(parent, "No parent classloader!");
+ this.parent = parent;
+ }
+
+ /**
+ * Returns the lock object for class loading operations.
+ */
+ protected Object getClassLoadingLock(String className) {
+ Object lock = parallelLockMap.get(className);
+ if (lock != null) {
+ return lock;
+ }
+
+ Object newLock = new Object();
+ lock = parallelLockMap.putIfAbsent(className, newLock);
+ if (lock == null) {
+ lock = newLock;
+ }
+ return lock;
+ }
+}
Propchange: hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassLoaderBase.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java?rev=1471466&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java (added)
+++ hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java Wed Apr 24 15:11:46 2013
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.concurrent.ConcurrentMap;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+import com.google.common.collect.MapMaker;
+
+/**
+ * ClassLoader used to load classes for Coprocessor instances.
+ * <p>
+ * This ClassLoader always tries to load classes from the specified coprocessor
+ * jar first actually using URLClassLoader logic before delegating to the parent
+ * ClassLoader, thus avoiding dependency conflicts between HBase's classpath and
+ * classes in the coprocessor jar.
+ * <p>
+ * Certain classes are exempt from being loaded by this ClassLoader because it
+ * would prevent them from being cast to the equivalent classes in the region
+ * server. For example, the Coprocessor interface needs to be loaded by the
+ * region server's ClassLoader to prevent a ClassCastException when casting the
+ * coprocessor implementation.
+ * <p>
+ * A HDFS path can be used to specify the coprocessor jar. In this case, the jar
+ * will be copied to local at first under some folder under ${hbase.local.dir}/jars/tmp/.
+ * The local copy will be removed automatically when the HBase server instance is
+ * stopped.
+ * <p>
+ * This ClassLoader also handles resource loading. In most cases this
+ * ClassLoader will attempt to load resources from the coprocessor jar first
+ * before delegating to the parent. However, like in class loading,
+ * some resources need to be handled differently. For all of the Hadoop
+ * default configurations (e.g. hbase-default.xml) we will check the parent
+ * ClassLoader first to prevent issues such as failing the HBase default
+ * configuration version check.
+ */
+@InterfaceAudience.Private
+public class CoprocessorClassLoader extends ClassLoaderBase {
+ private static final Log LOG = LogFactory.getLog(CoprocessorClassLoader.class);
+
+ // A temporary place ${hbase.local.dir}/jars/tmp/ to store the local
+ // copy of the jar file and the libraries contained in the jar.
+ private static final String TMP_JARS_DIR = File.separator
+ + "jars" + File.separator + "tmp" + File.separator;
+
+ /**
+ * External class loaders cache keyed by external jar path.
+ * ClassLoader instance is stored as a weak-reference
+ * to allow GC'ing when it is not used
+ * (@see HBASE-7205)
+ */
+ private static final ConcurrentMap<Path, CoprocessorClassLoader> classLoadersCache =
+ new MapMaker().concurrencyLevel(3).weakValues().makeMap();
+
+ /**
+ * If the class being loaded starts with any of these strings, we will skip
+ * trying to load it from the coprocessor jar and instead delegate
+ * directly to the parent ClassLoader.
+ */
+ private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
+ // Java standard library:
+ "com.sun.",
+ "launcher.",
+ "java.",
+ "javax.",
+ "org.ietf",
+ "org.omg",
+ "org.w3c",
+ "org.xml",
+ "sunw.",
+ // logging
+ "org.apache.commons.logging",
+ "org.apache.log4j",
+ "com.hadoop",
+ // Hadoop/HBase/ZK:
+ "org.apache.hadoop",
+ "org.apache.zookeeper",
+ };
+
+ /**
+ * If the resource being loaded matches any of these patterns, we will first
+ * attempt to load the resource with the parent ClassLoader. Only if the
+ * resource is not found by the parent do we attempt to load it from the coprocessor jar.
+ */
+ private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS =
+ new Pattern[] {
+ Pattern.compile("^[^-]+-default\\.xml$")
+ };
+
+ /**
+ * Creates a JarClassLoader that loads classes from the given paths.
+ */
+ private CoprocessorClassLoader(ClassLoader parent) {
+ super(parent);
+ }
+
+ private void init(Path path, String pathPrefix,
+ Configuration conf) throws IOException {
+ if (path == null) {
+ throw new IOException("The jar path is null");
+ }
+ if (!path.toString().endsWith(".jar")) {
+ throw new IOException(path.toString() + ": not a jar file?");
+ }
+
+ // Copy the jar to the local filesystem
+ String parentDirPath =
+ conf.get(LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + TMP_JARS_DIR;
+ File parentDir = new File(parentDirPath);
+ if (!parentDir.mkdirs() && !parentDir.isDirectory()) {
+ throw new RuntimeException("Failed to create local dir " + parentDir.getPath()
+ + ", CoprocessorClassLoader failed to init");
+ }
+
+ FileSystem fs = path.getFileSystem(conf);
+ File dst = new File(parentDir, "." + pathPrefix + "."
+ + path.getName() + "." + System.currentTimeMillis() + ".jar");
+ fs.copyToLocalFile(path, new Path(dst.toString()));
+ dst.deleteOnExit();
+
+ addURL(dst.getCanonicalFile().toURI().toURL());
+
+ JarFile jarFile = new JarFile(dst.toString());
+ try {
+ Enumeration<JarEntry> entries = jarFile.entries();
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ if (entry.getName().matches("/lib/[^/]+\\.jar")) {
+ File file = new File(parentDir, "." + pathPrefix + "." + path.getName()
+ + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
+ IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
+ file.deleteOnExit();
+ addURL(file.toURI().toURL());
+ }
+ }
+ } finally {
+ jarFile.close();
+ }
+ }
+
+ // This method is used in unit test
+ public static CoprocessorClassLoader getIfCached(final Path path) {
+ if (path == null) return null; // No class loader for null path
+ return classLoadersCache.get(path);
+ }
+
+ // This method is used in unit test
+ public static Collection<? extends ClassLoader> getAllCached() {
+ return classLoadersCache.values();
+ }
+
+ // This method is used in unit test
+ public static void clearCache() {
+ classLoadersCache.clear();
+ }
+
+ /**
+ * Get a CoprocessorClassLoader for a coprocessor jar path from cache.
+ * If not in cache, create one.
+ *
+ * @param path the path to the coprocessor jar file to load classes from
+ * @param parent the parent class loader for exempted classes
+ * @param pathPrefix a prefix used in temp path name to store the jar file locally
+ * @param conf the configuration used to create the class loader, if needed
+ * @return a CoprocessorClassLoader for the coprocessor jar path
+ * @throws IOException
+ */
+ public static CoprocessorClassLoader getClassLoader(final Path path,
+ final ClassLoader parent, final String pathPrefix,
+ final Configuration conf) throws IOException {
+ CoprocessorClassLoader cl = getIfCached(path);
+ if (cl != null) {
+ LOG.debug("Found classloader "+ cl + "for "+ path.toString());
+ return cl;
+ }
+
+ cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
+ @Override
+ public CoprocessorClassLoader run() {
+ return new CoprocessorClassLoader(parent);
+ }
+ });
+
+ cl.init(path, pathPrefix, conf);
+
+ // Cache class loader as a weak value, will be GC'ed when no reference left
+ CoprocessorClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
+ if (prev != null) {
+ // Lost update race, use already added class loader
+ cl = prev;
+ }
+ return cl;
+ }
+
+ @Override
+ public Class<?> loadClass(String name)
+ throws ClassNotFoundException {
+ // Delegate to the parent immediately if this class is exempt
+ if (isClassExempt(name)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping exempt class " + name +
+ " - delegating directly to parent");
+ }
+ return parent.loadClass(name);
+ }
+
+ synchronized (getClassLoadingLock(name)) {
+ // Check whether the class has already been loaded:
+ Class<?> clasz = findLoadedClass(name);
+ if (clasz != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Class " + name + " already loaded");
+ }
+ }
+ else {
+ try {
+ // Try to find this class using the URLs passed to this ClassLoader
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Finding class: " + name);
+ }
+ clasz = findClass(name);
+ } catch (ClassNotFoundException e) {
+ // Class not found using this ClassLoader, so delegate to parent
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Class " + name + " not found - delegating to parent");
+ }
+ try {
+ clasz = parent.loadClass(name);
+ } catch (ClassNotFoundException e2) {
+ // Class not found in this ClassLoader or in the parent ClassLoader
+ // Log some debug output before re-throwing ClassNotFoundException
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Class " + name + " not found in parent loader");
+ }
+ throw e2;
+ }
+ }
+ }
+ return clasz;
+ }
+ }
+
+ @Override
+ public URL getResource(String name) {
+ URL resource = null;
+ boolean parentLoaded = false;
+
+ // Delegate to the parent first if necessary
+ if (loadResourceUsingParentFirst(name)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking parent first for resource " + name);
+ }
+ resource = super.getResource(name);
+ parentLoaded = true;
+ }
+
+ if (resource == null) {
+ synchronized (getClassLoadingLock(name)) {
+ // Try to find the resource in this jar
+ resource = findResource(name);
+ if ((resource == null) && !parentLoaded) {
+ // Not found in this jar and we haven't attempted to load
+ // the resource in the parent yet; fall back to the parent
+ resource = super.getResource(name);
+ }
+ }
+ }
+ return resource;
+ }
+
+ /**
+ * Determines whether the given class should be exempt from being loaded
+ * by this ClassLoader.
+ * @param name the name of the class to test.
+ * @return true if the class should *not* be loaded by this ClassLoader;
+ * false otherwise.
+ */
+ protected boolean isClassExempt(String name) {
+ for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
+ if (name.startsWith(exemptPrefix)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Determines whether we should attempt to load the given resource using the
+ * parent first before attempting to load the resource using this ClassLoader.
+ * @param name the name of the resource to test.
+ * @return true if we should attempt to load the resource using the parent
+ * first; false if we should attempt to load the resource using this
+ * ClassLoader first.
+ */
+ protected boolean loadResourceUsingParentFirst(String name) {
+ for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
+ if (resourcePattern.matcher(name).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
Propchange: hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java?rev=1471466&r1=1471465&r2=1471466&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java (original)
+++ hbase/branches/0.95/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java Wed Apr 24 15:11:46 2013
@@ -21,7 +21,6 @@ import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
-import java.net.URLClassLoader;
import java.util.HashMap;
import org.apache.commons.logging.Log;
@@ -34,10 +33,17 @@ import org.apache.hadoop.fs.Path;
/**
* This is a class loader that can load classes dynamically from new
- * jar files under a configured folder. It always uses its parent class
- * loader to load a class at first. Only if its parent class loader
+ * jar files under a configured folder. The paths to the jar files are
+ * converted to URLs, and URLClassLoader logic is actually used to load
+ * classes. This class loader always uses its parent class loader
+ * to load a class at first. Only if its parent class loader
* can not load a class, we will try to load it using the logic here.
* <p>
+ * The configured folder can be a HDFS path. In this case, the jar files
+ * under that folder will be copied to local at first under ${hbase.local.dir}/jars/.
+ * The local copy will be updated if the remote copy is updated, according to its
+ * last modified timestamp.
+ * <p>
* We can't unload a class already loaded. So we will use the existing
* jar files we already know to load any class which can't be loaded
* using the parent class loader. If we still can't load the class from
@@ -50,18 +56,15 @@ import org.apache.hadoop.fs.Path;
* classes properly.
*/
@InterfaceAudience.Private
-public class DynamicClassLoader extends URLClassLoader {
+public class DynamicClassLoader extends ClassLoaderBase {
private static final Log LOG =
LogFactory.getLog(DynamicClassLoader.class);
- // Dynamic jars are put under ${hbase.local.dir}/dynamic/jars/
+ // Dynamic jars are put under ${hbase.local.dir}/jars/
private static final String DYNAMIC_JARS_DIR = File.separator
- + "dynamic" + File.separator + "jars" + File.separator;
+ + "jars" + File.separator;
- /**
- * Parent class loader used to load any class at first.
- */
- private final ClassLoader parent;
+ private static final String DYNAMIC_JARS_DIR_KEY = "hbase.dynamic.jars.dir";
private File localDir;
@@ -81,18 +84,18 @@ public class DynamicClassLoader extends
*/
public DynamicClassLoader(
final Configuration conf, final ClassLoader parent) {
- super(new URL[]{}, parent);
- this.parent = parent;
+ super(parent);
jarModifiedTime = new HashMap<String, Long>();
- String localDirPath = conf.get("hbase.local.dir") + DYNAMIC_JARS_DIR;
+ String localDirPath = conf.get(
+ LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + DYNAMIC_JARS_DIR;
localDir = new File(localDirPath);
if (!localDir.mkdirs() && !localDir.isDirectory()) {
throw new RuntimeException("Failed to create local dir " + localDir.getPath()
+ ", DynamicClassLoader failed to init");
}
- String remotePath = conf.get("hbase.dynamic.jars.dir");
+ String remotePath = conf.get(DYNAMIC_JARS_DIR_KEY);
if (remotePath == null || remotePath.equals(localDirPath)) {
remoteDir = null; // ignore if it is the same as the local path
} else {
@@ -117,33 +120,35 @@ public class DynamicClassLoader extends
LOG.debug("Class " + name + " not found - using dynamical class loader");
}
- // Check whether the class has already been loaded:
- Class<?> clasz = findLoadedClass(name);
- if (clasz != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Class " + name + " already loaded");
- }
- }
- else {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Finding class: " + name);
- }
- clasz = findClass(name);
- } catch (ClassNotFoundException cnfe) {
- // Load new jar files if any
+ synchronized (getClassLoadingLock(name)) {
+ // Check whether the class has already been loaded:
+ Class<?> clasz = findLoadedClass(name);
+ if (clasz != null) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Loading new jar files, if any");
+ LOG.debug("Class " + name + " already loaded");
}
- loadNewJars();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Finding class again: " + name);
+ }
+ else {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Finding class: " + name);
+ }
+ clasz = findClass(name);
+ } catch (ClassNotFoundException cnfe) {
+ // Load new jar files if any
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading new jar files, if any");
+ }
+ loadNewJars();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Finding class again: " + name);
+ }
+ clasz = findClass(name);
}
- clasz = findClass(name);
}
+ return clasz;
}
- return clasz;
}
}
Added: hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java?rev=1471466&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java (added)
+++ hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java Wed Apr 24 15:11:46 2013
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Some utilities to help class loader testing
+ */
+public class ClassLoaderTestHelper {
+ private static final Log LOG = LogFactory.getLog(ClassLoaderTestHelper.class);
+
+ /**
+ * Jar a list of files into a jar archive.
+ *
+ * @param archiveFile the target jar archive
+ * @param tobejared a list of files to be jared
+ */
+ private static boolean createJarArchive(File archiveFile, File[] tobeJared) {
+ try {
+ byte buffer[] = new byte[4096];
+ // Open archive file
+ FileOutputStream stream = new FileOutputStream(archiveFile);
+ JarOutputStream out = new JarOutputStream(stream, new Manifest());
+
+ for (int i = 0; i < tobeJared.length; i++) {
+ if (tobeJared[i] == null || !tobeJared[i].exists()
+ || tobeJared[i].isDirectory()) {
+ continue;
+ }
+
+ // Add archive entry
+ JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
+ jarAdd.setTime(tobeJared[i].lastModified());
+ out.putNextEntry(jarAdd);
+
+ // Write file to archive
+ FileInputStream in = new FileInputStream(tobeJared[i]);
+ while (true) {
+ int nRead = in.read(buffer, 0, buffer.length);
+ if (nRead <= 0)
+ break;
+ out.write(buffer, 0, nRead);
+ }
+ in.close();
+ }
+ out.close();
+ stream.close();
+ LOG.info("Adding classes to jar file completed");
+ return true;
+ } catch (Exception ex) {
+ LOG.error("Error: " + ex.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Create a test jar for testing purpose for a given class
+ * name with specified code string: save the class to a file,
+ * compile it, and jar it up. If the code string passed in is
+ * null, a bare empty class will be created and used.
+ *
+ * @param testDir the folder under which to store the test class and jar
+ * @param className the test class name
+ * @param code the optional test class code, which can be null.
+ * If null, a bare empty class will be used
+ * @return the test jar file generated
+ */
+ public static File buildJar(String testDir,
+ String className, String code) throws Exception {
+ return buildJar(testDir, className, code, testDir);
+ }
+
+ /**
+ * Create a test jar for testing purpose for a given class
+ * name with specified code string.
+ *
+ * @param testDir the folder under which to store the test class
+ * @param className the test class name
+ * @param code the optional test class code, which can be null.
+ * If null, an empty class will be used
+ * @param folder the folder under which to store the generated jar
+ * @return the test jar file generated
+ */
+ public static File buildJar(String testDir,
+ String className, String code, String folder) throws Exception {
+ String javaCode = code != null ? code : "public class " + className + " {}";
+ Path srcDir = new Path(testDir, "src");
+ File srcDirPath = new File(srcDir.toString());
+ srcDirPath.mkdirs();
+ File sourceCodeFile = new File(srcDir.toString(), className + ".java");
+ BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
+ bw.write(javaCode);
+ bw.close();
+
+ // compile it by JavaCompiler
+ JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+ ArrayList<String> srcFileNames = new ArrayList<String>();
+ srcFileNames.add(sourceCodeFile.toString());
+ StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
+ null);
+ Iterable<? extends JavaFileObject> cu =
+ fm.getJavaFileObjects(sourceCodeFile);
+ List<String> options = new ArrayList<String>();
+ options.add("-classpath");
+ // only add hbase classes to classpath. This is a little bit tricky: assume
+ // the classpath is {hbaseSrc}/target/classes.
+ String currentDir = new File(".").getAbsolutePath();
+ String classpath = currentDir + File.separator + "target"+ File.separator
+ + "classes" + System.getProperty("path.separator")
+ + System.getProperty("java.class.path") + System.getProperty("path.separator")
+ + System.getProperty("surefire.test.class.path");
+
+ options.add(classpath);
+ LOG.debug("Setting classpath to: " + classpath);
+
+ JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
+ options, null, cu);
+ assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
+
+ // build a jar file by the classes files
+ String jarFileName = className + ".jar";
+ File jarFile = new File(folder, jarFileName);
+ if (!createJarArchive(jarFile,
+ new File[]{new File(srcDir.toString(), className + ".class")})){
+ assertTrue("Build jar file failed.", false);
+ }
+ return jarFile;
+ }
+}
Propchange: hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/ClassLoaderTestHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java?rev=1471466&r1=1471465&r2=1471466&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java (original)
+++ hbase/branches/0.95/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDynamicClassLoader.java Wed Apr 24 15:11:46 2013
@@ -18,29 +18,14 @@
*/
package org.apache.hadoop.hbase.util;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-
-import javax.tools.JavaCompiler;
-import javax.tools.JavaFileObject;
-import javax.tools.StandardJavaFileManager;
-import javax.tools.ToolProvider;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.SmallTests;
@@ -62,95 +47,13 @@ public class TestDynamicClassLoader {
conf.set("hbase.dynamic.jars.dir", TEST_UTIL.getDataTestDir().toString());
}
- // generate jar file
- private boolean createJarArchive(File archiveFile, File[] tobeJared) {
- try {
- byte buffer[] = new byte[4096];
- // Open archive file
- FileOutputStream stream = new FileOutputStream(archiveFile);
- JarOutputStream out = new JarOutputStream(stream, new Manifest());
-
- for (int i = 0; i < tobeJared.length; i++) {
- if (tobeJared[i] == null || !tobeJared[i].exists()
- || tobeJared[i].isDirectory()) {
- continue;
- }
-
- // Add archive entry
- JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
- jarAdd.setTime(tobeJared[i].lastModified());
- out.putNextEntry(jarAdd);
-
- // Write file to archive
- FileInputStream in = new FileInputStream(tobeJared[i]);
- while (true) {
- int nRead = in.read(buffer, 0, buffer.length);
- if (nRead <= 0)
- break;
- out.write(buffer, 0, nRead);
- }
- in.close();
- }
- out.close();
- stream.close();
- LOG.info("Adding classes to jar file completed");
- return true;
- } catch (Exception ex) {
- LOG.error("Error: " + ex.getMessage());
- return false;
- }
- }
-
- private File buildJar(
- String className, String folder) throws Exception {
- String javaCode = "public class " + className + " {}";
- Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
- File srcDirPath = new File(srcDir.toString());
- srcDirPath.mkdirs();
- File sourceCodeFile = new File(srcDir.toString(), className + ".java");
- BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
- bw.write(javaCode);
- bw.close();
-
- // compile it by JavaCompiler
- JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
- ArrayList<String> srcFileNames = new ArrayList<String>();
- srcFileNames.add(sourceCodeFile.toString());
- StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
- null);
- Iterable<? extends JavaFileObject> cu =
- fm.getJavaFileObjects(sourceCodeFile);
- List<String> options = new ArrayList<String>();
- options.add("-classpath");
- // only add hbase classes to classpath. This is a little bit tricky: assume
- // the classpath is {hbaseSrc}/target/classes.
- String currentDir = new File(".").getAbsolutePath();
- String classpath =
- currentDir + File.separator + "target"+ File.separator + "classes" +
- System.getProperty("path.separator") + System.getProperty("java.class.path");
- options.add(classpath);
- LOG.debug("Setting classpath to: "+classpath);
-
- JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
- options, null, cu);
- assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
-
- // build a jar file by the classes files
- String jarFileName = className + ".jar";
- File jarFile = new File(folder, jarFileName);
- if (!createJarArchive(jarFile,
- new File[]{new File(srcDir.toString(), className + ".class")})){
- assertTrue("Build jar file failed.", false);
- }
- return jarFile;
- }
-
@Test
public void testLoadClassFromLocalPath() throws Exception {
ClassLoader parent = TestDynamicClassLoader.class.getClassLoader();
DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
String className = "TestLoadClassFromLocalPath";
+ deleteClass(className);
try {
classLoader.loadClass(className);
fail("Should not be able to load class " + className);
@@ -159,13 +62,12 @@ public class TestDynamicClassLoader {
}
try {
- buildJar(className, localDirPath());
+ String folder = TEST_UTIL.getDataTestDir().toString();
+ ClassLoaderTestHelper.buildJar(folder, className, null, localDirPath());
classLoader.loadClass(className);
} catch (ClassNotFoundException cnfe) {
LOG.error("Should be able to load class " + className, cnfe);
fail(cnfe.getMessage());
- } finally {
- deleteClass(className);
}
}
@@ -175,6 +77,7 @@ public class TestDynamicClassLoader {
DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
String className = "TestLoadClassFromAnotherPath";
+ deleteClass(className);
try {
classLoader.loadClass(className);
fail("Should not be able to load class " + className);
@@ -183,30 +86,32 @@ public class TestDynamicClassLoader {
}
try {
- buildJar(className, TEST_UTIL.getDataTestDir().toString());
+ String folder = TEST_UTIL.getDataTestDir().toString();
+ ClassLoaderTestHelper.buildJar(folder, className, null);
classLoader.loadClass(className);
} catch (ClassNotFoundException cnfe) {
LOG.error("Should be able to load class " + className, cnfe);
fail(cnfe.getMessage());
- } finally {
- deleteClass(className);
}
}
private String localDirPath() {
- return conf.get("hbase.local.dir") + File.separator
- + "dynamic" + File.separator + "jars" + File.separator;
+ return conf.get("hbase.local.dir")
+ + File.separator + "jars" + File.separator;
}
private void deleteClass(String className) throws Exception {
String jarFileName = className + ".jar";
File file = new File(TEST_UTIL.getDataTestDir().toString(), jarFileName);
- file.deleteOnExit();
+ file.delete();
+ assertFalse("Should be deleted: " + file.getPath(), file.exists());
file = new File(conf.get("hbase.dynamic.jars.dir"), jarFileName);
- file.deleteOnExit();
+ file.delete();
+ assertFalse("Should be deleted: " + file.getPath(), file.exists());
file = new File(localDirPath(), jarFileName);
- file.deleteOnExit();
+ file.delete();
+ assertFalse("Should be deleted: " + file.getPath(), file.exists());
}
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java?rev=1471466&r1=1471465&r2=1471466&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java Wed Apr 24 15:11:46 2013
@@ -19,39 +19,51 @@
package org.apache.hadoop.hbase.coprocessor;
-import com.google.common.collect.MapMaker;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.io.IOUtils;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.*;
-import java.util.concurrent.ConcurrentMap;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
/**
* Provides the common setup framework and runtime services for coprocessor
@@ -73,10 +85,6 @@ public abstract class CoprocessorHost<E
public static final String WAL_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.wal.classes";
- //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/
- private static final String COPROCESSOR_JARS_DIR = File.separator
- + "coprocessor" + File.separator + "jars" + File.separator;
-
private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
/** Ordered set of loaded coprocessors with lock */
protected SortedSet<E> coprocessors =
@@ -86,15 +94,6 @@ public abstract class CoprocessorHost<E
protected String pathPrefix;
protected volatile int loadSequence;
- /*
- * External classloaders cache keyed by external jar path.
- * ClassLoader instance is stored as a weak-reference
- * to allow GC'ing when no CoprocessorHost is using it
- * (@see HBASE-7205)
- */
- static ConcurrentMap<Path, ClassLoader> classLoadersCache =
- new MapMaker().concurrencyLevel(3).weakValues().makeMap();
-
public CoprocessorHost() {
pathPrefix = UUID.randomUUID().toString();
}
@@ -175,7 +174,6 @@ public abstract class CoprocessorHost<E
* @param conf configuration for coprocessor
* @throws java.io.IOException Exception
*/
- @SuppressWarnings("deprecation")
public E load(Path path, String className, int priority,
Configuration conf) throws IOException {
Class<?> implClass = null;
@@ -190,81 +188,8 @@ public abstract class CoprocessorHost<E
throw new IOException("No jar path specified for " + className);
}
} else {
- // Have we already loaded the class, perhaps from an earlier region open
- // for the same table?
- cl = classLoadersCache.get(path);
- if (cl != null){
- LOG.debug("Found classloader "+ cl + "for "+path.toString());
- try {
- implClass = cl.loadClass(className);
- } catch (ClassNotFoundException e) {
- LOG.info("Class " + className + " needs to be loaded from a file - " +
- path + ".");
- // go ahead to load from file system.
- }
- }
- }
-
- // If not, load
- if (implClass == null) {
- if (path == null) {
- throw new IOException("No jar path specified for " + className);
- }
- // copy the jar to the local filesystem
- if (!path.toString().endsWith(".jar")) {
- throw new IOException(path.toString() + ": not a jar file?");
- }
- FileSystem fs = path.getFileSystem(this.conf);
- File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
- parentDir.mkdirs();
- File dst = new File(parentDir, "." + pathPrefix +
- "." + className + "." + System.currentTimeMillis() + ".jar");
- fs.copyToLocalFile(path, new Path(dst.toString()));
- dst.deleteOnExit();
-
- // TODO: code weaving goes here
-
- // TODO: wrap heap allocations and enforce maximum usage limits
-
- /* TODO: inject code into loop headers that monitors CPU use and
- aborts runaway user code */
-
- // load the jar and get the implementation main class
- // NOTE: Path.toURL is deprecated (toURI instead) but the URLClassLoader
- // unsurprisingly wants URLs, not URIs; so we will use the deprecated
- // method which returns URLs for as long as it is available
- final List<URL> paths = new ArrayList<URL>();
- URL url = dst.getCanonicalFile().toURL();
- paths.add(url);
-
- JarFile jarFile = new JarFile(dst.toString());
- Enumeration<JarEntry> entries = jarFile.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = entries.nextElement();
- if (entry.getName().matches("/lib/[^/]+\\.jar")) {
- File file = new File(parentDir, "." + pathPrefix +
- "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
- IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
- file.deleteOnExit();
- paths.add(file.toURL());
- }
- }
- jarFile.close();
-
- cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
- @Override
- public CoprocessorClassLoader run() {
- return new CoprocessorClassLoader(paths, this.getClass().getClassLoader());
- }
- });
-
- // cache cp classloader as a weak value, will be GC'ed when no reference left
- ClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
- if (prev != null) {
- //lost update race, use already added classloader
- cl = prev;
- }
-
+ cl = CoprocessorClassLoader.getClassLoader(
+ path, getClass().getClassLoader(), pathPrefix, conf);
try {
implClass = cl.loadClass(className);
} catch (ClassNotFoundException e) {
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java?rev=1471466&r1=1471465&r2=1471466&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java Wed Apr 24 15:11:46 2013
@@ -26,13 +26,14 @@ import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
+import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
+import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.RegionLoad;
-import javax.tools.*;
import java.io.*;
import java.util.*;
import java.util.jar.*;
@@ -41,6 +42,7 @@ import org.junit.*;
import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
@@ -103,90 +105,11 @@ public class TestClassLoading {
TEST_UTIL.shutdownMiniCluster();
}
- // generate jar file
- private boolean createJarArchive(File archiveFile, File[] tobeJared) {
- try {
- byte buffer[] = new byte[BUFFER_SIZE];
- // Open archive file
- FileOutputStream stream = new FileOutputStream(archiveFile);
- JarOutputStream out = new JarOutputStream(stream, new Manifest());
-
- for (int i = 0; i < tobeJared.length; i++) {
- if (tobeJared[i] == null || !tobeJared[i].exists()
- || tobeJared[i].isDirectory()) {
- continue;
- }
-
- // Add archive entry
- JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
- jarAdd.setTime(tobeJared[i].lastModified());
- out.putNextEntry(jarAdd);
-
- // Write file to archive
- FileInputStream in = new FileInputStream(tobeJared[i]);
- while (true) {
- int nRead = in.read(buffer, 0, buffer.length);
- if (nRead <= 0)
- break;
- out.write(buffer, 0, nRead);
- }
- in.close();
- }
- out.close();
- stream.close();
- LOG.info("Adding classes to jar file completed");
- return true;
- } catch (Exception ex) {
- LOG.error("Error: " + ex.getMessage());
- return false;
- }
- }
-
- private File buildCoprocessorJar(String className) throws Exception {
- // compose a java source file.
- String javaCode = "import org.apache.hadoop.hbase.coprocessor.*;" +
+ static File buildCoprocessorJar(String className) throws Exception {
+ String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
"public class " + className + " extends BaseRegionObserver {}";
- Path baseDir = TEST_UTIL.getDataTestDir();
- Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
- File srcDirPath = new File(srcDir.toString());
- srcDirPath.mkdirs();
- File sourceCodeFile = new File(srcDir.toString(), className + ".java");
- BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
- bw.write(javaCode);
- bw.close();
-
- // compile it by JavaCompiler
- JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
- ArrayList<String> srcFileNames = new ArrayList<String>();
- srcFileNames.add(sourceCodeFile.toString());
- StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
- null);
- Iterable<? extends JavaFileObject> cu =
- fm.getJavaFileObjects(sourceCodeFile);
- List<String> options = new ArrayList<String>();
- options.add("-classpath");
- // only add hbase classes to classpath. This is a little bit tricky: assume
- // the classpath is {hbaseSrc}/target/classes.
- String currentDir = new File(".").getAbsolutePath();
- String classpath =
- currentDir + File.separator + "target"+ File.separator + "classes" +
- System.getProperty("path.separator") + System.getProperty("java.class.path");
- options.add(classpath);
- LOG.debug("Setting classpath to: "+classpath);
-
- JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
- options, null, cu);
- assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
-
- // build a jar file by the classes files
- String jarFileName = className + ".jar";
- File jarFile = new File(baseDir.toString(), jarFileName);
- if (!createJarArchive(jarFile,
- new File[]{new File(srcDir.toString(), className + ".class")})){
- assertTrue("Build jar file failed.", false);
- }
-
- return jarFile;
+ return ClassLoaderTestHelper.buildJar(
+ TEST_UTIL.getDataTestDir().toString(), className, code);
}
@Test
@@ -235,7 +158,7 @@ public class TestClassLoading {
}
admin.deleteTable(tableName);
}
- CoprocessorHost.classLoadersCache.clear();
+ CoprocessorClassLoader.clearCache();
byte[] startKey = {10, 63};
byte[] endKey = {12, 43};
admin.createTable(htd, startKey, endKey, 4);
@@ -282,22 +205,22 @@ public class TestClassLoading {
assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
// check if CP classloaders are cached
- assertTrue(jarFileOnHDFS1 + " was not cached",
- CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS1));
- assertTrue(jarFileOnHDFS2 + " was not cached",
- CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS2));
+ assertNotNull(jarFileOnHDFS1 + " was not cached",
+ CoprocessorClassLoader.getIfCached(pathOnHDFS1));
+ assertNotNull(jarFileOnHDFS2 + " was not cached",
+ CoprocessorClassLoader.getIfCached(pathOnHDFS2));
//two external jar used, should be one classloader per jar
assertEquals("The number of cached classloaders should be equal to the number" +
" of external jar files",
- 2, CoprocessorHost.classLoadersCache.size());
+ 2, CoprocessorClassLoader.getAllCached().size());
//check if region active classloaders are shared across all RS regions
Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
- CoprocessorHost.classLoadersCache.values());
+ CoprocessorClassLoader.getAllCached());
for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
- + " ClassLoader Cache:" + externalClassLoaders
- + " Region ClassLoaders:" + regionCP.getValue(),
- externalClassLoaders.containsAll(regionCP.getValue()));
+ + " ClassLoader Cache:" + externalClassLoaders
+ + " Region ClassLoaders:" + regionCP.getValue(),
+ externalClassLoaders.containsAll(regionCP.getValue()));
}
}