You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by je...@apache.org on 2016/07/10 01:05:54 UTC
hbase git commit: HBASE-14548 Expand how table coprocessor jar and
dependency path can be specified (Xiang Li)
Repository: hbase
Updated Branches:
refs/heads/master 496fd9837 -> 632969787
HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/63296978
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/63296978
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/63296978
Branch: refs/heads/master
Commit: 632969787aa9e9e1d73b83449b93e21f91110b73
Parents: 496fd98
Author: Jerry He <je...@apache.org>
Authored: Sat Jul 9 18:00:41 2016 -0700
Committer: Jerry He <je...@apache.org>
Committed: Sat Jul 9 18:01:49 2016 -0700
----------------------------------------------------------------------
.../hbase/util/CoprocessorClassLoader.java | 72 +++++++++++++-------
.../hbase/util/TestCoprocessorClassLoader.java | 43 ++++++++++++
src/main/asciidoc/_chapters/cp.adoc | 9 ++-
3 files changed, 97 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index 11016c3..c3635cb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.util;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
@@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.io.IOUtils;
@@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
super(parent);
}
- private void init(Path path, String pathPrefix,
+ private void init(Path pathPattern, String pathPrefix,
Configuration conf) throws IOException {
// Copy the jar to the local filesystem
String parentDirStr =
@@ -160,33 +163,50 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
}
}
- FileSystem fs = path.getFileSystem(conf);
- File dst = new File(parentDirStr, "." + pathPrefix + "."
- + path.getName() + "." + System.currentTimeMillis() + ".jar");
- fs.copyToLocalFile(path, new Path(dst.toString()));
- dst.deleteOnExit();
-
- addURL(dst.getCanonicalFile().toURI().toURL());
-
- JarFile jarFile = new JarFile(dst.toString());
- try {
- Enumeration<JarEntry> entries = jarFile.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = entries.nextElement();
- Matcher m = libJarPattern.matcher(entry.getName());
- if (m.matches()) {
- File file = new File(parentDirStr, "." + pathPrefix + "."
- + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
- try (FileOutputStream outStream = new FileOutputStream(file)) {
- IOUtils.copyBytes(jarFile.getInputStream(entry),
- outStream, conf, true);
+ FileSystem fs = pathPattern.getFileSystem(conf);
+ Path pathPattern1 = fs.isDirectory(pathPattern) ?
+ new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified
+ FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern
+ if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches
+ throw new FileNotFoundException(pathPattern1.toString());
+ } else {
+ boolean validFileEncountered = false;
+ for (Path path : FileUtil.stat2Paths(fileStatuses)) { // for each file that match the pattern
+ if (fs.isFile(path)) { // only process files, skip for directories
+ File dst = new File(parentDirStr, "." + pathPrefix + "."
+ + path.getName() + "." + System.currentTimeMillis() + ".jar");
+ fs.copyToLocalFile(path, new Path(dst.toString()));
+ dst.deleteOnExit();
+
+ addURL(dst.getCanonicalFile().toURI().toURL());
+
+ JarFile jarFile = new JarFile(dst.toString());
+ try {
+ Enumeration<JarEntry> entries = jarFile.entries(); // get entries inside a jar file
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ Matcher m = libJarPattern.matcher(entry.getName());
+ if (m.matches()) {
+ File file = new File(parentDirStr, "." + pathPrefix + "."
+ + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
+ try (FileOutputStream outStream = new FileOutputStream(file)) {
+ IOUtils.copyBytes(jarFile.getInputStream(entry),
+ outStream, conf, true);
+ }
+ file.deleteOnExit();
+ addURL(file.toURI().toURL());
+ }
+ }
+ } finally {
+ jarFile.close();
}
- file.deleteOnExit();
- addURL(file.toURI().toURL());
+
+ validFileEncountered = true; // Set to true when encountering a file
}
}
- } finally {
- jarFile.close();
+ if (validFileEncountered == false) { // all items returned by globStatus() are directories
+ throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
+ }
}
}
@@ -227,7 +247,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
return cl;
}
- if (!pathStr.endsWith(".jar")) {
+ if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
throw new IOException(pathStr + ": not a jar file?");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
index daba459..967d272 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.File;
@@ -109,4 +110,46 @@ public class TestCoprocessorClassLoader {
}
fail("Could not find the expected lib jar file");
}
+
+ // HBASE-14548
+ @Test
+ public void testDirectoryAndWildcard() throws Exception {
+ String testClassName = "TestClass";
+ String dataTestDir = TEST_UTIL.getDataTestDir().toString();
+ System.out.println(dataTestDir);
+ String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
+ ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
+ ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
+ CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
+
+ CoprocessorClassLoader coprocessorClassLoader = null;
+ Path testPath = null;
+
+ // Directory
+ testPath = new Path(localDirContainingJar);
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+ // Wildcard - *.jar
+ testPath = new Path(localDirContainingJar, "*.jar");
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+ // Wildcard - *.j*
+ testPath = new Path(localDirContainingJar, "*.j*");
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+ }
+
+ /**
+ * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
+ * @param coprocessorClassLoader the CoprocessorClassLoader to verify
+ * @param className the expected class to be loaded by the coprocessorClassLoader
+ * @throws ClassNotFoundException
+ */
+ private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className)
+ throws ClassNotFoundException{
+ assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
+ assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/src/main/asciidoc/_chapters/cp.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/cp.adoc b/src/main/asciidoc/_chapters/cp.adoc
index 793bb6f..376899a 100644
--- a/src/main/asciidoc/_chapters/cp.adoc
+++ b/src/main/asciidoc/_chapters/cp.adoc
@@ -320,7 +320,14 @@ The value contains four pieces of information which are separated by the pipe (`
* File path: The jar file containing the Coprocessor implementation must be in a location where
all region servers can read it. +
You could copy the file onto the local disk on each region server, but it is recommended to store
-it in HDFS.
+it in HDFS. +
+https://issues.apache.org/jira/browse/HBASE-14548[HBASE-14548] allows a directory containing the jars
+or some wildcards to be specified, such as: hdfs://<namenode>:<port>/user/<hadoop-user>/ or
+hdfs://<namenode>:<port>/user/<hadoop-user>/*.jar. Please note that if a directory is specified,
+all jar files(.jar) directly in the directory are added,
+but it does not search files in the subtree rooted in the directory.
+And do not contain any wildcard if you would like to specify a directory.
+This enhancement applies to the ways of using the JAVA API as well.
* Class name: The full class name of the Coprocessor.
* Priority: An integer. The framework will determine the execution sequence of all configured
observers registered at the same hook using priorities. This field can be left blank. In that