You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/08/01 23:04:51 UTC
[1/2] hbase git commit: HBASE-14548 Expand how table coprocessor jar
and dependency path can be specified (Xiang Li)
Repository: hbase
Updated Branches:
refs/heads/0.98 27c583467 -> 6dfdc0d89
HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f750acb7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f750acb7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f750acb7
Branch: refs/heads/0.98
Commit: f750acb751c5cc3b4a3292451c0504942b62bf39
Parents: 27c5834
Author: Jerry He <je...@apache.org>
Authored: Sat Jul 9 18:18:22 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 1 15:52:44 2016 -0700
----------------------------------------------------------------------
.../hbase/util/CoprocessorClassLoader.java | 74 +++++++++++++-------
.../hbase/util/TestCoprocessorClassLoader.java | 43 ++++++++++++
2 files changed, 93 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f750acb7/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index fb2a127..9b65807 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.util;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
@@ -39,6 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.io.IOUtils;
import com.google.common.base.Preconditions;
@@ -155,7 +159,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
super(parent);
}
- private void init(Path path, String pathPrefix,
+ private void init(Path pathPattern, String pathPrefix,
Configuration conf) throws IOException {
// Copy the jar to the local filesystem
String parentDirStr =
@@ -173,31 +177,53 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
}
}
- FileSystem fs = path.getFileSystem(conf);
- File dst = new File(parentDirStr, "." + pathPrefix + "."
- + path.getName() + "." + System.currentTimeMillis() + ".jar");
- fs.copyToLocalFile(path, new Path(dst.toString()));
- dst.deleteOnExit();
-
- addURL(dst.getCanonicalFile().toURI().toURL());
+ FileSystem fs = pathPattern.getFileSystem(conf);
+ Path pathPattern1 = fs.isDirectory(pathPattern) ?
+ new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified
+ FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern
+ if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches
+ throw new FileNotFoundException(pathPattern1.toString());
+ } else {
+ boolean validFileEncountered = false;
+ for (Path path : FileUtil.stat2Paths(fileStatuses)) { // for each file that match the pattern
+ if (fs.isFile(path)) { // only process files, skip for directories
+ File dst = new File(parentDirStr, "." + pathPrefix + "."
+ + path.getName() + "." + System.currentTimeMillis() + ".jar");
+ fs.copyToLocalFile(path, new Path(dst.toString()));
+ dst.deleteOnExit();
+
+ addURL(dst.getCanonicalFile().toURI().toURL());
+
+ JarFile jarFile = new JarFile(dst.toString());
+ try {
+ Enumeration<JarEntry> entries = jarFile.entries(); // get entries inside a jar file
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ Matcher m = libJarPattern.matcher(entry.getName());
+ if (m.matches()) {
+ File file = new File(parentDirStr, "." + pathPrefix + "."
+ + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
+ FileOutputStream outStream = new FileOutputStream(file);
+ try {
+ IOUtils.copyBytes(jarFile.getInputStream(entry),
+ outStream, conf, true);
+ } finally {
+ outStream.close();
+ }
+ file.deleteOnExit();
+ addURL(file.toURI().toURL());
+ }
+ }
+ } finally {
+ jarFile.close();
+ }
- JarFile jarFile = new JarFile(dst.toString());
- try {
- Enumeration<JarEntry> entries = jarFile.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = entries.nextElement();
- Matcher m = libJarPattern.matcher(entry.getName());
- if (m.matches()) {
- File file = new File(parentDirStr, "." + pathPrefix + "."
- + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
- IOUtils.copyBytes(jarFile.getInputStream(entry),
- new FileOutputStream(file), conf, true);
- file.deleteOnExit();
- addURL(file.toURI().toURL());
+ validFileEncountered = true; // Set to true when encountering a file
}
}
- } finally {
- jarFile.close();
+ if (validFileEncountered == false) { // all items returned by globStatus() are directories
+ throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
+ }
}
}
@@ -238,7 +264,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
return cl;
}
- if (!pathStr.endsWith(".jar")) {
+ if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
throw new IOException(pathStr + ": not a jar file?");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f750acb7/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
index f4b2002..e1048da 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.File;
@@ -108,4 +109,46 @@ public class TestCoprocessorClassLoader {
}
fail("Could not find the expected lib jar file");
}
+
+ // HBASE-14548
+ @Test
+ public void testDirectoryAndWildcard() throws Exception {
+ String testClassName = "TestClass";
+ String dataTestDir = TEST_UTIL.getDataTestDir().toString();
+ System.out.println(dataTestDir);
+ String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
+ ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
+ ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
+ CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
+
+ CoprocessorClassLoader coprocessorClassLoader = null;
+ Path testPath = null;
+
+ // Directory
+ testPath = new Path(localDirContainingJar);
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+ // Wildcard - *.jar
+ testPath = new Path(localDirContainingJar, "*.jar");
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+ // Wildcard - *.j*
+ testPath = new Path(localDirContainingJar, "*.j*");
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+ }
+
+ /**
+ * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
+ * @param coprocessorClassLoader the CoprocessorClassLoader to verify
+ * @param className the expected class to be loaded by the coprocessorClassLoader
+ * @throws ClassNotFoundException
+ */
+ private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className)
+ throws ClassNotFoundException{
+ assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
+ assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
+ }
}
[2/2] hbase git commit: Revert "HBASE-16272 Overflow in ServerName's
compareTo method (Huaxiang Sun)"
Posted by ap...@apache.org.
Revert "HBASE-16272 Overflow in ServerName's compareTo method (Huaxiang Sun)"
This reverts commit 6e54ac352ae2a7aae9cc804132126aee0c44b971.
JDK/JRE 6 does not have Long#compare
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6dfdc0d8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6dfdc0d8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6dfdc0d8
Branch: refs/heads/0.98
Commit: 6dfdc0d898b92371b8694ff4b32897ade8f53bcb
Parents: f750acb
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Aug 1 16:03:51 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 1 16:03:51 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/ServerName.java | 2 +-
.../org/apache/hadoop/hbase/io/hfile/LruBlockCache.java | 7 ++++---
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 12 ++++++++----
.../java/org/apache/hadoop/hbase/util/HBaseFsck.java | 2 +-
4 files changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6dfdc0d8/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
index ad03816..ec8fcb2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java
@@ -290,7 +290,7 @@ public class ServerName implements Comparable<ServerName>, Serializable {
if (compare != 0) return compare;
compare = this.getPort() - other.getPort();
if (compare != 0) return compare;
- return Long.compare(this.getStartcode(), other.getStartcode());
+ return (int)(this.getStartcode() - other.getStartcode());
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/6dfdc0d8/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 65683b7..4afd556 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -714,7 +714,8 @@ public class LruBlockCache implements BlockCache, HeapSize {
}
public int compareTo(BlockBucket that) {
- return Long.compare(this.overflow(), that.overflow());
+ if(this.overflow() == that.overflow()) return 0;
+ return this.overflow() > that.overflow() ? 1 : -1;
}
@Override
@@ -949,13 +950,13 @@ public class LruBlockCache implements BlockCache, HeapSize {
public int compareTo(CachedBlock other) {
int diff = this.getFilename().compareTo(other.getFilename());
if (diff != 0) return diff;
- diff = Long.compare(this.getOffset(), other.getOffset());
+ diff = (int)(this.getOffset() - other.getOffset());
if (diff != 0) return diff;
if (other.getCachedTime() < 0 || this.getCachedTime() < 0) {
throw new IllegalStateException("" + this.getCachedTime() + ", " +
other.getCachedTime());
}
- return Long.compare(other.getCachedTime(), this.getCachedTime());
+ return (int)(other.getCachedTime() - this.getCachedTime());
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/6dfdc0d8/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 319d9c4..137a909 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -1090,7 +1090,9 @@ public class BucketCache implements BlockCache, HeapSize {
@Override
public int compare(BucketEntry o1, BucketEntry o2) {
- return Long.compare(o2.accessCounter, o1.accessCounter);
+ long accessCounter1 = o1.accessCounter;
+ long accessCounter2 = o2.accessCounter;
+ return accessCounter1 < accessCounter2 ? 1 : accessCounter1 == accessCounter2 ? 0 : -1;
}
};
@@ -1211,7 +1213,9 @@ public class BucketCache implements BlockCache, HeapSize {
@Override
public int compareTo(BucketEntryGroup that) {
- return Long.compare(this.overflow(), that.overflow());
+ if (this.overflow() == that.overflow())
+ return 0;
+ return this.overflow() > that.overflow() ? 1 : -1;
}
@Override
@@ -1358,13 +1362,13 @@ public class BucketCache implements BlockCache, HeapSize {
public int compareTo(CachedBlock other) {
int diff = this.getFilename().compareTo(other.getFilename());
if (diff != 0) return diff;
- diff = Long.compare(this.getOffset(), other.getOffset());
+ diff = (int)(this.getOffset() - other.getOffset());
if (diff != 0) return diff;
if (other.getCachedTime() < 0 || this.getCachedTime() < 0) {
throw new IllegalStateException("" + this.getCachedTime() + ", " +
other.getCachedTime());
}
- return Long.compare(other.getCachedTime(), this.getCachedTime());
+ return (int)(other.getCachedTime() - this.getCachedTime());
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/6dfdc0d8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 67d4a4e..5a1f84f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -3118,7 +3118,7 @@ public class HBaseFsck extends Configured {
final Comparator<Cell> comp = new Comparator<Cell>() {
@Override
public int compare(Cell k1, Cell k2) {
- return Long.compare(k1.getTimestamp(), k2.getTimestamp());
+ return (int)(k1.getTimestamp() - k2.getTimestamp());
}
};