You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2013/01/09 22:13:06 UTC
svn commit: r1431081 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/coprocessor/
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/regionserver/ main/resources/
test/java/org/apache/hadoop/hbase/ test/ja...
Author: enis
Date: Wed Jan 9 21:13:06 2013
New Revision: 1431081
URL: http://svn.apache.org/viewvc?rev=1431081&view=rev
Log:
HBASE-6824. Introduce ${hbase.local.dir} and save coprocessor jars there
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
hbase/trunk/hbase-server/src/main/resources/hbase-default.xml
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java?rev=1431081&r1=1431080&r2=1431081&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java Wed Jan 9 21:13:06 2013
@@ -72,6 +72,10 @@ public abstract class CoprocessorHost<E
public static final String WAL_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.wal.classes";
+ //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/
+ private static final String COPROCESSOR_JARS_DIR = File.separator
+ + "coprocessor" + File.separator + "jars" + File.separator;
+
private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
/** Ordered set of loaded coprocessors with lock */
protected SortedSet<E> coprocessors =
@@ -209,13 +213,13 @@ public abstract class CoprocessorHost<E
if (!path.toString().endsWith(".jar")) {
throw new IOException(path.toString() + ": not a jar file?");
}
- FileSystem fs = path.getFileSystem(HBaseConfiguration.create());
- Path dst = new Path(System.getProperty("java.io.tmpdir") +
- java.io.File.separator +"." + pathPrefix +
+ FileSystem fs = path.getFileSystem(this.conf);
+ File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
+ parentDir.mkdirs();
+ File dst = new File(parentDir, "." + pathPrefix +
"." + className + "." + System.currentTimeMillis() + ".jar");
- fs.copyToLocalFile(path, dst);
- File tmpLocal = new File(dst.toString());
- tmpLocal.deleteOnExit();
+ fs.copyToLocalFile(path, new Path(dst.toString()));
+ dst.deleteOnExit();
// TODO: code weaving goes here
@@ -229,7 +233,7 @@ public abstract class CoprocessorHost<E
// unsurprisingly wants URLs, not URIs; so we will use the deprecated
// method which returns URLs for as long as it is available
List<URL> paths = new ArrayList<URL>();
- URL url = new File(dst.toString()).getCanonicalFile().toURL();
+ URL url = dst.getCanonicalFile().toURL();
paths.add(url);
JarFile jarFile = new JarFile(dst.toString());
@@ -237,8 +241,7 @@ public abstract class CoprocessorHost<E
while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement();
if (entry.getName().matches("/lib/[^/]+\\.jar")) {
- File file = new File(System.getProperty("java.io.tmpdir") +
- java.io.File.separator +"." + pathPrefix +
+ File file = new File(parentDir, "." + pathPrefix +
"." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
file.deleteOnExit();
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java?rev=1431081&r1=1431080&r2=1431081&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java Wed Jan 9 21:13:06 2013
@@ -63,6 +63,7 @@ public class MasterCoprocessorHost
private MasterServices masterServices;
MasterCoprocessorHost(final MasterServices services, final Configuration conf) {
+ this.conf = conf;
this.masterServices = services;
loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java?rev=1431081&r1=1431080&r2=1431081&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java Wed Jan 9 21:13:06 2013
@@ -134,6 +134,7 @@ public class RegionCoprocessorHost
*/
public RegionCoprocessorHost(final HRegion region,
final RegionServerServices rsServices, final Configuration conf) {
+ this.conf = conf;
this.rsServices = rsServices;
this.region = region;
this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode());
Modified: hbase/trunk/hbase-server/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/resources/hbase-default.xml?rev=1431081&r1=1431080&r2=1431081&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/hbase-server/src/main/resources/hbase-default.xml Wed Jan 9 21:13:06 2013
@@ -49,7 +49,7 @@
</property>
<property>
<name>hbase.tmp.dir</name>
- <value>/tmp/hbase-${user.name}</value>
+ <value>${java.io.tmpdir}/hbase-${user.name}</value>
<description>Temporary directory on the local filesystem.
Change this setting to point to a location more permanent
than '/tmp' (The '/tmp' directory is often cleared on
@@ -57,6 +57,13 @@
</description>
</property>
<property>
+ <name>hbase.local.dir</name>
+ <value>${hbase.tmp.dir}/local/</value>
+ <description>Directory on the local filesystem to be used
+ as a local storage.
+ </description>
+ </property>
+ <property>
<name>hbase.master.info.port</name>
<value>60010</value>
<description>The port for the HBase Master web UI.
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1431081&r1=1431080&r2=1431081&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Wed Jan 9 21:13:06 2013
@@ -266,6 +266,9 @@ public class HBaseTestingUtility extends
"mapred.local.dir",
testPath, "mapred-local-dir");
+ createSubDir(
+ "hbase.local.dir",
+ testPath, "hbase-local-dir");
return testPath;
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java?rev=1431081&r1=1431080&r2=1431081&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java Wed Jan 9 21:13:06 2013
@@ -169,7 +169,7 @@ public class TestClassLoading {
// the classpath is {hbaseSrc}/target/classes.
String currentDir = new File(".").getAbsolutePath();
String classpath =
- currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" +
+ currentDir + File.separator + "target"+ File.separator + "classes" +
System.getProperty("path.separator") +
// Note that the below trick only works if mvn is running the test;
// doesn't work in eclipse for example.
@@ -304,6 +304,10 @@ public class TestClassLoading {
}
}
+ private String getLocalPath(File file) {
+ return new Path(file.toURI()).toString();
+ }
+
@Test
// HBASE-3516: Test CP Class loading from local file system
public void testClassLoadingFromLocalFS() throws Exception {
@@ -312,7 +316,7 @@ public class TestClassLoading {
// create a table that references the jar
HTableDescriptor htd = new HTableDescriptor(cpName3);
htd.addFamily(new HColumnDescriptor("test"));
- htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName3 + "|" +
+ htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
Coprocessor.PRIORITY_USER);
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(htd);
@@ -338,7 +342,7 @@ public class TestClassLoading {
// create a table that references the jar
HTableDescriptor htd = new HTableDescriptor(cpName4);
htd.addFamily(new HColumnDescriptor("test"));
- htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName4 + "|" +
+ htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
Coprocessor.PRIORITY_USER);
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(htd);
@@ -376,9 +380,9 @@ public class TestClassLoading {
String cpKey2 = " Coprocessor$2 ";
String cpKey3 = " coprocessor$03 ";
- String cpValue1 = jarFile1.toString() + "|" + cpName1 + "|" +
+ String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
Coprocessor.PRIORITY_USER;
- String cpValue2 = jarFile2.toString() + " | " + cpName2 + " | ";
+ String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
// load from default class loader
String cpValue3 =
" | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
@@ -393,13 +397,13 @@ public class TestClassLoading {
htd.setValue(cpKey3, cpValue3);
// add 2 coprocessor by using new htd.addCoprocessor() api
- htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()),
+ htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
Coprocessor.PRIORITY_USER, null);
Map<String, String> kvs = new HashMap<String, String>();
kvs.put("k1", "v1");
kvs.put("k2", "v2");
kvs.put("k3", "v3");
- htd.addCoprocessor(cpName6, new Path(jarFile6.getPath()),
+ htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
Coprocessor.PRIORITY_USER, kvs);
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();