You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/16 15:10:50 UTC
kylin git commit: KYLIN-2428 support findJar by PreferLibraryName
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2428 f10d9d9ba -> 303dea95d
KYLIN-2428 support findJar by PreferLibraryName
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/303dea95
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/303dea95
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/303dea95
Branch: refs/heads/KYLIN-2428
Commit: 303dea95d262c5f334f10dc52db517f89e82e2d4
Parents: f10d9d9
Author: Billy Liu <bi...@apache.org>
Authored: Thu Feb 16 23:10:38 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Feb 16 23:10:38 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ClassUtil.java | 50 ++++++++++++++++++++
.../apache/kylin/common/util/ClassUtilTest.java | 32 +++++++++++++
.../spark/SparkBatchCubingJobBuilder2.java | 22 ++++-----
3 files changed, 93 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/303dea95/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index 3c63c41..9af34c3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -19,9 +19,12 @@
package org.apache.kylin.common.util;
import java.io.File;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
+import java.net.URLDecoder;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.WeakHashMap;
@@ -89,4 +92,51 @@ public class ClassUtil {
}
}
+ public static String findContainingJar(Class<?> clazz) {
+ return findContainingJar(clazz, null);
+ }
+
+ /**
+ * Load the first jar library contains clazz with preferJarKeyword matched. If preferJarKeyword is null, just load the
+ * jar likes Hadoop Commons' ClassUtil
+ * @param clazz
+ * @param preferJarKeyWord
+ * @return
+ */
+ public static String findContainingJar(Class<?> clazz, String preferJarKeyWord) {
+ ClassLoader loader = clazz.getClassLoader();
+ String classFile = clazz.getName().replaceAll("\\.", "/") + ".class";
+
+ try {
+ Enumeration e = loader.getResources(classFile);
+
+ URL url = null;
+ do {
+ if (!e.hasMoreElements()) {
+ if (url == null)
+ return null;
+ else
+ break;
+ }
+
+ url = (URL) e.nextElement();
+ if (!"jar".equals(url.getProtocol()))
+ break;
+ if (preferJarKeyWord != null && url.getPath().indexOf(preferJarKeyWord) != -1)
+ break;
+ if (preferJarKeyWord == null)
+ break;
+ } while (true);
+
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ return toReturn.replaceAll("!.*$", "");
+ } catch (IOException var6) {
+ throw new RuntimeException(var6);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/303dea95/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
new file mode 100644
index 0000000..75fa574
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ClassUtilTest {
+
+ @Test
+ public void testFindContainingJar() throws ClassNotFoundException {
+ Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.beanutils.BeanUtils")).contains("commons-beanutils"));
+ Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.beanutils.BeanUtils"), "core").contains("commons-beanutils-core"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/303dea95/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 00b9378..f68194e 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -18,8 +18,8 @@
package org.apache.kylin.engine.spark;
-import org.apache.hadoop.util.ClassUtil;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.EngineFactory;
@@ -53,14 +53,14 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
StringBuilder jars = new StringBuilder();
- StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace")); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration")); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
- StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar
- StringUtil.appendWithSeparator(jars, findJar("com.google.common.collect.Maps")); //guava.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration", null)); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace", null)); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration", null)); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection", null)); // hbase-client.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration", null)); // hbase-common.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer", null)); // hbase-protocol.jar
+ StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge", null)); // metrics-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("com.google.common.collect.Maps", "guava")); //guava.jar
StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars());
sparkExecutable.setJars(jars.toString());
@@ -74,9 +74,9 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
}
- private String findJar(String className) {
+ private String findJar(String className, String perferLibraryName) {
try {
- return ClassUtil.findContainingJar(Class.forName(className));
+ return ClassUtil.findContainingJar(Class.forName(className), perferLibraryName);
} catch (ClassNotFoundException e) {
logger.warn("failed to locate jar for class " + className + ", ignore it", e);
}