You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/09 03:33:55 UTC
hive git commit: HIVE-11732: LLAP: MiniLlapCluster integration broke
hadoop-1 build (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/llap 772c4b90f -> 6413e8679
HIVE-11732: LLAP: MiniLlapCluster integration broke hadoop-1 build (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6413e867
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6413e867
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6413e867
Branch: refs/heads/llap
Commit: 6413e8679b203b5ab8d20bb6287d5f91cf874c5f
Parents: 772c4b9
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Tue Sep 8 20:25:51 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Tue Sep 8 20:25:51 2015 -0500
----------------------------------------------------------------------
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 2 +-
itests/qtest/pom.xml | 26 +++----
itests/util/pom.xml | 14 ----
.../org/apache/hadoop/hive/ql/QTestUtil.java | 54 +-------------
.../hive/llap/daemon/MiniLlapCluster.java | 18 +++++
.../hadoop/hive/shims/Hadoop20SShims.java | 6 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 77 +++++++++++++++++---
.../apache/hadoop/hive/shims/HadoopShims.java | 2 +-
8 files changed, 107 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index 32a0ae8..f635347 100644
--- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -181,7 +181,7 @@ public class MiniHS2 extends AbstractHiveService {
// Initialize the execution engine based on cluster type
switch (miniClusterType) {
case TEZ:
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, false);
break;
case MR:
mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1);
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 6c2be68..c276db9 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -97,19 +97,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-llap-client</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-llap-server</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-registry</artifactId>
<version>${hadoop-23.version}</version>
@@ -270,6 +257,19 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-server</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-server</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${commons-logging.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/itests/util/pom.xml
----------------------------------------------------------------------
diff --git a/itests/util/pom.xml b/itests/util/pom.xml
index d641189..0743f01 100644
--- a/itests/util/pom.xml
+++ b/itests/util/pom.xml
@@ -54,20 +54,6 @@
<artifactId>hive-common</artifactId>
<version>${project.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-llap-server</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-llap-server</artifactId>
- <type>test-jar</type>
- <version>${project.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-cli</artifactId>
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 520f0ec..68141ea 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -45,10 +45,8 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
@@ -60,7 +58,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -73,9 +70,6 @@ import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream;
import org.apache.hadoop.hive.common.io.SortPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
-import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
-import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -154,7 +148,6 @@ public class QTestUtil {
private HadoopShims.MiniMrShim mr = null;
private HadoopShims.MiniDFSShim dfs = null;
private HadoopShims.HdfsEncryptionShim hes = null;
- private MiniLlapCluster llapCluster = null;
private final boolean miniMr = false;
private String hadoopVer = null;
private QTestSetup setup = null;
@@ -409,48 +402,13 @@ public class QTestUtil {
String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
if (clusterType == MiniClusterType.tez) {
- mr = shims.getMiniTezCluster(conf, 4, uriString);
+ mr = shims.getMiniTezCluster(conf, 4, uriString, false);
} else if (clusterType == MiniClusterType.llap) {
- Configuration daemonConf;
if (confDir != null && !confDir.isEmpty()) {
- URL llapDaemonConfURL = new URL("file://"
- + new File(confDir).toURI().getPath() + "/llap-daemon-site.xml");
- daemonConf = new LlapConfiguration(conf, llapDaemonConfURL);
- } else {
- daemonConf = new LlapConfiguration(conf);
- }
- final String clusterName = "llap";
- final long maxMemory = LlapDaemon.getTotalHeapSize();
- // 15% for io cache
- final long memoryForCache = (long) (0.15f * maxMemory);
- // 75% for 4 executors
- final long totalExecutorMemory = (long) (0.75f * maxMemory);
- final int numExecutors = daemonConf.getInt(LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS,
- LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT);
- final boolean asyncIOEnabled = true;
- // enabling this will cause test failures in Mac OS X
- final boolean directMemoryEnabled = false;
- final int numLocalDirs = 1;
- LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache
- + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors
- + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled
- + " numLocalDirs: " + numLocalDirs);
- llapCluster = MiniLlapCluster.create(clusterName,
- numExecutors,
- totalExecutorMemory,
- asyncIOEnabled,
- directMemoryEnabled,
- memoryForCache,
- numLocalDirs);
- llapCluster.init(daemonConf);
- llapCluster.start();
- Configuration llapConf = llapCluster.getClusterSpecificConfiguration();
- Iterator<Entry<String, String>> confIter = llapConf.iterator();
- while (confIter.hasNext()) {
- Entry<String, String> entry = confIter.next();
- conf.set(entry.getKey(), entry.getValue());
+ conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ + "/llap-daemon-site.xml"));
}
- mr = shims.getMiniTezCluster(conf, 2, uriString);
+ mr = shims.getMiniTezCluster(conf, 2, uriString, true);
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
mr = shims.getMiniSparkCluster(conf, 4, uriString, 1);
} else {
@@ -501,10 +459,6 @@ public class QTestUtil {
sparkSession = null;
}
}
- if (llapCluster != null) {
- llapCluster.stop();
- llapCluster = null;
- }
if (mr != null) {
mr.shutdown();
mr = null;
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
index 7fb9900..df967c4 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
@@ -17,6 +17,8 @@ package org.apache.hadoop.hive.llap.daemon;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -54,6 +56,22 @@ public class MiniLlapCluster extends AbstractService {
llapIoEnabled, ioIsDirect, ioBytesPerService, numLocalDirs);
}
+ public static MiniLlapCluster createAndLaunch(Configuration conf, String clusterName,
+ int numExecutorsPerService, long execBytePerService, boolean llapIoEnabled,
+ boolean ioIsDirect, long ioBytesPerService, int numLocalDirs) {
+ MiniLlapCluster miniLlapCluster = create(clusterName, numExecutorsPerService,
+ execBytePerService, llapIoEnabled, ioIsDirect, ioBytesPerService, numLocalDirs);
+ miniLlapCluster.init(conf);
+ miniLlapCluster.start();
+ Configuration llapConf = miniLlapCluster.getClusterSpecificConfiguration();
+ Iterator<Map.Entry<String, String>> confIter = llapConf.iterator();
+ while (confIter.hasNext()) {
+ Map.Entry<String, String> entry = confIter.next();
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ return miniLlapCluster;
+ }
+
// TODO Add support for multiple instances
private MiniLlapCluster(String clusterName, int numExecutorsPerService, long execMemoryPerService,
boolean llapIoEnabled, boolean ioIsDirect, long ioBytesPerService, int numLocalDirs) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index 42d8c3d..d47e3ff 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.shims;
import java.io.IOException;
-import java.lang.Override;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
@@ -69,12 +68,11 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.KerberosName;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.tools.distcp2.DistCp;
import org.apache.hadoop.tools.distcp2.DistCpOptions;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.VersionInfo;
@@ -233,7 +231,7 @@ public class Hadoop20SShims extends HadoopShimsSecure {
@Override
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode) throws IOException {
+ String nameNode, boolean isLlap) throws IOException {
throw new IOException("Cannot run tez on current hadoop, Version: " + VersionInfo.getVersion());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index f762724..e650cfd 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -21,10 +21,14 @@ import java.io.DataInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
+import java.net.URL;
import java.nio.ByteBuffer;
import java.security.AccessControlException;
import java.security.NoSuchAlgorithmException;
@@ -65,13 +69,11 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
-import org.apache.hadoop.hdfs.client.HdfsAdmin;
-import org.apache.hadoop.hdfs.protocol.EncryptionZone;
-import org.apache.hadoop.hive.shims.HadoopShims.TextReaderShim;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.ClusterStatus;
@@ -93,9 +95,9 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.util.Progressable;
@@ -372,8 +374,8 @@ public class Hadoop23Shims extends HadoopShimsSecure {
*/
@Override
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode) throws IOException {
- return new MiniTezShim(conf, numberOfTaskTrackers, nameNode);
+ String nameNode, boolean isLlap) throws IOException {
+ return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, isLlap);
}
/**
@@ -383,9 +385,16 @@ public class Hadoop23Shims extends HadoopShimsSecure {
private final MiniTezCluster mr;
private final Configuration conf;
+ private Class<?> miniLlapKlass;
+ private Object miniLlapCluster;
- public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode)
- throws IOException {
+ public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode,
+ boolean isLlap) throws IOException {
+ if (isLlap) {
+ createAndLaunchLlapDaemon(conf);
+ } else {
+ miniLlapCluster = null;
+ }
mr = new MiniTezCluster("hive", numberOfTaskTrackers);
conf.set("fs.defaultFS", nameNode);
conf.set("tez.am.log.level", "DEBUG");
@@ -395,6 +404,47 @@ public class Hadoop23Shims extends HadoopShimsSecure {
this.conf = mr.getConfig();
}
+ private void createAndLaunchLlapDaemon(final Configuration conf) {
+ try {
+ final String clusterName = "llap";
+ Class<?> llapDaemonKlass =
+ Class.forName("org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon",
+ false, ShimLoader.class.getClassLoader());
+ Method totalMemMethod = llapDaemonKlass.getMethod("getTotalHeapSize");
+ final long maxMemory = (long) totalMemMethod.invoke(null);
+ // 15% for io cache
+ final long memoryForCache = (long) (0.15f * maxMemory);
+ // 75% for executors
+ final long totalExecutorMemory = (long) (0.75f * maxMemory);
+ final int numExecutors = conf.getInt("llap.daemon.num.executors", 4);
+ final boolean asyncIOEnabled = true;
+ // enabling this will cause test failures in Mac OS X
+ final boolean directMemoryEnabled = false;
+ final int numLocalDirs = 1;
+ LOG.info("MiniLlap Configs - maxMemory: " + maxMemory + " memoryForCache: " + memoryForCache
+ + " totalExecutorMemory: " + totalExecutorMemory + " numExecutors: " + numExecutors
+ + " asyncIOEnabled: " + asyncIOEnabled + " directMemoryEnabled: " + directMemoryEnabled
+ + " numLocalDirs: " + numLocalDirs);
+
+ miniLlapKlass = Class.forName("org.apache.hadoop.hive.llap.daemon.MiniLlapCluster",
+ false, ShimLoader.class.getClassLoader());
+ Method create = miniLlapKlass.getMethod("createAndLaunch", new Class[]{Configuration.class,
+ String.class, Integer.TYPE, Long.TYPE, Boolean.TYPE, Boolean.TYPE,
+ Long.TYPE, Integer.TYPE});
+ miniLlapCluster = create.invoke(null,
+ conf,
+ clusterName,
+ numExecutors,
+ totalExecutorMemory,
+ asyncIOEnabled,
+ directMemoryEnabled,
+ memoryForCache,
+ numLocalDirs);
+ } catch (Exception e) {
+ LOG.error("Unable to create MiniLlapCluster. Exception: " + e.getMessage());
+ }
+ }
+
@Override
public int getJobTrackerPort() throws UnsupportedOperationException {
String address = conf.get("yarn.resourcemanager.address");
@@ -410,6 +460,15 @@ public class Hadoop23Shims extends HadoopShimsSecure {
@Override
public void shutdown() throws IOException {
mr.stop();
+
+ if (miniLlapKlass != null && miniLlapCluster != null) {
+ try {
+ Method stop = miniLlapKlass.getMethod("stop", new Class[]{});
+ stop.invoke(miniLlapCluster);
+ } catch (Exception e) {
+ LOG.error("Unable to stop llap daemon. Exception: " + e.getMessage());
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/6413e867/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index f108068..dae9a1d 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -97,7 +97,7 @@ public interface HadoopShims {
String nameNode, int numDir) throws IOException;
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode) throws IOException;
+ String nameNode, boolean isLlap) throws IOException;
public MiniMrShim getMiniSparkCluster(Configuration conf, int numberOfTaskTrackers,
String nameNode, int numDir) throws IOException;