You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2019/07/26 13:32:32 UTC
[hive] branch master updated: HIVE-22043: Make LLAP's Yarn package
dir on HDFS configurable (Adam Szita, reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
szita pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4f7feaa HIVE-22043: Make LLAP's Yarn package dir on HDFS configurable (Adam Szita, reviewed by Ashutosh Chauhan)
4f7feaa is described below
commit 4f7feaad4e91e140d283b5ed2af06d06fdfdae0b
Author: Adam Szita <sz...@cloudera.com>
AuthorDate: Wed Jul 24 18:09:28 2019 +0200
HIVE-22043: Make LLAP's Yarn package dir on HDFS configurable (Adam Szita, reviewed by Ashutosh Chauhan)
---
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../hive/llap/cli/service/LlapConfigJsonCreator.java | 3 +++
.../hadoop/hive/llap/cli/service/LlapServiceDriver.java | 8 +++++---
llap-server/src/main/resources/package.py | 14 ++++++++------
llap-server/src/main/resources/templates.py | 6 +++---
5 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 3088f99..776a449 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4262,6 +4262,9 @@ public class HiveConf extends Configuration {
"Port to use for LLAP plugin rpc server"),
LLAP_PLUGIN_RPC_NUM_HANDLERS("hive.llap.plugin.rpc.num.handlers", 1,
"Number of RPC handlers for AM LLAP plugin endpoint."),
+ LLAP_HDFS_PACKAGE_DIR("hive.llap.hdfs.package.dir", ".yarn",
+ "Package directory on HDFS used for holding collected configuration and libraries" +
+ " required for YARN launch. Note: this should be set to the same as yarn.service.base.path"),
LLAP_DAEMON_WORK_DIRS("hive.llap.daemon.work.dirs", "",
"Working directories for the daemon. This should not be set if running as a YARN\n" +
"Service. It must be set when not running on YARN. If the value is set when\n" +
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapConfigJsonCreator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapConfigJsonCreator.java
index 8e9b939..750a69d 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapConfigJsonCreator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapConfigJsonCreator.java
@@ -102,6 +102,9 @@ class LlapConfigJsonCreator {
long maxDirect = (xmx > 0 && cache > 0 && xmx < cache * 1.25) ? (long) (cache * 1.25) : -1;
configs.put("max_direct_memory", Long.toString(maxDirect));
+ configs.put(ConfVars.LLAP_HDFS_PACKAGE_DIR.varname,
+ conf.getVar(ConfVars.LLAP_HDFS_PACKAGE_DIR));
+
return configs;
}
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java
index bbc7265..033cabf 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/LlapServiceDriver.java
@@ -57,7 +57,7 @@ import java.util.concurrent.Executors;
public class LlapServiceDriver {
private static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName());
- private static final String LLAP_PACKAGE_DIR = ".yarn/package/LLAP/";
+ private static final String LLAP_RELATIVE_PACKAGE_DIR = "/package/LLAP/";
private static final String OUTPUT_DIR_PREFIX = "llap-yarn-";
/**
@@ -356,8 +356,10 @@ public class LlapServiceDriver {
}
LOG.info("Uploading the app tarball");
CoreFileSystem fs = new CoreFileSystem(conf);
- fs.createWithPermissions(new Path(LLAP_PACKAGE_DIR), FsPermission.getDirDefault());
- fs.copyLocalFileToHdfs(new File(packageDir.toString(), packageName), new Path(LLAP_PACKAGE_DIR),
+ String llapPackageDir = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_HDFS_PACKAGE_DIR)
+ + LLAP_RELATIVE_PACKAGE_DIR;
+ fs.createWithPermissions(new Path(llapPackageDir), FsPermission.getDirDefault());
+ fs.copyLocalFileToHdfs(new File(packageDir.toString(), packageName), new Path(llapPackageDir),
new FsPermission("755"));
LOG.info("Executing the launch command");
diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py
index c48ff79..1e777f6 100644
--- a/llap-server/src/main/resources/package.py
+++ b/llap-server/src/main/resources/package.py
@@ -73,7 +73,7 @@ def construct_service_site_global_string(kvs):
def main(args):
version = os.getenv("HIVE_VERSION")
if not version:
- version = strftime("%d%b%Y", gmtime())
+ version = strftime("%d%b%Y", gmtime())
home = os.getenv("HIVE_HOME")
output = "llap-yarn-%(version)s" % ({"version": version})
parser = argparse.ArgumentParser()
@@ -116,10 +116,12 @@ def main(args):
service_keytab_dir = args.service_keytab_dir
service_keytab = args.service_keytab
service_principal = args.service_principal
+
+ config = json_parse(open(join(input, "config.json")).read())
# set the defaults only if the defaults are enabled
if args.service_default_keytab:
if not service_keytab_dir:
- service_keytab_dir = ".yarn/keytabs/llap"
+ service_keytab_dir = config["hive.llap.hdfs.package.dir"] + "/keytabs/llap"
if not service_keytab:
service_keytab = "llap.keytab"
if not service_principal:
@@ -135,7 +137,6 @@ def main(args):
print "Cannot find input files"
sys.exit(1)
return
- config = json_parse(open(join(input, "config.json")).read())
java_home = config["java.home"]
max_direct_memory = config["max_direct_memory"]
@@ -167,12 +168,13 @@ def main(args):
"placement" : args.service_placement,
"health_percent": args.health_percent,
"health_time_window": args.health_time_window_secs,
- "health_init_delay": args.health_init_delay_secs
+ "health_init_delay": args.health_init_delay_secs,
+ "hdfs_package_dir": config["hive.llap.hdfs.package.dir"]
}
-
+
if not exists(output):
os.makedirs(output)
-
+
src = join(home, "scripts", "llap", "bin")
dst = join(input, "bin")
if exists(dst):
diff --git a/llap-server/src/main/resources/templates.py b/llap-server/src/main/resources/templates.py
index 1dc1611..81e2fa9 100644
--- a/llap-server/src/main/resources/templates.py
+++ b/llap-server/src/main/resources/templates.py
@@ -17,7 +17,7 @@ yarnfile = """
"number_of_containers": %(instances)d,
"launch_command": "$LLAP_DAEMON_BIN_HOME/llapDaemon.sh start &> $LLAP_DAEMON_TMP_DIR/shell.out",
"artifact": {
- "id": ".yarn/package/LLAP/llap-%(version)s.tar.gz",
+ "id": "%(hdfs_package_dir)s/package/LLAP/llap-%(version)s.tar.gz",
"type": "TARBALL"
},
"resource": {
@@ -73,7 +73,7 @@ runner = """
BASEDIR=$(dirname $0)
yarn app -stop %(name)s
yarn app -destroy %(name)s
-hdfs dfs -mkdir -p .yarn/package/LLAP
-hdfs dfs -copyFromLocal -f $BASEDIR/llap-%(version)s.tar.gz .yarn/package/LLAP
+hdfs dfs -mkdir -p %(hdfs_package_dir)s/package/LLAP
+hdfs dfs -copyFromLocal -f $BASEDIR/llap-%(version)s.tar.gz %(hdfs_package_dir)s/package/LLAP
yarn app -launch %(name)s $BASEDIR/Yarnfile
"""