You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2020/03/10 04:43:28 UTC
[hive] branch master updated: HIVE-22988: LLAP: If consistent
splits is disabled ordering instances is not required (Prasanth
Jayachandran reviewed by Slim Bouguerra)
This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b5de84a HIVE-22988: LLAP: If consistent splits is disabled ordering instances is not required (Prasanth Jayachandran reviewed by Slim Bouguerra)
b5de84a is described below
commit b5de84a6ce27904562bb15d7fe800b6ae3c32670
Author: Prasanth Jayachandran <pr...@apache.org>
AuthorDate: Mon Mar 9 21:42:33 2020 -0700
HIVE-22988: LLAP: If consistent splits is disabled ordering instances is not required (Prasanth Jayachandran reviewed by Slim Bouguerra)
---
.../hive/llap/tezplugins/LlapTaskSchedulerService.java | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
index d11bf13..48501e5 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -296,6 +296,7 @@ public class LlapTaskSchedulerService extends TaskScheduler {
private int totalGuaranteed = 0, unusedGuaranteed = 0;
+ private final boolean consistentSplits;
/**
* An internal version to make sure we don't race and overwrite a newer totalGuaranteed count in
* ZK with an older one, without requiring us to make ZK updates under the main writeLock.
@@ -345,6 +346,7 @@ public class LlapTaskSchedulerService extends TaskScheduler {
LOCK_METRICS);
readLock = lock.readLock();
writeLock = lock.writeLock();
+ this.consistentSplits = HiveConf.getBoolVar(conf, ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS);
if (conf.getBoolean(LLAP_PLUGIN_ENDPOINT_ENABLED, false)) {
JobTokenSecretManager sm = null;
@@ -444,8 +446,8 @@ public class LlapTaskSchedulerService extends TaskScheduler {
String hostsString = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
LOG.info("Running with configuration: hosts={}, numSchedulableTasksPerNode={}, "
- + "nodeBlacklistConf={}, localityConf={}",
- hostsString, numSchedulableTasksPerNode, nodeBlacklistConf, localityDelayConf);
+ + "nodeBlacklistConf={}, localityConf={} consistentSplits={}",
+ hostsString, numSchedulableTasksPerNode, nodeBlacklistConf, localityDelayConf, consistentSplits);
this.amRegistry = TezAmRegistryImpl.create(conf, true);
synchronized (LlapTaskCommunicator.pluginInitLock) {
@@ -1476,7 +1478,13 @@ public class LlapTaskSchedulerService extends TaskScheduler {
}
/* fall through - miss in locality or no locality-requested */
- Collection<LlapServiceInstance> instances = activeInstances.getAllInstancesOrdered(true);
+ Collection<LlapServiceInstance> instances;
+ if (consistentSplits) {
+ instances = activeInstances.getAllInstancesOrdered(true);
+ } else {
+ // if consistent splits are not used we don't need the ordering as there will be no cache benefit anyways
+ instances = activeInstances.getAll();
+ }
List<NodeInfo> allNodes = new ArrayList<>(instances.size());
List<NodeInfo> activeNodesWithFreeSlots = new ArrayList<>();
for (LlapServiceInstance inst : instances) {