You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2020/06/05 23:47:07 UTC
[hive] branch master updated: HIVE-23582: LLAP: Make
SplitLocationProvider impl pluggable (Prasanth Jayachandran reviewed by
Gopal V)
This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e633ff8 HIVE-23582: LLAP: Make SplitLocationProvider impl pluggable (Prasanth Jayachandran reviewed by Gopal V)
e633ff8 is described below
commit e633ff8186c14377443881d8eb02f2644e20b510
Author: Prasanth Jayachandran <pr...@apache.org>
AuthorDate: Fri Jun 5 16:46:27 2020 -0700
HIVE-23582: LLAP: Make SplitLocationProvider impl pluggable (Prasanth
Jayachandran reviewed by Gopal V)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 4 ++
.../org/apache/hadoop/hive/ql/exec/tez/Utils.java | 27 ++++++++++-
.../apache/hadoop/hive/ql/exec/tez/TestUtils.java | 53 ++++++++++++++++++++++
3 files changed, 82 insertions(+), 2 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index abd12c9..1464d6a 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4704,6 +4704,10 @@ public class HiveConf extends Configuration {
"instead of using the locations provided by the split itself. If there is no llap daemon " +
"running, fall back to locations provided by the split. This is effective only if " +
"hive.execution.mode is llap"),
+ LLAP_SPLIT_LOCATION_PROVIDER_CLASS("hive.llap.split.location.provider.class",
+ "org.apache.hadoop.hive.ql.exec.tez.HostAffinitySplitLocationProvider",
+ "Split location provider class to use during split generation for LLAP. This class should implement\n" +
+ "org.apache.hadoop.mapred.split.SplitLocationProvider interface"),
LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true,
"Whether LLAP should reject permissive ACLs in some cases (e.g. its own management\n" +
"protocol or ZK paths), similar to how ssh refuses a key with bad access permissions."),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
index d266bb1..92d64df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.split.SplitLocationProvider;
import org.apache.tez.common.counters.TezCounters;
@@ -48,8 +50,29 @@ public class Utils {
&& HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS)
&& useCacheAffinity;
SplitLocationProvider splitLocationProvider;
- LOG.info("SplitGenerator using llap affinitized locations: " + useCustomLocations);
- if (useCustomLocations) {
+ final String locationProviderClass = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS);
+ final boolean customLocationProvider =
+ !HostAffinitySplitLocationProvider.class.getName().equals(locationProviderClass);
+ LOG.info("SplitGenerator using llap affinitized locations: {} locationProviderClass: {}", useCustomLocations, locationProviderClass);
+ if (customLocationProvider) {
+ SplitLocationProvider locationProviderImpl;
+ try {
+ // the implementation of SplitLocationProvider may have Configuration as a single arg constructor, so we try
+ // invoking that constructor first. If that does not exist, the fallback will use no-arg constructor.
+ locationProviderImpl = JavaUtils
+ .newInstance(JavaUtils.getClass(locationProviderClass, SplitLocationProvider.class),
+ new Class<?>[]{Configuration.class}, new Object[]{conf});
+ } catch (Exception e) {
+ LOG.warn("Unable to instantiate {} class. Will try no-arg constructor invocation..", locationProviderClass, e);
+ try {
+ locationProviderImpl = JavaUtils.newInstance(JavaUtils.getClass(locationProviderClass,
+ SplitLocationProvider.class));
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
+ return locationProviderImpl;
+ } else if (useCustomLocations) {
LlapRegistryService serviceRegistry = LlapRegistryService.getClient(conf);
return getCustomSplitLocationProvider(serviceRegistry, LOG);
} else {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java
index d9aa032..71ce812 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.tez;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet;
@@ -50,6 +51,7 @@ import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
@@ -156,4 +158,55 @@ public class TestUtils {
knownLocations.remove(null);
assertArrayEquals(expectedLocations, knownLocations.toArray(new String[] {}));
}
+
+ public static class NoConstructorSplitLocationProvider implements SplitLocationProvider {
+
+ @Override
+ public String[] getLocations(final InputSplit inputSplit) throws IOException {
+ return new String[0];
+ }
+ }
+
+ public static class DefaultConstructorSplitLocationProvider implements SplitLocationProvider {
+
+ public DefaultConstructorSplitLocationProvider() {
+
+ }
+
+ @Override
+ public String[] getLocations(final InputSplit inputSplit) throws IOException {
+ return new String[0];
+ }
+ }
+
+ public static class ConfConstructorSplitLocationProvider implements SplitLocationProvider {
+ private Configuration configuration;
+
+ public ConfConstructorSplitLocationProvider(Configuration conf) {
+ this.configuration = conf;
+ }
+
+ @Override
+ public String[] getLocations(final InputSplit inputSplit) throws IOException {
+ return new String[0];
+ }
+ }
+
+ @Test
+ public void testCustomSplitLocationProvider() throws IOException {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_MODE, "llap");
+
+ conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS, NoConstructorSplitLocationProvider.class.getName());
+ SplitLocationProvider splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG);
+ assertTrue(splitLocationProvider instanceof NoConstructorSplitLocationProvider);
+
+ conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS, DefaultConstructorSplitLocationProvider.class.getName());
+ splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG);
+ assertTrue(splitLocationProvider instanceof DefaultConstructorSplitLocationProvider);
+
+ conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS, ConfConstructorSplitLocationProvider.class.getName());
+ splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG);
+ assertTrue(splitLocationProvider instanceof ConfConstructorSplitLocationProvider);
+ }
}