You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/02/02 23:15:10 UTC
[1/3] hive git commit: HIVE-12985 : distribution configs are wrong
and out of date (Sergey Shelukhin, reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/branch-2.0 b45f4c69f -> 902fcb822
refs/heads/master db4bdb3f1 -> 1b6303849
HIVE-12985 : distribution configs are wrong and out of date (Sergey Shelukhin, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/902fcb82
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/902fcb82
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/902fcb82
Branch: refs/heads/branch-2.0
Commit: 902fcb8223d32482a1be306ffb9235f0d89a9538
Parents: b45f4c6
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Feb 2 14:13:39 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Feb 2 14:13:39 2016 -0800
----------------------------------------------------------------------
packaging/pom.xml | 25 +++++++++++++++++++++++++
packaging/src/main/assembly/bin.xml | 8 ++++++++
packaging/src/main/assembly/src.xml | 10 +++++++++-
3 files changed, 42 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/902fcb82/packaging/pom.xml
----------------------------------------------------------------------
diff --git a/packaging/pom.xml b/packaging/pom.xml
index c10c488..580ccb0 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -177,6 +177,31 @@
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-hplsql</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-orc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-tez</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-llap-server</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/902fcb82/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index baa122a..a5f3975 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -113,6 +113,9 @@
<includes>
<include>/*.*</include>
</includes>
+ <excludes>
+ <exclude>/*.jar</exclude>
+ </excludes>
<outputDirectory>examples/files</outputDirectory>
</fileSet>
@@ -131,10 +134,15 @@
<include>hive</include>
<include>llap/**/*</include>
<include>metatool</include>
+ <include>hplsql</include>
<include>schematool</include>
<include>beeline</include>
<include>hiveserver2</include>
<include>hive-config.sh</include>
+ <include>hive-config.cmd</include>
+ <include>hive.cmd</include>
+ <include>hplsql.cmd</include>
+ <include>beeline.cmd</include>
<include>ext/**/*</include>
</includes>
<outputDirectory>bin</outputDirectory>
http://git-wip-us.apache.org/repos/asf/hive/blob/902fcb82/packaging/src/main/assembly/src.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml
index 4bbafc0..e8fc980 100644
--- a/packaging/src/main/assembly/src.xml
+++ b/packaging/src/main/assembly/src.xml
@@ -35,11 +35,12 @@
<directory>${project.parent.basedir}</directory>
<excludes>
+ <exclude>**/*.jar</exclude>
<exclude>**/target/**</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/.settings/**</exclude>
- <exclude>itests/thirdparty/**</exclude>
+ <exclude>**/thirdparty/**</exclude>
<exclude>testutils/ptest2/velocity.log*</exclude>
</excludes>
@@ -70,14 +71,20 @@
<include>find-bugs/**/*</include>
<include>hbase-handler/**/*</include>
<include>hcatalog/**/*</include>
+ <include>hplsql/**/*</include>
<include>hwi/**/*</include>
<include>itests/**/*</include>
<include>jdbc/**/*</include>
<include>contrib/**/*</include>
<include>metastore/**/*</include>
+ <include>llap-common/**/*</include>
+ <include>llap-client/**/*</include>
+ <include>llap-tez/**/*</include>
+ <include>llap-server/**/*</include>
<include>lib/**/*</include>
<include>findbugs/**/*</include>
<include>odbc/**/*</include>
+ <include>orc/**/*</include>
<include>packaging/pom.xml</include>
<include>packaging/src/**/*</include>
<include>pom.xml</include>
@@ -86,6 +93,7 @@
<include>service/**/*</include>
<include>shims/**/*</include>
<include>spark-client/**/*</include>
+ <include>storage-api/**/*</include>
<include>testutils/**/*</include>
</includes>
<outputDirectory>/</outputDirectory>
[2/3] hive git commit: HIVE-12855 : LLAP: add checks when resolving
UDFs to enforce whitelist (Sergey Shelukhin, reviewed by Jason Dere)
Posted by se...@apache.org.
HIVE-12855 : LLAP: add checks when resolving UDFs to enforce whitelist (Sergey Shelukhin, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/715f410c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/715f410c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/715f410c
Branch: refs/heads/master
Commit: 715f410c11c6fccb6766eae7b7190d0ff7753470
Parents: db4bdb3
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Feb 1 15:27:37 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Feb 2 14:13:53 2016 -0800
----------------------------------------------------------------------
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 1 -
.../llap/daemon/impl/FunctionLocalizer.java | 52 +++++--
.../hive/llap/daemon/impl/LlapDaemon.java | 60 ++++++--
.../hadoop/hive/ql/exec/FunctionRegistry.java | 11 ++
.../apache/hadoop/hive/ql/exec/Registry.java | 11 ++
.../hive/ql/exec/SerializationUtilities.java | 149 ++++++++++++++++++-
.../hive/ql/udf/generic/GenericUDFBridge.java | 25 +++-
7 files changed, 283 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/715f410c/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 0399798..d01c8ce 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -332,7 +332,6 @@ public class LlapServiceDriver {
}
}
-// TODO#: assumes throw
private void localizeJarForClass(FileSystem lfs, Path libDir, String className, boolean doThrow)
throws IOException {
String jarPath = null;
http://git-wip-us.apache.org/repos/asf/hive/blob/715f410c/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
index bc0ad02..ce6112f 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
@@ -20,6 +20,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLClassLoader;
import java.util.ArrayList;
+import java.util.IdentityHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
@@ -29,12 +30,14 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.ResourceUri;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.FunctionTask;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.util.ResourceDownloader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,18 +45,21 @@ import org.slf4j.LoggerFactory;
/**
* This class localizes and manages jars for the functions allowed inside LLAP.
*/
-public class FunctionLocalizer {
+public class FunctionLocalizer implements GenericUDFBridge.UdfWhitelistChecker {
private static final String DIR_NAME = "fnresources";
private static final Logger LOG = LoggerFactory.getLogger(FunctionLocalizer.class);
private ResourceDownloader resourceDownloader;
private final LinkedBlockingQueue<LocalizerWork> workQueue = new LinkedBlockingQueue<>();
private volatile boolean isClosed = false;
private final List<String> recentlyLocalizedJars = new LinkedList<String>();
+ private final List<String> recentlyLocalizedClasses = new LinkedList<String>();
private final Thread workThread;
private final File localDir;
private final Configuration conf;
private final URLClassLoader executorClassloader;
+
+ private final IdentityHashMap<Class<?>, Boolean> allowedUdfClasses = new IdentityHashMap<>();
private final ConcurrentHashMap<String, FnResources> resourcesByFn = new ConcurrentHashMap<>();
private final ConcurrentHashMap<URI, RefCountedResource> localFiles = new ConcurrentHashMap<>();
@@ -70,7 +76,7 @@ public class FunctionLocalizer {
});
}
- void init() throws IOException {
+ public void init() throws IOException {
if (localDir.exists()) {
// TODO: We don't want some random jars of unknown provenance sitting around. Or do we care?
// Ideally, we should try to reuse jars and verify using some checksum.
@@ -80,6 +86,10 @@ public class FunctionLocalizer {
workThread.start();
}
+ public boolean isUdfAllowed(Class<?> clazz) {
+ return FunctionRegistry.isBuiltInFuncClass(clazz) || allowedUdfClasses.containsKey(clazz);
+ }
+
public ClassLoader getClassLoader() {
return executorClassloader;
}
@@ -92,7 +102,7 @@ public class FunctionLocalizer {
if (resources == null || resources.isEmpty()) continue; // Nothing to localize.
FnResources result = new FnResources();
resourcesByFn.put(fqfn, result);
- workQueue.add(new LocalizeFn(fqfn, resources, result, false));
+ workQueue.add(new LocalizeFn(fqfn, resources, result, fn.getClassName(), false));
}
workQueue.add(new RefreshClassloader());
}
@@ -145,16 +155,18 @@ public class FunctionLocalizer {
private final FnResources result;
private final String fqfn;
private final boolean doRefreshClassloader;
+ private final String className;
public LocalizeFn(String fqfn, List<ResourceUri> resources, FnResources result,
- boolean doRefreshClassloader) {
+ String className, boolean doRefreshClassloader) {
this.resources = resources;
this.result = result;
this.fqfn = fqfn;
+ this.className = className;
this.doRefreshClassloader = doRefreshClassloader;
}
public void run(FunctionLocalizer parent) throws URISyntaxException, IOException {
- parent.localizeFunctionResources(fqfn, resources, result, doRefreshClassloader);
+ parent.localizeFunctionResources(fqfn, resources, className, result, doRefreshClassloader);
}
public String toString() {
@@ -206,20 +218,35 @@ public class FunctionLocalizer {
}
} catch (Throwable t) {
// TODO: we could fall back to trying one by one and only ignore the failed ones.
- String jarringError = "Unable to register jars: ";
- for (String jar : jars) {
- jarringError += (jar + ", ");
- }
- throw new IOException(jarringError, t);
+ logRefreshError("Unable to localize jars: ", jars, t);
+ return; // logRefreshError always throws.
}
if (updatedCl != executorClassloader) {
throw new AssertionError("Classloader was replaced despite using UDFClassLoader: new "
+ updatedCl + ", old " + executorClassloader);
}
+ String[] classNames = recentlyLocalizedClasses.toArray(jars);
+ recentlyLocalizedClasses.clear();
+ try {
+ for (String className : classNames) {
+ allowedUdfClasses.put(Class.forName(className, false, executorClassloader), Boolean.TRUE);
+ }
+ } catch (Throwable t) {
+ // TODO: we could fall back to trying one by one and only ignore the failed ones.
+ logRefreshError("Unable to instantiate localized classes: ", classNames, t);
+ return; // logRefreshError always throws.
+ }
+ }
+
+ private void logRefreshError(String what, String[] items, Throwable t) throws IOException {
+ for (String item : items) {
+ what += (item + ", ");
+ }
+ throw new IOException(what, t);
}
private void localizeFunctionResources(String fqfn, List<ResourceUri> resources,
- FnResources result, boolean doRefreshClassloader) throws URISyntaxException, IOException {
+ String className, FnResources result, boolean doRefreshClassloader) throws URISyntaxException, IOException {
// We will download into fn-scoped subdirectories to avoid name collisions (we assume there
// are no collisions within the same fn). That doesn't mean we download for every fn.
if (LOG.isInfoEnabled()) {
@@ -230,6 +257,7 @@ public class FunctionLocalizer {
ResourceType rt = FunctionTask.getResourceType(resource.getResourceType());
localizeOneResource(fqfn, srcUri, rt, result);
}
+ recentlyLocalizedClasses.add(className);
if (doRefreshClassloader) {
refreshClassloader();
}
@@ -285,4 +313,4 @@ public class FunctionLocalizer {
originals.add(rcr);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/715f410c/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 94b3b41..c0d4690 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -47,6 +47,10 @@ import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem;
import org.apache.hadoop.hive.llap.metrics.MetricsUtils;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
+import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ExitUtil;
@@ -149,6 +153,20 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
// Less frequently set parameter, not passing in as a param.
int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS);
+ // Initialize the function localizer.
+ ClassLoader executorClassLoader = null;
+ if (HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS)) {
+ this.fnLocalizer = new FunctionLocalizer(daemonConf, localDirs[0]);
+ executorClassLoader = fnLocalizer.getClassLoader();
+ // Set up the hook that will disallow creating non-whitelisted UDFs anywhere in the plan.
+ // We are not using a specific hook for GenericUDFBridge - that doesn't work in MiniLlap
+ // because the daemon is embedded, so the client also gets this hook and Kryo is brittle.
+ SerializationUtilities.setGlobalHook(new LlapGlobalUdfChecker(fnLocalizer));
+ } else {
+ this.fnLocalizer = null;
+ executorClassLoader = Thread.currentThread().getContextClassLoader();
+ }
+
// Initialize the metrics system
LlapMetricsSystem.initialize("LlapDaemon");
this.pauseMonitor = new JvmPauseMonitor(daemonConf);
@@ -167,15 +185,6 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
this.server = new LlapProtocolServerImpl(
numHandlers, this, srvAddress, mngAddress, srvPort, mngPort);
- ClassLoader executorClassLoader = null;
- if (HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS)) {
- this.fnLocalizer = new FunctionLocalizer(daemonConf, localDirs[0]);
- executorClassLoader = fnLocalizer.getClassLoader();
- } else {
- this.fnLocalizer = null;
- executorClassLoader = Thread.currentThread().getContextClassLoader();
- }
-
this.containerRunner = new ContainerRunnerImpl(daemonConf, numExecutors, waitQueueSize,
enablePreemption, localDirs, this.shufflePort, srvAddress, executorMemoryBytes, metrics,
amReporter, executorClassLoader);
@@ -409,6 +418,39 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
return maxJvmMemory;
}
+ /**
+ * A global hook that checks all subclasses of GenericUDF against the whitelist. It also injects
+ * us into GenericUDFBridge-s, to check with the whitelist before instantiating a UDF.
+ */
+ private static final class LlapGlobalUdfChecker extends SerializationUtilities.Hook {
+ private FunctionLocalizer fnLocalizer;
+ public LlapGlobalUdfChecker(FunctionLocalizer fnLocalizer) {
+ this.fnLocalizer = fnLocalizer;
+ }
+
+ @Override
+ public boolean preRead(Class<?> type) {
+ // 1) Don't call postRead - we will have checked everything here.
+ // 2) Ignore GenericUDFBridge, it's checked separately in LlapUdfBridgeChecker.
+ if (GenericUDFBridge.class == type) return true; // Run post-hook.
+ if (!(GenericUDF.class.isAssignableFrom(type) || UDF.class.isAssignableFrom(type))
+ || fnLocalizer.isUdfAllowed(type)) return false;
+ throw new SecurityException("UDF " + type.getCanonicalName() + " is not allowed");
+ }
+
+ @Override
+ public Object postRead(Object o) {
+ if (o == null) return o;
+ Class<?> type = o.getClass();
+ if (GenericUDFBridge.class == type) {
+ ((GenericUDFBridge)o).setUdfChecker(fnLocalizer);
+ }
+ // This won't usually be called otherwise.
+ preRead(type);
+ return o;
+ }
+ }
+
private static class LlapDaemonUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/715f410c/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 5353062..e0e030f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -464,6 +464,12 @@ public final class FunctionRegistry {
system.registerTableFunction(NOOP_STREAMING_MAP_TABLE_FUNCTION, NoopWithMapStreamingResolver.class);
system.registerTableFunction(WINDOWING_TABLE_FUNCTION, WindowingTableFunctionResolver.class);
system.registerTableFunction(MATCH_PATH_TABLE_FUNCTION, MatchPathResolver.class);
+
+ // Arithmetic specializations are done in a convoluted manner; mark them as built-in.
+ system.registerHiddenBuiltIn(GenericUDFOPDTIMinus.class);
+ system.registerHiddenBuiltIn(GenericUDFOPDTIPlus.class);
+ system.registerHiddenBuiltIn(GenericUDFOPNumericMinus.class);
+ system.registerHiddenBuiltIn(GenericUDFOPNumericPlus.class);
}
public static String getNormalizedFunctionName(String fn) throws SemanticException {
@@ -1599,6 +1605,11 @@ public final class FunctionRegistry {
return false;
}
+ /** Unlike isBuiltInFuncExpr, does not expand GenericUdfBridge. */
+ public static boolean isBuiltInFuncClass(Class<?> clazz) {
+ return system.isBuiltInFunc(clazz);
+ }
+
/**
* Setup blocked flag for all builtin UDFs as per udf whitelist and blacklist
* @param whiteListStr
http://git-wip-us.apache.org/repos/asf/hive/blob/715f410c/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index ea9813c..16c8c8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -147,6 +147,17 @@ public class Registry {
return fI;
}
+ /**
+ * Registers the UDF class as a built-in function; used for dynamically created UDFs, like
+ * GenericUDFOP*Minus/Plus.
+ */
+ public synchronized void registerHiddenBuiltIn(Class<? extends GenericUDF> functionClass) {
+ if (!isNative) {
+ throw new RuntimeException("Builtin is not for this registry");
+ }
+ builtIns.add(functionClass);
+ }
+
public FunctionInfo registerGenericUDTF(String functionName,
Class<? extends GenericUDTF> genericUDTFClass, FunctionResource... resources) {
validateClass(genericUDTFClass, GenericUDTF.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/715f410c/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
index 0e6e2de..b05a79e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
@@ -30,7 +30,9 @@ import java.lang.reflect.Field;
import java.net.URI;
import java.sql.Timestamp;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.tuple.Pair;
@@ -60,6 +62,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Registration;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.pool.KryoFactory;
@@ -72,14 +75,154 @@ import com.esotericsoftware.kryo.serializers.FieldSerializer;
public class SerializationUtilities {
private static final String CLASS_NAME = SerializationUtilities.class.getName();
private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
+ public static class Hook {
+ public boolean preRead(Class<?> type) {
+ return true;
+ }
+
+ public Object postRead(Object o) {
+ return o;
+ }
+ }
+ private static final Map<Class<?>, Hook> kryoTypeHooks = new HashMap<>();
+ private static Hook globalHook = null;
+
+ /**
+ * Must be called before any serialization takes place (e.g. in some static/service init)!
+ * Not thread safe.
+ *
+ * Will not work if different classes are added in different order on two sides of the
+ * communication, due to explicit class registration that we use causing class ID mismatch.
+ * Some processing might be added for this later (e.g. sorting the overrides here if the order
+ * is hard to enforce, and making sure they are added symmetrically everywhere, or just
+ * reverting back to hardcoding stuff if all else fails).
+ * For now, this seems to work, but Kryo seems pretty brittle. Seems to be ok to add class on
+ * read side but not write side, the other way doesn't work. Kryo needs a proper event system,
+ * otherwise this is all rather brittle.
+ */
+ public static void addKryoTypeHook(Class<?> clazz, Hook hook) {
+ kryoTypeHooks.put(clazz, hook);
+ }
+
+ /**
+ * Must be called before any serialization takes place (e.g. in some static/service init)!
+ * Not thread safe.
+ *
+ * This is somewhat brittle because there's no way to add proper superclass hook in Kryo.
+ * On the other hand, it doesn't suffer from the mismatch problems that register() causes!
+ */
+ public static void setGlobalHook(Hook hook) {
+ globalHook = hook;
+ }
+
+ /**
+ * Provides general-purpose hooks for specific types, as well as a global hook.
+ */
+ private static class KryoWithHooks extends Kryo {
+ private Hook globalHook;
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private static final class SerializerWithHook extends com.esotericsoftware.kryo.Serializer {
+ private final com.esotericsoftware.kryo.Serializer old;
+ private final Hook hook;
+
+ private SerializerWithHook(com.esotericsoftware.kryo.Serializer old, Hook hook) {
+ this.old = old;
+ this.hook = hook;
+ }
+
+ @Override
+ public Object read(Kryo kryo, Input input, Class type) {
+ return hook.preRead(type)
+ ? hook.postRead(old.read(kryo, input, type)) : old.read(kryo, input, type);
+ }
+
+ @Override
+ public void write(Kryo kryo, Output output, Object object) {
+ // Add write hooks if needed.
+ old.write(kryo, output, object);
+ }
+ }
+
+ public Kryo processHooks(Map<Class<?>, Hook> hooks, Hook globalHook) {
+ for (Map.Entry<Class<?>, Hook> e : hooks.entrySet()) {
+ register(e.getKey(), new SerializerWithHook(
+ newDefaultSerializer(e.getKey()), e.getValue()));
+ }
+ this.globalHook = globalHook;
+ return this; // To make it more explicit below that processHooks needs to be called last.
+ }
+
+ // The globalHook stuff. There's no proper way to insert this, so we add it everywhere.
+
+ private Hook ponderGlobalPreReadHook(Class<?> clazz) {
+ Hook globalHook = this.globalHook;
+ return (globalHook != null && globalHook.preRead(clazz)) ? globalHook : null;
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> T ponderGlobalPostReadHook(Hook hook, T result) {
+ return (hook == null) ? result : (T)hook.postRead(result);
+ }
+
+ private Object ponderGlobalPostHook(Object result) {
+ Hook globalHook = this.globalHook;
+ return (globalHook != null) ? globalHook.postRead(result) : result;
+ }
+
+ @Override
+ public Object readClassAndObject(Input input) {
+ return ponderGlobalPostHook(super.readClassAndObject(input));
+ }
+
+ @Override
+ public Registration readClass(Input input) {
+ Registration reg = super.readClass(input);
+ if (reg != null) {
+ ponderGlobalPreReadHook(reg.getType()); // Needed to intercept readClassAndObject.
+ }
+ return reg;
+ }
+
+ @Override
+ public <T> T readObjectOrNull(Input input, Class<T> type) {
+ Hook hook = ponderGlobalPreReadHook(type);
+ T result = super.readObjectOrNull(input, type);
+ return ponderGlobalPostReadHook(hook, result);
+ }
+
+ @Override
+ public <T> T readObjectOrNull(Input input, Class<T> type,
+ @SuppressWarnings("rawtypes") com.esotericsoftware.kryo.Serializer serializer) {
+ Hook hook = ponderGlobalPreReadHook(type);
+ T result = super.readObjectOrNull(input, type, serializer);
+ return ponderGlobalPostReadHook(hook, result);
+ }
+
+ @Override
+ public <T> T readObject(Input input, Class<T> type) {
+ Hook hook = ponderGlobalPreReadHook(type);
+ T result = super.readObject(input, type);
+ return ponderGlobalPostReadHook(hook, result);
+ }
+
+ @Override
+ public <T> T readObject(Input input, Class<T> type,
+ @SuppressWarnings("rawtypes") com.esotericsoftware.kryo.Serializer serializer) {
+ Hook hook = ponderGlobalPreReadHook(type);
+ T result = super.readObject(input, type, serializer);
+ return ponderGlobalPostReadHook(hook, result);
+ }
+ }
private static KryoFactory factory = new KryoFactory() {
public Kryo create() {
- Kryo kryo = new Kryo();
+ KryoWithHooks kryo = new KryoWithHooks();
kryo.register(java.sql.Date.class, new SqlDateSerializer());
kryo.register(java.sql.Timestamp.class, new TimestampSerializer());
kryo.register(Path.class, new PathSerializer());
kryo.register(Arrays.asList("").getClass(), new ArraysAsListSerializer());
+
((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy())
.setFallbackInstantiatorStrategy(
new StdInstantiatorStrategy());
@@ -100,7 +243,9 @@ public class SerializationUtilities {
kryo.register(SparkEdgeProperty.class);
kryo.register(SparkWork.class);
kryo.register(Pair.class);
- return kryo;
+
+ // This must be called after all the explicit register calls.
+ return kryo.processHooks(kryoTypeHooks, globalHook);
}
};
http://git-wip-us.apache.org/repos/asf/hive/blob/715f410c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java
index 6098ddd..7433263 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBridge.java
@@ -81,6 +81,8 @@ public class GenericUDFBridge extends GenericUDF implements Serializable {
*/
private transient Object[] realArguments;
+ private transient UdfWhitelistChecker udfChecker;
+
/**
* Create a new GenericUDFBridge object.
*
@@ -127,17 +129,29 @@ public class GenericUDFBridge extends GenericUDF implements Serializable {
public Class<? extends UDF> getUdfClass() {
try {
- return (Class<? extends UDF>) Class.forName(udfClassName, true, Utilities.getSessionSpecifiedClassLoader());
+ return getUdfClassInternal();
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
+ /** Gets the UDF class and checks it against the whitelist, if any. */
+ private Class<? extends UDF> getUdfClassInternal()
+ throws ClassNotFoundException {
+ @SuppressWarnings("unchecked")
+ Class<? extends UDF> clazz = (Class<? extends UDF>) Class.forName(
+ udfClassName, true, Utilities.getSessionSpecifiedClassLoader());
+ if (udfChecker != null && !udfChecker.isUdfAllowed(clazz)) {
+ throw new SecurityException("UDF " + clazz.getCanonicalName() + " is not allowed");
+ }
+ return clazz;
+ }
+
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
try {
- udf = (UDF) Class.forName(udfClassName, true, Utilities.getSessionSpecifiedClassLoader()).newInstance();
+ udf = (UDF)getUdfClassInternal().newInstance();
} catch (Exception e) {
throw new UDFArgumentException(
"Unable to instantiate UDF implementation class " + udfClassName + ": " + e);
@@ -228,4 +242,11 @@ public class GenericUDFBridge extends GenericUDF implements Serializable {
return udf.getRequiredFiles();
}
+ public void setUdfChecker(UdfWhitelistChecker udfChecker) {
+ this.udfChecker = udfChecker;
+ }
+
+ public interface UdfWhitelistChecker {
+ boolean isUdfAllowed(Class<?> clazz);
+ }
}
[3/3] hive git commit: HIVE-12985 : distribution configs are wrong
and out of date (Sergey Shelukhin, reviewed by Alan Gates)
Posted by se...@apache.org.
HIVE-12985 : distribution configs are wrong and out of date (Sergey Shelukhin, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1b630384
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1b630384
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1b630384
Branch: refs/heads/master
Commit: 1b630384905e3bfeeb06d0d8696b2b0f83740554
Parents: 715f410
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Feb 2 14:13:39 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Feb 2 14:14:06 2016 -0800
----------------------------------------------------------------------
packaging/pom.xml | 25 +++++++++++++++++++++++++
packaging/src/main/assembly/bin.xml | 8 ++++++++
packaging/src/main/assembly/src.xml | 10 +++++++++-
3 files changed, 42 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1b630384/packaging/pom.xml
----------------------------------------------------------------------
diff --git a/packaging/pom.xml b/packaging/pom.xml
index 4ebd386..e7feb62 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -182,6 +182,31 @@
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-hplsql</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-orc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-tez</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-llap-server</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/1b630384/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index baa122a..a5f3975 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -113,6 +113,9 @@
<includes>
<include>/*.*</include>
</includes>
+ <excludes>
+ <exclude>/*.jar</exclude>
+ </excludes>
<outputDirectory>examples/files</outputDirectory>
</fileSet>
@@ -131,10 +134,15 @@
<include>hive</include>
<include>llap/**/*</include>
<include>metatool</include>
+ <include>hplsql</include>
<include>schematool</include>
<include>beeline</include>
<include>hiveserver2</include>
<include>hive-config.sh</include>
+ <include>hive-config.cmd</include>
+ <include>hive.cmd</include>
+ <include>hplsql.cmd</include>
+ <include>beeline.cmd</include>
<include>ext/**/*</include>
</includes>
<outputDirectory>bin</outputDirectory>
http://git-wip-us.apache.org/repos/asf/hive/blob/1b630384/packaging/src/main/assembly/src.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml
index 4bbafc0..e8fc980 100644
--- a/packaging/src/main/assembly/src.xml
+++ b/packaging/src/main/assembly/src.xml
@@ -35,11 +35,12 @@
<directory>${project.parent.basedir}</directory>
<excludes>
+ <exclude>**/*.jar</exclude>
<exclude>**/target/**</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/.settings/**</exclude>
- <exclude>itests/thirdparty/**</exclude>
+ <exclude>**/thirdparty/**</exclude>
<exclude>testutils/ptest2/velocity.log*</exclude>
</excludes>
@@ -70,14 +71,20 @@
<include>find-bugs/**/*</include>
<include>hbase-handler/**/*</include>
<include>hcatalog/**/*</include>
+ <include>hplsql/**/*</include>
<include>hwi/**/*</include>
<include>itests/**/*</include>
<include>jdbc/**/*</include>
<include>contrib/**/*</include>
<include>metastore/**/*</include>
+ <include>llap-common/**/*</include>
+ <include>llap-client/**/*</include>
+ <include>llap-tez/**/*</include>
+ <include>llap-server/**/*</include>
<include>lib/**/*</include>
<include>findbugs/**/*</include>
<include>odbc/**/*</include>
+ <include>orc/**/*</include>
<include>packaging/pom.xml</include>
<include>packaging/src/**/*</include>
<include>pom.xml</include>
@@ -86,6 +93,7 @@
<include>service/**/*</include>
<include>shims/**/*</include>
<include>spark-client/**/*</include>
+ <include>storage-api/**/*</include>
<include>testutils/**/*</include>
</includes>
<outputDirectory>/</outputDirectory>