You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2015/03/04 05:30:00 UTC
svn commit: r1663855 [2/2] - in /hive/branches/llap: ./
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/ com...
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java Wed Mar 4 04:29:59 2015
@@ -20,10 +20,12 @@ package org.apache.hadoop.hive.ql.exec.t
import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.Callable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -55,13 +57,15 @@ public class MergeFileRecordProcessor ex
protected Operator<? extends OperatorDesc> mergeOp;
private ExecMapperContext execContext = null;
protected static final String MAP_PLAN_KEY = "__MAP_PLAN__";
+ private String cacheKey;
private MergeFileWork mfWork;
MRInputLegacy mrInput = null;
private boolean abort = false;
private final Object[] row = new Object[2];
+ ObjectCache cache;
@Override
- void init(JobConf jconf, ProcessorContext processorContext,
+ void init(final JobConf jconf, ProcessorContext processorContext,
MRTaskReporter mrReporter, Map<String, LogicalInput> inputs,
Map<String, LogicalOutput> outputs) throws Exception {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS);
@@ -85,17 +89,20 @@ public class MergeFileRecordProcessor ex
}
org.apache.hadoop.hive.ql.exec.ObjectCache cache = ObjectCacheFactory
- .getCache(jconf);
+ .getCache(jconf);
+
try {
execContext.setJc(jconf);
- // create map and fetch operators
- MapWork mapWork = (MapWork) cache.retrieve(MAP_PLAN_KEY);
- if (mapWork == null) {
- mapWork = Utilities.getMapWork(jconf);
- cache.cache(MAP_PLAN_KEY, mapWork);
- } else {
- Utilities.setMapWork(jconf, mapWork);
- }
+
+ String queryId = HiveConf.getVar(jconf, HiveConf.ConfVars.HIVEQUERYID);
+ cacheKey = queryId + MAP_PLAN_KEY;
+
+ MapWork mapWork = (MapWork) cache.retrieve(cacheKey, new Callable<Object>() {
+ public Object call() {
+ return Utilities.getMapWork(jconf);
+ }
+ });
+ Utilities.setMapWork(jconf, mapWork);
if (mapWork instanceof MergeFileWork) {
mfWork = (MergeFileWork) mapWork;
@@ -144,6 +151,11 @@ public class MergeFileRecordProcessor ex
@Override
void close() {
+
+ if (cache != null && cacheKey != null) {
+ cache.release(cacheKey);
+ }
+
// check if there are IOExceptions
if (!abort) {
abort = execContext.getIoCxt().getIOExceptions();
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java Wed Mar 4 04:29:59 2015
@@ -18,9 +18,12 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import java.util.concurrent.Callable;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tez.runtime.api.ObjectRegistry;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import com.google.common.base.Preconditions;
@@ -29,17 +32,17 @@ import com.google.common.base.Preconditi
*
*/
public class ObjectCache implements org.apache.hadoop.hive.ql.exec.ObjectCache {
-
+
private static final Log LOG = LogFactory.getLog(ObjectCache.class.getName());
-
+
// ObjectRegistry is available via the Input/Output/ProcessorContext.
// This is setup as part of the Tez Processor construction, so that it is available whenever an
// instance of the ObjectCache is created. The assumption is that Tez will initialize the Processor
// before anything else.
private volatile static ObjectRegistry staticRegistry;
-
+
private final ObjectRegistry registry;
-
+
public ObjectCache() {
Preconditions.checkNotNull(staticRegistry,
"Object registry not setup yet. This should have been setup by the TezProcessor");
@@ -49,18 +52,27 @@ public class ObjectCache implements org.
public static void setupObjectRegistry(ObjectRegistry objectRegistry) {
staticRegistry = objectRegistry;
}
-
+
@Override
- public void cache(String key, Object value) {
- LOG.info("Adding " + key + " to cache with value " + value);
- registry.cacheForVertex(key, value);
+ public void release(String key) {
+ // nothing to do
+ LOG.info("Releasing key: " + key);
}
@Override
- public Object retrieve(String key) {
- Object o = registry.get(key);
- if (o != null) {
- LOG.info("Found " + key + " in cache with value: " + o);
+ public Object retrieve(String key, Callable<?> fn) throws HiveException {
+ Object o;
+ try {
+ o = registry.get(key);
+ if (o == null) {
+ o = fn.call();
+ LOG.info("Caching key: " + key);
+ registry.cacheForVertex(key, o);
+ } else {
+ LOG.info("Found " + key + " in cache with value: " + o);
+ }
+ } catch (Exception e) {
+ throw new HiveException(e);
}
return o;
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java Wed Mar 4 04:29:59 2015
@@ -22,9 +22,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.Callable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.ObjectCache;
@@ -55,6 +57,10 @@ public class ReduceRecordProcessor exte
private static final String REDUCE_PLAN_KEY = "__REDUCE_PLAN__";
+ private ObjectCache cache;
+
+ private String cacheKey;
+
public static final Log l4j = LogFactory.getLog(ReduceRecordProcessor.class);
private ReduceWork redWork;
@@ -68,20 +74,22 @@ public class ReduceRecordProcessor exte
private boolean abort;
@Override
- void init(JobConf jconf, ProcessorContext processorContext, MRTaskReporter mrReporter,
- Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws Exception {
+ void init(final JobConf jconf, ProcessorContext processorContext,
+ MRTaskReporter mrReporter, Map<String, LogicalInput> inputs,
+ Map<String, LogicalOutput> outputs) throws Exception {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS);
super.init(jconf, processorContext, mrReporter, inputs, outputs);
ObjectCache cache = ObjectCacheFactory.getCache(jconf);
- redWork = (ReduceWork) cache.retrieve(REDUCE_PLAN_KEY);
- if (redWork == null) {
- redWork = Utilities.getReduceWork(jconf);
- cache.cache(REDUCE_PLAN_KEY, redWork);
- } else {
- Utilities.setReduceWork(jconf, redWork);
- }
+ String queryId = HiveConf.getVar(jconf, HiveConf.ConfVars.HIVEQUERYID);
+ cacheKey = queryId + REDUCE_PLAN_KEY;
+ redWork = (ReduceWork) cache.retrieve(cacheKey, new Callable<Object>() {
+ public Object call() {
+ return Utilities.getReduceWork(jconf);
+ }
+ });
+ Utilities.setReduceWork(jconf, redWork);
reducer = redWork.getReducer();
reducer.getParentOperators().clear();
@@ -188,6 +196,10 @@ public class ReduceRecordProcessor exte
@Override
void close(){
+ if (cache != null) {
+ cache.release(cacheKey);
+ }
+
try {
for (ReduceRecordSource rs: sources) {
abort = abort && rs.close();
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezCacheAccess.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezCacheAccess.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezCacheAccess.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezCacheAccess.java Wed Mar 4 04:29:59 2015
@@ -22,59 +22,61 @@ import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ObjectCache;
import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
/**
* Access to the Object cache from Tez, along with utility methods for accessing specific Keys.
*/
public class TezCacheAccess {
- private TezCacheAccess(ObjectCache cache) {
+ private TezCacheAccess(ObjectCache cache, String qId) {
+ this.qId = qId;
this.cache = cache;
}
private ObjectCache cache;
+ private String qId;
public static TezCacheAccess createInstance(Configuration conf) {
ObjectCache cache = ObjectCacheFactory.getCache(conf);
- return new TezCacheAccess(cache);
+ String qId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID);
+ return new TezCacheAccess(cache, qId);
}
private static final String CACHED_INPUT_KEY = "CACHED_INPUTS";
-
+
private final ReentrantLock cachedInputLock = new ReentrantLock();
- public boolean isInputCached(String inputName) {
+ private Set<String> get() throws HiveException {
+ return (Set<String>) cache.retrieve(CACHED_INPUT_KEY,
+ new Callable<Object>() {
+ public Object call() {
+ return Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ }
+ });
+ }
+
+ public boolean isInputCached(String inputName) throws HiveException {
this.cachedInputLock.lock();
try {
- @SuppressWarnings("unchecked")
- Set<String> cachedInputs = (Set<String>) cache.retrieve(CACHED_INPUT_KEY);
- if (cachedInputs == null) {
- return false;
- } else {
- return cachedInputs.contains(inputName);
- }
+ return get().contains(qId+inputName);
} finally {
this.cachedInputLock.unlock();
}
}
- public void registerCachedInput(String inputName) {
+ public void registerCachedInput(String inputName) throws HiveException {
this.cachedInputLock.lock();
try {
- @SuppressWarnings("unchecked")
- Set<String> cachedInputs = (Set<String>) cache.retrieve(CACHED_INPUT_KEY);
- if (cachedInputs == null) {
- cachedInputs = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
- cache.cache(CACHED_INPUT_KEY, cachedInputs);
- }
- cachedInputs.add(inputName);
+ get().add(qId+inputName);
} finally {
this.cachedInputLock.unlock();
}
}
-
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java Wed Mar 4 04:29:59 2015
@@ -886,10 +886,6 @@ public class VectorGroupByOperator exten
for (int i = 0; i < aggregators.length; ++i) {
forwardCache[fi++] = aggregators[i].evaluateOutput(agg.getAggregationBuffer(i));
}
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("forwarding keys: %s: %s",
- kw, Arrays.toString(forwardCache)));
- }
forward(forwardCache, outputObjInspector);
} else {
// Output keys and aggregates into the output batch.
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java Wed Mar 4 04:29:59 2015
@@ -52,7 +52,7 @@ public class ATSHook implements ExecuteW
private enum EntityTypes { HIVE_QUERY_ID };
private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED };
private enum OtherInfoTypes { QUERY, STATUS, TEZ, MAPRED };
- private enum PrimaryFilterTypes { user, operationid };
+ private enum PrimaryFilterTypes { user, requestuser, operationid };
private static final int WAIT_TIME = 3;
public ATSHook() {
@@ -101,9 +101,12 @@ public class ATSHook implements ExecuteW
String opId = hookContext.getOperationId();
long queryStartTime = plan.getQueryStartTime();
String user = hookContext.getUgi().getUserName();
+ String requestuser = hookContext.getUserName();
+ if (hookContext.getUserName() == null ){
+ requestuser = hookContext.getUgi().getUserName() ;
+ }
int numMrJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size();
-
if (numMrJobs + numTezJobs <= 0) {
return; // ignore client only queries
}
@@ -117,13 +120,13 @@ public class ATSHook implements ExecuteW
JSONObject explainPlan = explain.getJSONPlan(null, null, rootTasks,
plan.getFetchTask(), true, false, false);
fireAndForget(conf, createPreHookEvent(queryId, query,
- explainPlan, queryStartTime, user, numMrJobs, numTezJobs, opId));
+ explainPlan, queryStartTime, user, requestuser, numMrJobs, numTezJobs, opId));
break;
case POST_EXEC_HOOK:
- fireAndForget(conf, createPostHookEvent(queryId, currentTime, user, true, opId));
+ fireAndForget(conf, createPostHookEvent(queryId, currentTime, user, requestuser, true, opId));
break;
case ON_FAILURE_HOOK:
- fireAndForget(conf, createPostHookEvent(queryId, currentTime, user, false, opId));
+ fireAndForget(conf, createPostHookEvent(queryId, currentTime, user, requestuser , false, opId));
break;
default:
//ignore
@@ -137,7 +140,7 @@ public class ATSHook implements ExecuteW
}
TimelineEntity createPreHookEvent(String queryId, String query, JSONObject explainPlan,
- long startTime, String user, int numMrJobs, int numTezJobs, String opId) throws Exception {
+ long startTime, String user, String requestuser, int numMrJobs, int numTezJobs, String opId) throws Exception {
JSONObject queryObj = new JSONObject();
queryObj.put("queryText", query);
@@ -153,6 +156,8 @@ public class ATSHook implements ExecuteW
atsEntity.setEntityId(queryId);
atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name());
atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user);
+ atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser);
+
if (opId != null) {
atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId);
}
@@ -168,7 +173,7 @@ public class ATSHook implements ExecuteW
return atsEntity;
}
- TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, boolean success,
+ TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success,
String opId) {
LOG.info("Received post-hook notification for :" + queryId);
@@ -176,6 +181,7 @@ public class ATSHook implements ExecuteW
atsEntity.setEntityId(queryId);
atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name());
atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user);
+ atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser);
if (opId != null) {
atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId);
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java Wed Mar 4 04:29:59 2015
@@ -28,7 +28,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -220,7 +220,7 @@ public class CompactIndexHandler extends
// We can only perform a binary search with HiveInputFormat and CombineHiveInputFormat
// and BucketizedHiveInputFormat
try {
- if (!HiveInputFormat.class.isAssignableFrom(Class.forName(inputFormat))) {
+ if (!HiveInputFormat.class.isAssignableFrom(JavaUtils.loadClass(inputFormat))) {
work = null;
break;
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java Wed Mar 4 04:29:59 2015
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat.CombineHiveInputSplit;
import org.apache.hadoop.io.Writable;
@@ -50,7 +51,7 @@ public class CombineHiveRecordReader<K e
String inputFormatClassName = hsplit.inputFormatClassName();
Class inputFormatClass = null;
try {
- inputFormatClass = Class.forName(inputFormatClassName);
+ inputFormatClass = JavaUtils.loadClass(inputFormatClassName);
} catch (ClassNotFoundException e) {
throw new IOException("CombineHiveRecordReader: class not found "
+ inputFormatClassName);
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Wed Mar 4 04:29:59 2015
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -232,8 +233,8 @@ public final class HiveFileFormatUtils {
jc_output = new JobConf(jc);
String codecStr = conf.getCompressCodec();
if (codecStr != null && !codecStr.trim().equals("")) {
- Class<? extends CompressionCodec> codec = (Class<? extends CompressionCodec>) Class
- .forName(codecStr);
+ Class<? extends CompressionCodec> codec =
+ (Class<? extends CompressionCodec>) JavaUtils.loadClass(codecStr);
FileOutputFormat.setOutputCompressorClass(jc_output, codec);
}
String type = conf.getCompressType();
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java Wed Mar 4 04:29:59 2015
@@ -21,8 +21,6 @@ package org.apache.hadoop.hive.ql.io.mer
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
-import org.apache.hadoop.hive.ql.exec.ObjectCache;
-import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -54,17 +52,7 @@ public class MergeFileMapper extends Map
@Override
public void configure(JobConf job) {
jc = job;
- ObjectCache cache = ObjectCacheFactory.getCache(job);
- MapWork mapWork = (MapWork) cache.retrieve(PLAN_KEY);
-
- // if map work is found in object cache then return it else retrieve the
- // plan from filesystem and cache it
- if (mapWork == null) {
- mapWork = Utilities.getMapWork(job);
- cache.cache(PLAN_KEY, mapWork);
- } else {
- Utilities.setMapWork(job, mapWork);
- }
+ MapWork mapWork = Utilities.getMapWork(job);
try {
if (mapWork instanceof MergeFileWork) {
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Wed Mar 4 04:29:59 2015
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.IOConstants;
@@ -324,7 +325,7 @@ public class WriterImpl implements Write
try {
Class<? extends CompressionCodec> lzo =
(Class<? extends CompressionCodec>)
- Class.forName("org.apache.hadoop.hive.ql.io.orc.LzoCodec");
+ JavaUtils.loadClass("org.apache.hadoop.hive.ql.io.orc.LzoCodec");
return lzo.newInstance();
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("LZO is not available.", e);
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java Wed Mar 4 04:29:59 2015
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
@@ -144,8 +145,7 @@ public class PartialScanTask extends Tas
LOG.info("Using " + inpFormat);
try {
- job.setInputFormat((Class<? extends InputFormat>) (Class
- .forName(inpFormat)));
+ job.setInputFormat((Class<? extends InputFormat>) JavaUtils.loadClass(inpFormat));
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.getMessage(), e);
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java Wed Mar 4 04:29:59 2015
@@ -24,6 +24,7 @@ import java.io.Serializable;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
@@ -120,8 +121,7 @@ public class ColumnTruncateTask extends
LOG.info("Using " + inpFormat);
try {
- job.setInputFormat((Class<? extends InputFormat>) (Class
- .forName(inpFormat)));
+ job.setInputFormat((Class<? extends InputFormat>) JavaUtils.loadClass(inpFormat));
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.getMessage(), e);
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Wed Mar 4 04:29:59 2015
@@ -61,6 +61,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.PartitionDropOptions;
import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
@@ -1920,8 +1921,14 @@ private void constructOneLBLocationMap(F
public boolean dropPartition(String db_name, String tbl_name,
List<String> part_vals, boolean deleteData) throws HiveException {
+ return dropPartition(db_name, tbl_name, part_vals,
+ PartitionDropOptions.instance().deleteData(deleteData));
+ }
+
+ public boolean dropPartition(String dbName, String tableName, List<String> partVals, PartitionDropOptions options)
+ throws HiveException {
try {
- return getMSC().dropPartition(db_name, tbl_name, part_vals, deleteData);
+ return getMSC().dropPartition(dbName, tableName, partVals, options);
} catch (NoSuchObjectException e) {
throw new HiveException("Partition or table doesn't exist.", e);
} catch (Exception e) {
@@ -1939,7 +1946,21 @@ private void constructOneLBLocationMap(F
public List<Partition> dropPartitions(String dbName, String tblName,
List<DropTableDesc.PartSpec> partSpecs, boolean deleteData, boolean ignoreProtection,
boolean ifExists) throws HiveException {
- //TODO: add support for ifPurge
+ return dropPartitions(dbName, tblName, partSpecs,
+ PartitionDropOptions.instance()
+ .deleteData(deleteData)
+ .ignoreProtection(ignoreProtection)
+ .ifExists(ifExists));
+ }
+
+ public List<Partition> dropPartitions(String tblName, List<DropTableDesc.PartSpec> partSpecs,
+ PartitionDropOptions dropOptions) throws HiveException {
+ String[] names = Utilities.getDbTableName(tblName);
+ return dropPartitions(names[0], names[1], partSpecs, dropOptions);
+ }
+
+ public List<Partition> dropPartitions(String dbName, String tblName,
+ List<DropTableDesc.PartSpec> partSpecs, PartitionDropOptions dropOptions) throws HiveException {
try {
Table tbl = getTable(dbName, tblName);
List<ObjectPair<Integer, byte[]>> partExprs =
@@ -1949,7 +1970,7 @@ private void constructOneLBLocationMap(F
Utilities.serializeExpressionToKryo(partSpec.getPartSpec())));
}
List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().dropPartitions(
- dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists);
+ dbName, tblName, partExprs, dropOptions);
return convertFromMetastore(tbl, tParts, null);
} catch (NoSuchObjectException e) {
throw new HiveException("Partition or table doesn't exist.", e);
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java Wed Mar 4 04:29:59 2015
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.Stack;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -440,10 +441,10 @@ abstract public class AbstractSMBJoinPro
Class<? extends BigTableSelectorForAutoSMJ> bigTableMatcherClass = null;
try {
+ String selector = HiveConf.getVar(pGraphContext.getConf(),
+ HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR);
bigTableMatcherClass =
- (Class<? extends BigTableSelectorForAutoSMJ>)
- (Class.forName(HiveConf.getVar(pGraphContext.getConf(),
- HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR)));
+ (Class<? extends BigTableSelectorForAutoSMJ>) JavaUtils.loadClass(selector);
} catch (ClassNotFoundException e) {
throw new SemanticException(e.getMessage());
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Wed Mar 4 04:29:59 2015
@@ -28,6 +28,7 @@ import java.util.Stack;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
@@ -181,10 +182,10 @@ public class ConvertJoinMapJoin implemen
}
Class<? extends BigTableSelectorForAutoSMJ> bigTableMatcherClass = null;
try {
+ String selector = HiveConf.getVar(context.parseContext.getConf(),
+ HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR);
bigTableMatcherClass =
- (Class<? extends BigTableSelectorForAutoSMJ>) (Class.forName(HiveConf.getVar(
- context.parseContext.getConf(),
- HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR)));
+ (Class<? extends BigTableSelectorForAutoSMJ>) JavaUtils.loadClass(selector);
} catch (ClassNotFoundException e) {
throw new SemanticException(e.getMessage());
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java Wed Mar 4 04:29:59 2015
@@ -54,10 +54,12 @@ import org.apache.hadoop.hive.ql.parse.S
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
@@ -68,7 +70,10 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -152,8 +157,13 @@ public class RexNodeConverter {
// TODO: 1) Expand to other functions as needed 2) What about types other than primitive.
TypeInfo tgtDT = null;
GenericUDF tgtUdf = func.getGenericUDF();
- boolean isNumeric = tgtUdf instanceof GenericUDFBaseNumeric,
- isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare;
+
+ boolean isNumeric = (tgtUdf instanceof GenericUDFBaseBinary
+ && func.getTypeInfo().getCategory() == Category.PRIMITIVE
+ && (PrimitiveGrouping.NUMERIC_GROUP == PrimitiveObjectInspectorUtils.getPrimitiveGrouping(
+ ((PrimitiveTypeInfo) func.getTypeInfo()).getPrimitiveCategory())));
+ boolean isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare;
+
if (isNumeric) {
tgtDT = func.getTypeInfo();
@@ -175,8 +185,7 @@ public class RexNodeConverter {
} else if (isNumeric) {
// For numeric, we'll do minimum necessary cast - if we cast to the type
// of expression, bad things will happen.
- GenericUDFBaseNumeric numericUdf = (GenericUDFBaseNumeric)tgtUdf;
- PrimitiveTypeInfo minArgType = numericUdf.deriveMinArgumentCast(childExpr, tgtDT);
+ PrimitiveTypeInfo minArgType = ExprNodeDescUtils.deriveMinArgumentCast(childExpr, tgtDT);
tmpExprNode = ParseUtils.createConversionCast(childExpr, minArgType);
} else {
throw new AssertionError("Unexpected " + tgtDT + " - not a numeric op or compare");
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Wed Mar 4 04:29:59 2015
@@ -45,6 +45,7 @@ import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -1036,7 +1037,7 @@ public class DDLSemanticAnalyzer extends
typeName = indexType.getHandlerClsName();
} else {
try {
- Class.forName(typeName);
+ JavaUtils.loadClass(typeName);
} catch (Exception e) {
throw new SemanticException("class name provided for index handler not found.", e);
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Wed Mar 4 04:29:59 2015
@@ -32,6 +32,10 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.util.ReflectionUtils;
@@ -427,4 +431,17 @@ public class ExprNodeDescUtils {
}
return true;
}
+
+ public static PrimitiveTypeInfo deriveMinArgumentCast(
+ ExprNodeDesc childExpr, TypeInfo targetType) {
+ assert targetType instanceof PrimitiveTypeInfo : "Not a primitive type" + targetType;
+ PrimitiveTypeInfo pti = (PrimitiveTypeInfo)targetType;
+ // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed.
+ // We also don't do anything for non-primitive children (maybe we should assert).
+ if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL)
+ || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti;
+ PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo();
+ // If the child is also decimal, no cast is needed (we hope - can target type be narrower?).
+ return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi);
+ }
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java Wed Mar 4 04:29:59 2015
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -288,7 +288,7 @@ public class PTFDeserializer {
try {
@SuppressWarnings("unchecked")
Class<? extends TableFunctionResolver> rCls = (Class<? extends TableFunctionResolver>)
- Class.forName(className);
+ JavaUtils.loadClass(className);
return ReflectionUtils.newInstance(rCls, null);
} catch (Exception e) {
throw new HiveException(e);
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Wed Mar 4 04:29:59 2015
@@ -31,6 +31,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -144,7 +145,7 @@ public final class PlanUtils {
serdeConstants.SERIALIZATION_LIB, localDirectoryDesc.getSerName());
}
if (localDirectoryDesc.getOutputFormat() != null){
- ret.setOutputFileFormatClass(Class.forName(localDirectoryDesc.getOutputFormat()));
+ ret.setOutputFileFormatClass(JavaUtils.loadClass(localDirectoryDesc.getOutputFormat()));
}
if (localDirectoryDesc.getNullFormat() != null) {
properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT,
@@ -307,7 +308,7 @@ public final class PlanUtils {
try {
if (crtTblDesc.getSerName() != null) {
- Class c = Class.forName(crtTblDesc.getSerName());
+ Class c = JavaUtils.loadClass(crtTblDesc.getSerName());
serdeClass = c;
}
@@ -356,8 +357,8 @@ public final class PlanUtils {
// replace the default input & output file format with those found in
// crtTblDesc
- Class c1 = Class.forName(crtTblDesc.getInputFormat());
- Class c2 = Class.forName(crtTblDesc.getOutputFormat());
+ Class c1 = JavaUtils.loadClass(crtTblDesc.getInputFormat());
+ Class c2 = JavaUtils.loadClass(crtTblDesc.getOutputFormat());
Class<? extends InputFormat> in_class = c1;
Class<? extends HiveOutputFormat> out_class = c2;
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Wed Mar 4 04:29:59 2015
@@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -69,7 +70,7 @@ public class JDBCStatsAggregator impleme
this.sourceTask = sourceTask;
try {
- Class.forName(driver).newInstance();
+ JavaUtils.loadClass(driver).newInstance();
} catch (Exception e) {
LOG.error("Error during instantiating JDBC driver " + driver + ". ", e);
return false;
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Wed Mar 4 04:29:59 2015
@@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
@@ -69,7 +70,7 @@ public class JDBCStatsPublisher implemen
String driver = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER);
try {
- Class.forName(driver).newInstance();
+ JavaUtils.loadClass(driver).newInstance();
} catch (Exception e) {
LOG.error("Error during instantiating JDBC driver " + driver + ". ", e);
return false;
@@ -272,7 +273,7 @@ public class JDBCStatsPublisher implemen
this.hiveconf = hconf;
connectionString = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING);
String driver = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER);
- Class.forName(driver).newInstance();
+ JavaUtils.loadClass(driver).newInstance();
synchronized(DriverManager.class) {
DriverManager.setLoginTimeout(timeout);
conn = DriverManager.getConnection(connectionString);
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java Wed Mar 4 04:29:59 2015
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.BlockLocatio
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -647,7 +648,7 @@ public class CompactorMR {
private static <T> T instantiate(Class<T> classType, String classname) throws IOException {
T t = null;
try {
- Class c = Class.forName(classname);
+ Class c = JavaUtils.loadClass(classname);
Object o = c.newInstance();
if (classType.isAssignableFrom(o.getClass())) {
t = (T)o;
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java Wed Mar 4 04:29:59 2015
@@ -44,7 +44,7 @@ import org.apache.hadoop.io.BooleanWrita
* GenericUDF Base Class for operations.
*/
@Description(name = "op", value = "a op b - Returns the result of operation")
-public abstract class GenericUDFBaseCompare extends GenericUDF {
+public abstract class GenericUDFBaseCompare extends GenericUDFBaseBinary {
public enum CompareType {
// Now only string, text, int, long, byte and boolean comparisons are
// treated as special cases.
@@ -53,9 +53,6 @@ public abstract class GenericUDFBaseComp
COMPARE_BOOL, SAME_TYPE, NEED_CONVERT
}
- protected String opName;
- protected String opDisplayName;
-
protected transient ObjectInspector[] argumentOIs;
protected transient ReturnObjectInspectorResolver conversionHelper = null;
@@ -182,12 +179,4 @@ public abstract class GenericUDFBaseComp
o0, argumentOIs[0], o1, argumentOIs[1]);
}
}
-
- @Override
- public String getDisplayString(String[] children) {
- assert (children.length == 2);
- return "(" + children[0] + " " + opDisplayName + " " + children[1] + ")";
-
- }
-
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java Wed Mar 4 04:29:59 2015
@@ -59,9 +59,7 @@ import org.apache.hive.common.HiveCompat
* GenericUDF Base Class for operations.
*/
@Description(name = "op", value = "a op b - Returns the result of operation")
-public abstract class GenericUDFBaseNumeric extends GenericUDF {
- protected String opName;
- protected String opDisplayName;
+public abstract class GenericUDFBaseNumeric extends GenericUDFBaseBinary {
protected transient PrimitiveObjectInspector leftOI;
protected transient PrimitiveObjectInspector rightOI;
@@ -82,7 +80,6 @@ public abstract class GenericUDFBaseNume
protected boolean ansiSqlArithmetic = false;
public GenericUDFBaseNumeric() {
- opName = getClass().getSimpleName();
}
@Override
@@ -291,12 +288,6 @@ public abstract class GenericUDFBaseNume
protected abstract DecimalTypeInfo deriveResultDecimalTypeInfo(int prec1, int scale1, int prec2, int scale2);
- @Override
- public String getDisplayString(String[] children) {
- assert (children.length == 2) : opDisplayName + " with " + children.length + " children";
- return "(" + children[0] + " " + opDisplayName + " " + children[1] + ")";
- }
-
public void copyToNewInstance(Object newInstance) throws UDFArgumentException {
super.copyToNewInstance(newInstance);
GenericUDFBaseNumeric other = (GenericUDFBaseNumeric) newInstance;
@@ -319,17 +310,4 @@ public abstract class GenericUDFBaseNume
public void setAnsiSqlArithmetic(boolean ansiSqlArithmetic) {
this.ansiSqlArithmetic = ansiSqlArithmetic;
}
-
- public PrimitiveTypeInfo deriveMinArgumentCast(
- ExprNodeDesc childExpr, TypeInfo targetType) {
- assert targetType instanceof PrimitiveTypeInfo : "Not a primitive type" + targetType;
- PrimitiveTypeInfo pti = (PrimitiveTypeInfo)targetType;
- // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed.
- // We also don't do anything for non-primitive children (maybe we should assert).
- if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL)
- || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti;
- PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo();
- // If the child is also decimal, no cast is needed (we hope - can target type be narrower?).
- return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi);
- }
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMinus.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMinus.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMinus.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMinus.java Wed Mar 4 04:29:59 2015
@@ -18,19 +18,10 @@
package org.apache.hadoop.hive.ql.udf.generic;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
+
@Description(name = "-", value = "a _FUNC_ b - Returns the difference a-b")
@VectorizedExpressions({LongColSubtractLongColumn.class, LongColSubtractDoubleColumn.class,
@@ -41,7 +32,7 @@ import org.apache.hadoop.io.LongWritable
DoubleScalarSubtractLongColumn.class, DoubleScalarSubtractDoubleColumn.class,
DecimalColSubtractDecimalColumn.class, DecimalColSubtractDecimalScalar.class,
DecimalScalarSubtractDecimalColumn.class})
-public class GenericUDFOPMinus extends GenericUDFBaseNumeric {
+public class GenericUDFOPMinus extends GenericUDFBaseArithmetic {
public GenericUDFOPMinus() {
super();
@@ -49,57 +40,13 @@ public class GenericUDFOPMinus extends G
}
@Override
- protected ByteWritable evaluate(ByteWritable left, ByteWritable right) {
- byteWritable.set((byte)(left.get() - right.get()));
- return byteWritable;
- }
-
- @Override
- protected ShortWritable evaluate(ShortWritable left, ShortWritable right) {
- shortWritable.set((short)(left.get() - right.get()));
- return shortWritable;
- }
-
- @Override
- protected IntWritable evaluate(IntWritable left, IntWritable right) {
- intWritable.set(left.get() - right.get());
- return intWritable;
- }
-
- @Override
- protected LongWritable evaluate(LongWritable left, LongWritable right) {
- longWritable.set(left.get() - right.get());
- return longWritable;
+ protected GenericUDFBaseNumeric instantiateNumericUDF() {
+ return new GenericUDFOPNumericMinus();
}
@Override
- protected FloatWritable evaluate(FloatWritable left, FloatWritable right) {
- floatWritable.set(left.get() - right.get());
- return floatWritable;
+ protected GenericUDF instantiateDTIUDF() {
+ // TODO: implement date-time/interval version of UDF
+ return new GenericUDFOPNumericMinus();
}
-
- @Override
- protected DoubleWritable evaluate(DoubleWritable left, DoubleWritable right) {
- doubleWritable.set(left.get() - right.get());
- return doubleWritable;
- }
-
- @Override
- protected HiveDecimalWritable evaluate(HiveDecimal left, HiveDecimal right) {
- HiveDecimal dec = left.subtract(right);
- if (dec == null) {
- return null;
- }
- decimalWritable.set(dec);
- return decimalWritable;
- }
-
- @Override
- protected DecimalTypeInfo deriveResultDecimalTypeInfo(int prec1, int scale1, int prec2, int scale2) {
- int intPart = Math.max(prec1 - scale1, prec2 - scale2);
- int scale = Math.max(scale1, scale2);
- int prec = Math.min(intPart + scale + 1, HiveDecimal.MAX_PRECISION);
- return TypeInfoFactory.getDecimalTypeInfo(prec, scale);
- }
-
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPPlus.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPPlus.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPPlus.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPPlus.java Wed Mar 4 04:29:59 2015
@@ -18,19 +18,9 @@
package org.apache.hadoop.hive.ql.udf.generic;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
/**
* The reason that we list evaluate methods with all numeric types is for both
@@ -48,7 +38,7 @@ import org.apache.hadoop.io.LongWritable
LongScalarAddLongColumn.class, LongScalarAddDoubleColumn.class, DoubleScalarAddLongColumn.class,
DoubleScalarAddDoubleColumn.class, DecimalScalarAddDecimalColumn.class, DecimalColAddDecimalColumn.class,
DecimalColAddDecimalScalar.class})
-public class GenericUDFOPPlus extends GenericUDFBaseNumeric {
+public class GenericUDFOPPlus extends GenericUDFBaseArithmetic {
public GenericUDFOPPlus() {
super();
@@ -56,59 +46,13 @@ public class GenericUDFOPPlus extends Ge
}
@Override
- protected ByteWritable evaluate(ByteWritable left, ByteWritable right) {
- byteWritable.set((byte)(left.get() + right.get()));
- return byteWritable;
+ protected GenericUDFBaseNumeric instantiateNumericUDF() {
+ return new GenericUDFOPNumericPlus();
}
@Override
- protected ShortWritable evaluate(ShortWritable left, ShortWritable right) {
- shortWritable.set((short)(left.get() + right.get()));
- return shortWritable;
+ protected GenericUDF instantiateDTIUDF() {
+ // TODO: implement date-time/interval version of UDF
+ return new GenericUDFOPNumericPlus();
}
-
- @Override
- protected IntWritable evaluate(IntWritable left, IntWritable right) {
- intWritable.set(left.get() + right.get());
- return intWritable;
- }
-
- @Override
- protected LongWritable evaluate(LongWritable left, LongWritable right) {
- longWritable.set(left.get() + right.get());
- return longWritable;
- }
-
- @Override
- protected FloatWritable evaluate(FloatWritable left, FloatWritable right) {
- floatWritable.set(left.get() + right.get());
- return floatWritable;
- }
-
- @Override
- protected DoubleWritable evaluate(DoubleWritable left, DoubleWritable right) {
- doubleWritable.set(left.get() + right.get());
- return doubleWritable;
- }
-
- @Override
- protected HiveDecimalWritable evaluate(HiveDecimal left, HiveDecimal right) {
- HiveDecimal dec = left.add(right);
-
- if (dec == null) {
- return null;
- }
-
- decimalWritable.set(dec);
- return decimalWritable;
- }
-
- @Override
- protected DecimalTypeInfo deriveResultDecimalTypeInfo(int prec1, int scale1, int prec2, int scale2) {
- int intPart = Math.max(prec1 - scale1, prec2 - scale2);
- int scale = Math.max(scale1, scale2);
- int prec = Math.min(intPart + scale + 1, HiveDecimal.MAX_PRECISION);
- return TypeInfoFactory.getDecimalTypeInfo(prec, scale);
- }
-
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java Wed Mar 4 04:29:59 2015
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf.ge
import java.lang.reflect.Method;
import java.util.Arrays;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
@@ -102,7 +103,7 @@ public class GenericUDFReflect extends A
className = ObjectInspectorUtils.copyToStandardObject(newClassName, inputClassNameOI);
String classNameString = classNameOI.getPrimitiveJavaObject(className);
try {
- c = Class.forName(classNameString);
+ c = JavaUtils.loadClass(classNameString);
} catch (ClassNotFoundException ex) {
throw new HiveException("UDFReflect evaluate ", ex);
}
Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java Wed Mar 4 04:29:59 2015
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
+import com.google.common.collect.ImmutableMap;
import junit.framework.TestCase;
import org.apache.hadoop.fs.FileStatus;
@@ -36,6 +37,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.PartitionDropOptions;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -426,6 +428,158 @@ public class TestHive extends TestCase {
}
}
+ private FileStatus[] getTrashContents() throws Exception {
+ FileSystem fs = FileSystem.get(hiveConf);
+ Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(hiveConf, fs);
+ return fs.globStatus(trashDir.suffix("/*"));
+ }
+
+ private Table createPartitionedTable(String dbName, String tableName) throws Exception {
+ try {
+
+ hm.dropTable(dbName, tableName);
+ hm.createTable(tableName,
+ Arrays.asList("key", "value"), // Data columns.
+ Arrays.asList("ds", "hr"), // Partition columns.
+ TextInputFormat.class,
+ HiveIgnoreKeyTextOutputFormat.class);
+ return hm.getTable(dbName, tableName);
+ }
+ catch (Exception exception) {
+ fail("Unable to drop and create table " + dbName + "." + tableName
+ + " because " + StringUtils.stringifyException(exception));
+ throw exception;
+ }
+ }
+
+ private void cleanUpTableQuietly(String dbName, String tableName) {
+ try {
+ hm.dropTable(dbName, tableName, true, true, true);
+ }
+ catch(Exception exception) {
+ fail("Unexpected exception: " + StringUtils.stringifyException(exception));
+ }
+ }
+
+ /**
+ * Test for PURGE support for dropping partitions.
+ * 1. Drop partitions without PURGE, and check that the data isn't moved to Trash.
+ * 2. Drop partitions with PURGE, and check that the data is moved to Trash.
+ * @throws Exception on failure.
+ */
+ public void testDropPartitionsWithPurge() throws Exception {
+ String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
+ String tableName = "table_for_testDropPartitionsWithPurge";
+
+ try {
+
+ Map<String, String> partitionSpec = new ImmutableMap.Builder<String, String>()
+ .put("ds", "20141216")
+ .put("hr", "12")
+ .build();
+
+ int trashSizeBeforeDrop = getTrashContents().length;
+
+ Table table = createPartitionedTable(dbName, tableName);
+ hm.createPartition(table, partitionSpec);
+
+ Partition partition = hm.getPartition(table, partitionSpec, false);
+ assertNotNull("Newly created partition shouldn't be null!", partition);
+
+ hm.dropPartition(dbName, tableName,
+ partition.getValues(),
+ PartitionDropOptions.instance()
+ .deleteData(true)
+ .purgeData(true)
+ );
+
+ int trashSizeAfterDropPurge = getTrashContents().length;
+
+ assertEquals("After dropPartitions(purge), trash should've remained unchanged!",
+ trashSizeBeforeDrop, trashSizeAfterDropPurge);
+
+ // Repeat, and drop partition without purge.
+ hm.createPartition(table, partitionSpec);
+
+ partition = hm.getPartition(table, partitionSpec, false);
+ assertNotNull("Newly created partition shouldn't be null!", partition);
+
+ hm.dropPartition(dbName, tableName,
+ partition.getValues(),
+ PartitionDropOptions.instance()
+ .deleteData(true)
+ .purgeData(false)
+ );
+
+ int trashSizeWithoutPurge = getTrashContents().length;
+
+ assertEquals("After dropPartitions(noPurge), data should've gone to trash!",
+ trashSizeBeforeDrop, trashSizeWithoutPurge);
+
+ }
+ catch (Exception e) {
+ fail("Unexpected exception: " + StringUtils.stringifyException(e));
+ }
+ finally {
+ cleanUpTableQuietly(dbName, tableName);
+ }
+ }
+
+ /**
+ * Test that tables set up with auto-purge skip trash-directory when tables/partitions are dropped.
+ * @throws Throwable
+ */
+ public void testAutoPurgeTablesAndPartitions() throws Throwable {
+
+ String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
+ String tableName = "table_for_testAutoPurgeTablesAndPartitions";
+ try {
+
+ Table table = createPartitionedTable(dbName, tableName);
+ table.getParameters().put("auto.purge", "true");
+ hm.alterTable(tableName, table);
+
+ Map<String, String> partitionSpec = new ImmutableMap.Builder<String, String>()
+ .put("ds", "20141216")
+ .put("hr", "12")
+ .build();
+
+ int trashSizeBeforeDrop = getTrashContents().length;
+
+ hm.createPartition(table, partitionSpec);
+
+ Partition partition = hm.getPartition(table, partitionSpec, false);
+ assertNotNull("Newly created partition shouldn't be null!", partition);
+
+ hm.dropPartition(dbName, tableName,
+ partition.getValues(),
+ PartitionDropOptions.instance()
+ .deleteData(true)
+ .purgeData(false)
+ );
+
+ int trashSizeAfterDrop = getTrashContents().length;
+
+ assertEquals("After dropPartition(noPurge), data should still have skipped trash.",
+ trashSizeBeforeDrop, trashSizeAfterDrop);
+
+ // Repeat the same check for dropTable.
+
+ trashSizeBeforeDrop = trashSizeAfterDrop;
+ hm.dropTable(dbName, tableName);
+ trashSizeAfterDrop = getTrashContents().length;
+
+ assertEquals("After dropTable(noPurge), data should still have skipped trash.",
+ trashSizeBeforeDrop, trashSizeAfterDrop);
+
+ }
+ catch(Exception e) {
+ fail("Unexpected failure: " + StringUtils.stringifyException(e));
+ }
+ finally {
+ cleanUpTableQuietly(dbName, tableName);
+ }
+ }
public void testPartition() throws Throwable {
try {
Modified: hive/branches/llap/ql/src/test/queries/clientpositive/vector_cast_constant.q
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/queries/clientpositive/vector_cast_constant.q?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/queries/clientpositive/vector_cast_constant.q (original)
+++ hive/branches/llap/ql/src/test/queries/clientpositive/vector_cast_constant.q Wed Mar 4 04:29:59 2015
@@ -1,5 +1,7 @@
SET hive.vectorized.execution.enabled=true;
+-- JAVA_VERSION_SPECIFIC_OUTPUT
+
DROP TABLE over1k;
DROP TABLE over1korc;
Modified: hive/branches/llap/ql/src/test/results/clientnegative/invalid_arithmetic_type.q.out
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/results/clientnegative/invalid_arithmetic_type.q.out?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/results/clientnegative/invalid_arithmetic_type.q.out (original)
+++ hive/branches/llap/ql/src/test/results/clientnegative/invalid_arithmetic_type.q.out Wed Mar 4 04:29:59 2015
@@ -1 +1 @@
-FAILED: SemanticException Line 0:-1 Wrong arguments ''2000-01-01 00:00:01'': No matching method for class org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMinus with (timestamp, timestamp)
+FAILED: SemanticException Line 0:-1 Wrong arguments ''2000-01-01 00:00:01'': No matching method for class org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNumericMinus with (timestamp, timestamp)
Modified: hive/branches/llap/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/results/clientpositive/show_functions.q.out?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/branches/llap/ql/src/test/results/clientpositive/show_functions.q.out Wed Mar 4 04:29:59 2015
@@ -188,6 +188,7 @@ to_unix_timestamp
to_utc_timestamp
translate
trim
+trunc
ucase
unbase64
unhex
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java?rev=1663855&r1=1663854&r2=1663855&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java Wed Mar 4 04:29:59 2015
@@ -19,8 +19,8 @@
package org.apache.hive.service.server;
-import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadFactory;
import org.apache.hadoop.hive.metastore.RawStore;
@@ -43,7 +43,7 @@ import org.apache.hadoop.hive.metastore.
*/
public class ThreadFactoryWithGarbageCleanup implements ThreadFactory {
- private static Map<Long, RawStore> threadRawStoreMap = new HashMap<Long, RawStore>();
+ private static Map<Long, RawStore> threadRawStoreMap = new ConcurrentHashMap<Long, RawStore>();
private final String namePrefix;