You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/04/15 20:41:11 UTC
svn commit: r1673926 - in /hive/trunk:
common/src/java/org/apache/hive/common/util/
metastore/src/java/org/apache/hadoop/hive/metastore/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ ql/src/java/org...
Author: sershe
Date: Wed Apr 15 18:41:11 2015
New Revision: 1673926
URL: http://svn.apache.org/r1673926
Log:
HIVE-10329 : Hadoop reflectionutils has issues (Sergey Shelukhin, reviewed by Vikram Dixit K)
Added:
hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
Added: hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java?rev=1673926&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java (added)
+++ hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java Wed Apr 15 18:41:11 2015
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+/**
+ * Same as Hadoop ReflectionUtils, but (1) does not leak classloaders (or shouldn't anyway, we
+ * rely on Guava cache, and could fix it otherwise); (2) does not have a hidden epic lock.
+ */
+public class ReflectionUtil {
+
+ // TODO: expireAfterAccess locks cache segments on put and expired get. It doesn't look too bad,
+ // but if we find some perf issues it might be a good idea to remove this - we are probably
+ // not caching that many constructors.
+ // Note that weakKeys causes "==" to be used for key compare; this will only work
+ // for classes in the same classloader. Should be ok in this case.
+ private static final Cache<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
+ CacheBuilder.newBuilder().expireAfterAccess(15, TimeUnit.MINUTES)
+ .concurrencyLevel(64)
+ .weakKeys().weakValues().build();
+ private static final Class<?>[] EMPTY_ARRAY = new Class[] {};
+ private static final Class<?> jobConfClass, jobConfigurableClass;
+ private static final Method configureMethod;
+
+ static {
+ Class<?> jobConfClassLocal, jobConfigurableClassLocal;
+ Method configureMethodLocal;
+ try {
+ jobConfClassLocal = Class.forName("org.apache.hadoop.mapred.JobConf");
+ jobConfigurableClassLocal = Class.forName("org.apache.hadoop.mapred.JobConfigurable");
+ configureMethodLocal = jobConfigurableClassLocal.getMethod("configure", jobConfClassLocal);
+ } catch (Throwable t) {
+ // Meh.
+ jobConfClassLocal = jobConfigurableClassLocal = null;
+ configureMethodLocal = null;
+ }
+ jobConfClass = jobConfClassLocal;
+ jobConfigurableClass = jobConfigurableClassLocal;
+ configureMethod = configureMethodLocal;
+ }
+
+ /**
+ * Create an object for the given class and initialize it from conf
+ * @param theClass class of which an object is created
+ * @param conf Configuration
+ * @return a new object
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(Class<T> theClass, Configuration conf) {
+ T result;
+ try {
+ Constructor<?> ctor = CONSTRUCTOR_CACHE.getIfPresent(theClass);
+ if (ctor == null) {
+ ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY);
+ ctor.setAccessible(true);
+ CONSTRUCTOR_CACHE.put(theClass, ctor);
+ }
+ result = (T)ctor.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ setConf(result, conf);
+ return result;
+ }
+
+
+ /**
+ * Check and set 'configuration' if necessary.
+ *
+ * @param theObject object for which to set configuration
+ * @param conf Configuration
+ */
+ public static void setConf(Object theObject, Configuration conf) {
+ if (conf != null) {
+ if (theObject instanceof Configurable) {
+ ((Configurable) theObject).setConf(conf);
+ }
+ setJobConf(theObject, conf);
+ }
+ }
+
+ private static void setJobConf(Object theObject, Configuration conf) {
+ if (configureMethod == null) return;
+ try {
+ if (jobConfClass.isAssignableFrom(conf.getClass()) &&
+ jobConfigurableClass.isAssignableFrom(theObject.getClass())) {
+ configureMethod.invoke(theObject, conf);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error in configuring object", e);
+ }
+ }
+}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Wed Apr 15 18:41:11 2015
@@ -80,7 +80,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
import javax.annotation.Nullable;
@@ -372,7 +372,7 @@ public class MetaStoreUtils {
return null;
}
try {
- Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(lib).
+ Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib).
asSubclass(Deserializer.class), conf);
if (skipConfError) {
SerDeUtils.initializeSerDeWithoutErrorCheck(deserializer, conf,
@@ -419,7 +419,7 @@ public class MetaStoreUtils {
org.apache.hadoop.hive.metastore.api.Table table) throws MetaException {
String lib = part.getSd().getSerdeInfo().getSerializationLib();
try {
- Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(lib).
+ Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib).
asSubclass(Deserializer.class), conf);
SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table),
MetaStoreUtils.getPartitionMetadata(part, table));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Wed Apr 15 18:41:11 2015
@@ -207,9 +207,9 @@ import org.apache.hadoop.hive.shims.Hado
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.tools.HadoopArchives;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.common.util.AnnotationUtils;
+import org.apache.hive.common.util.ReflectionUtil;
import org.stringtemplate.v4.ST;
/**
@@ -3786,7 +3786,7 @@ public class DDLTask extends Task<DDLWor
private void validateSerDe(String serdeName) throws HiveException {
try {
- Deserializer d = ReflectionUtils.newInstance(conf.getClassByName(serdeName).
+ Deserializer d = ReflectionUtil.newInstance(conf.getClassByName(serdeName).
asSubclass(Deserializer.class), conf);
if (d != null) {
LOG.debug("Found class for " + serdeName);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java Wed Apr 15 18:41:11 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.serde2.Del
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* serialize row by user specified serde and call toString() to make string type result
@@ -50,12 +50,13 @@ public class DefaultFetchFormatter<T> im
}
}
+//TODO#: THIS
private SerDe initializeSerde(Configuration conf, Properties props) throws Exception {
String serdeName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE);
Class<? extends SerDe> serdeClass = Class.forName(serdeName, true,
Utilities.getSessionSpecifiedClassLoader()).asSubclass(SerDe.class);
// cast only needed for Hadoop 0.17 compatibility
- SerDe serde = ReflectionUtils.newInstance(serdeClass, null);
+ SerDe serde = ReflectionUtil.newInstance(serdeClass, null);
Properties serdeProps = new Properties();
if (serde instanceof DelimitedJSONSerDe) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java Wed Apr 15 18:41:11 2015
@@ -39,7 +39,7 @@ import org.apache.hadoop.hive.serde2.Des
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* DemuxOperator is an operator used by MapReduce Jobs optimized by
@@ -134,12 +134,12 @@ public class DemuxOperator extends Opera
cntrs[newTag] = 0;
nextCntrs[newTag] = 0;
TableDesc keyTableDesc = conf.getKeysSerializeInfos().get(newTag);
- Deserializer inputKeyDeserializer = ReflectionUtils.newInstance(keyTableDesc
+ Deserializer inputKeyDeserializer = ReflectionUtil.newInstance(keyTableDesc
.getDeserializerClass(), null);
SerDeUtils.initializeSerDe(inputKeyDeserializer, null, keyTableDesc.getProperties(), null);
TableDesc valueTableDesc = conf.getValuesSerializeInfos().get(newTag);
- Deserializer inputValueDeserializer = ReflectionUtils.newInstance(valueTableDesc
+ Deserializer inputValueDeserializer = ReflectionUtil.newInstance(valueTableDesc
.getDeserializerClass(), null);
SerDeUtils.initializeSerDe(inputValueDeserializer, null, valueTableDesc.getProperties(),
null);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Wed Apr 15 18:41:11 2015
@@ -69,9 +69,9 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobConfigurable;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.common.util.AnnotationUtils;
+import org.apache.hive.common.util.ReflectionUtil;
import com.google.common.collect.Iterators;
@@ -204,13 +204,12 @@ public class FetchOperator implements Se
JobConf conf) throws IOException {
if (Configurable.class.isAssignableFrom(inputFormatClass) ||
JobConfigurable.class.isAssignableFrom(inputFormatClass)) {
- return ReflectionUtils
- .newInstance(inputFormatClass, conf);
+ return ReflectionUtil.newInstance(inputFormatClass, conf);
}
InputFormat format = inputFormats.get(inputFormatClass.getName());
if (format == null) {
try {
- format = ReflectionUtils.newInstance(inputFormatClass, conf);
+ format = ReflectionUtil.newInstance(inputFormatClass, conf);
inputFormats.put(inputFormatClass.getName(), format);
} catch (Exception e) {
throw new IOException("Cannot create an instance of InputFormat class "
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java Wed Apr 15 18:41:11 2015
@@ -44,7 +44,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
public class JoinUtil {
@@ -268,7 +268,7 @@ public class JoinUtil {
if (desc == null) {
return null;
}
- SerDe sd = (SerDe) ReflectionUtils.newInstance(desc.getDeserializerClass(),
+ SerDe sd = (SerDe) ReflectionUtil.newInstance(desc.getDeserializerClass(),
null);
try {
SerDeUtils.initializeSerDe(sd, null, desc.getProperties(), null);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Wed Apr 15 18:41:11 2015
@@ -61,7 +61,7 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
import static org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition;
import static org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer.KeyValueHelper;
@@ -229,8 +229,8 @@ public class MapJoinOperator extends Abs
try {
TableDesc keyTableDesc = conf.getKeyTblDesc();
- SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
- null);
+ SerDe keySerializer = (SerDe) ReflectionUtil.newInstance(
+ keyTableDesc.getDeserializerClass(), null);
SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null);
MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false);
for (int pos = 0; pos < order.length; pos++) {
@@ -243,8 +243,8 @@ public class MapJoinOperator extends Abs
} else {
valueTableDesc = conf.getValueFilteredTblDescs().get(pos);
}
- SerDe valueSerDe =
- (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
+ SerDe valueSerDe = (SerDe) ReflectionUtil.newInstance(
+ valueTableDesc.getDeserializerClass(), null);
SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
MapJoinObjectSerDeContext valueContext =
new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Wed Apr 15 18:41:11 2015
@@ -51,7 +51,7 @@ import org.apache.hadoop.io.WritableComp
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.util.PriorityQueue;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* Sorted Merge Map Join Operator.
@@ -525,7 +525,7 @@ public class SMBMapJoinOperator extends
BucketMapJoinContext bucketMatcherCxt = localWork.getBucketMapjoinContext();
Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
- BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls, null);
+ BucketMatcher bucketMatcher = ReflectionUtil.newInstance(bucketMatcherCls, null);
getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputPath.toString()));
if (isLogInfoEnabled) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java Wed Apr 15 18:41:11 2015
@@ -52,8 +52,8 @@ import org.apache.hadoop.mapred.InputFor
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hive.common.util.ReflectionUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
@@ -150,7 +150,7 @@ public class StatsNoJobTask extends Task
boolean statsAvailable = false;
for(FileStatus file: fileList) {
if (!file.isDir()) {
- InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
+ InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtil.newInstance(
partn.getInputFormatClass(), jc);
InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0,
new String[] { partn.getLocation() });
@@ -248,7 +248,7 @@ public class StatsNoJobTask extends Task
boolean statsAvailable = false;
for(FileStatus file: fileList) {
if (!file.isDir()) {
- InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
+ InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtil.newInstance(
table.getInputFormatClass(), jc);
InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { table
.getDataLocation().toString() });
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Apr 15 18:41:11 2015
@@ -181,8 +181,8 @@ import org.apache.hadoop.mapred.Reporter
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Shell;
+import org.apache.hive.common.util.ReflectionUtil;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
@@ -1375,7 +1375,7 @@ public final class Utilities {
if (isCompressed) {
Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
DefaultCodec.class);
- CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jc);
+ CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc);
return codec.createOutputStream(out);
} else {
return (out);
@@ -1424,7 +1424,7 @@ public final class Utilities {
if ((hiveOutputFormat instanceof HiveIgnoreKeyTextOutputFormat) && isCompressed) {
Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
DefaultCodec.class);
- CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jc);
+ CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc);
return codec.getDefaultExtension();
}
return "";
@@ -1476,7 +1476,7 @@ public final class Utilities {
if (isCompressed) {
compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc);
codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
- codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
+ codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc);
}
return SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec,
progressable);
@@ -1500,7 +1500,7 @@ public final class Utilities {
CompressionCodec codec = null;
if (isCompressed) {
Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
- codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
+ codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc);
}
return new RCFile.Writer(fs, jc, file, progressable, codec);
}
@@ -2948,7 +2948,7 @@ public final class Utilities {
if (reworkInputFormats.size() > 0) {
for (Class<? extends InputFormat> inputFormatCls : reworkInputFormats) {
- ReworkMapredInputFormat inst = (ReworkMapredInputFormat) ReflectionUtils
+ ReworkMapredInputFormat inst = (ReworkMapredInputFormat) ReflectionUtil
.newInstance(inputFormatCls, null);
inst.rework(conf, mapredWork);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java Wed Apr 15 18:41:11 2015
@@ -46,7 +46,7 @@ import org.apache.hadoop.mapred.InputFor
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* Simple persistent container for rows.
@@ -213,7 +213,7 @@ public class RowContainer<ROW extends Li
JobConf localJc = getLocalFSJobConfClone(jc);
if (inputSplits == null) {
if (this.inputFormat == null) {
- inputFormat = ReflectionUtils.newInstance(
+ inputFormat = ReflectionUtil.newInstance(
tblDesc.getInputFileFormatClass(), localJc);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java Wed Apr 15 18:41:11 2015
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.HiveInputFormat.HiveInputSplit;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* HiveInputSplit encapsulates an InputSplit with its corresponding
@@ -137,12 +137,11 @@ public class BucketizedHiveInputSplit ex
@Override
public void readFields(DataInput in) throws IOException {
String inputSplitClassName = in.readUTF();
-
int numSplits = in.readInt();
inputSplits = new InputSplit[numSplits];
for (int i = 0; i < numSplits; i++) {
try {
- inputSplits[i] = (InputSplit) ReflectionUtils.newInstance(conf
+ inputSplits[i] = (InputSplit) ReflectionUtil.newInstance(conf
.getClassByName(inputSplitClassName), conf);
} catch (Exception e) {
throw new IOException(
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Wed Apr 15 18:41:11 2015
@@ -59,7 +59,7 @@ import org.apache.hadoop.mapred.Sequence
import org.apache.hadoop.mapred.TaskAttemptContext;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* An util class for various Hive file format tasks.
@@ -274,7 +274,7 @@ public final class HiveFileFormatUtils {
private static HiveOutputFormat<?, ?> getHiveOutputFormat(
Configuration conf, Class<? extends OutputFormat> outputClass) throws HiveException {
- OutputFormat<?, ?> outputFormat = ReflectionUtils.newInstance(outputClass, conf);
+ OutputFormat<?, ?> outputFormat = ReflectionUtil.newInstance(outputClass, conf);
if (!(outputFormat instanceof HiveOutputFormat)) {
outputFormat = new HivePassThroughOutputFormat(outputFormat);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Wed Apr 15 18:41:11 2015
@@ -23,7 +23,6 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -61,7 +60,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobConfigurable;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* HiveInputFormat is a parameterized InputFormat which looks at the path name
@@ -156,11 +155,12 @@ public class HiveInputFormat<K extends W
return inputSplit.getLocations();
}
+ // TODO#: THIS
@Override
public void readFields(DataInput in) throws IOException {
String inputSplitClassName = in.readUTF();
try {
- inputSplit = (InputSplit) ReflectionUtils.newInstance(conf
+ inputSplit = (InputSplit) ReflectionUtil.newInstance(conf
.getClassByName(inputSplitClassName), conf);
} catch (Exception e) {
throw new IOException(
@@ -200,7 +200,7 @@ public class HiveInputFormat<K extends W
InputFormat<WritableComparable, Writable> instance = inputFormats.get(inputFormatClass);
if (instance == null) {
try {
- instance = (InputFormat<WritableComparable, Writable>) ReflectionUtils
+ instance = (InputFormat<WritableComparable, Writable>) ReflectionUtil
.newInstance(inputFormatClass, job);
// HBase input formats are not thread safe today. See HIVE-8808.
String inputFormatName = inputFormatClass.getName().toLowerCase();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Wed Apr 15 18:41:11 2015
@@ -40,7 +40,7 @@ import org.apache.hadoop.hive.serde2.Des
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -138,7 +138,7 @@ public class PartitionDesc implements Se
public Deserializer getDeserializer(Configuration conf) throws Exception {
Properties schema = getProperties();
String clazzName = getDeserializerClassName();
- Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(clazzName)
+ Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName)
.asSubclass(Deserializer.class), conf);
SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema);
return deserializer;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java Wed Apr 15 18:41:11 2015
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -40,7 +39,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* ReduceWork represents all the information used to run a reduce task on the cluster.
@@ -114,7 +113,7 @@ public class ReduceWork extends BaseWork
private ObjectInspector getObjectInspector(TableDesc desc) {
ObjectInspector objectInspector;
try {
- Deserializer deserializer = ReflectionUtils.newInstance(desc
+ Deserializer deserializer = ReflectionUtil.newInstance(desc
.getDeserializerClass(), null);
SerDeUtils.initializeSerDe(deserializer, null, desc.getProperties(), null);
objectInspector = deserializer.getObjectInspector();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java Wed Apr 15 18:41:11 2015
@@ -34,8 +34,9 @@ import org.apache.hadoop.hive.serde2.Des
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.util.ReflectionUtils;
+
import org.apache.hive.common.util.HiveStringUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* TableDesc.
@@ -90,7 +91,7 @@ public class TableDesc implements Serial
}
public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception {
- Deserializer de = ReflectionUtils.newInstance(
+ Deserializer de = ReflectionUtil.newInstance(
getDeserializerClass().asSubclass(Deserializer.class), conf);
if (ignoreError) {
SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null);