You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/04/15 20:41:11 UTC

svn commit: r1673926 - in /hive/trunk: common/src/java/org/apache/hive/common/util/ metastore/src/java/org/apache/hadoop/hive/metastore/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ ql/src/java/org...

Author: sershe
Date: Wed Apr 15 18:41:11 2015
New Revision: 1673926

URL: http://svn.apache.org/r1673926
Log:
HIVE-10329 : Hadoop reflectionutils has issues (Sergey Shelukhin, reviewed by Vikram Dixit K)

Added:
    hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java
Modified:
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java

Added: hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java?rev=1673926&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java (added)
+++ hive/trunk/common/src/java/org/apache/hive/common/util/ReflectionUtil.java Wed Apr 15 18:41:11 2015
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+/**
+ * Same as Hadoop ReflectionUtils, but (1) does not leak classloaders (or shouldn't anyway, we
+ * rely on Guava cache, and could fix it otherwise); (2) does not have a hidden epic lock.
+ */
+public class ReflectionUtil {
+
+  // TODO: expireAfterAccess locks cache segments on put and expired get. It doesn't look too bad,
+  //       but if we find some perf issues it might be a good idea to remove this - we are probably
+  //       not caching that many constructors.
+  // Note that weakKeys causes "==" to be used for key compare; this will only work
+  // for classes in the same classloader. Should be ok in this case.
+  private static final Cache<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
+      CacheBuilder.newBuilder().expireAfterAccess(15, TimeUnit.MINUTES)
+                               .concurrencyLevel(64)
+                               .weakKeys().weakValues().build();
+  private static final Class<?>[] EMPTY_ARRAY = new Class[] {};
+  private static final Class<?> jobConfClass, jobConfigurableClass;
+  private static final Method configureMethod;
+
+  static {
+    Class<?> jobConfClassLocal, jobConfigurableClassLocal;
+    Method configureMethodLocal;
+    try {
+      jobConfClassLocal = Class.forName("org.apache.hadoop.mapred.JobConf");
+      jobConfigurableClassLocal = Class.forName("org.apache.hadoop.mapred.JobConfigurable");
+      configureMethodLocal = jobConfigurableClassLocal.getMethod("configure", jobConfClassLocal);
+    } catch (Throwable t) {
+      // Meh.
+      jobConfClassLocal = jobConfigurableClassLocal = null;
+      configureMethodLocal = null;
+    }
+    jobConfClass = jobConfClassLocal;
+    jobConfigurableClass = jobConfigurableClassLocal;
+    configureMethod = configureMethodLocal;
+  }
+
+  /**
+   * Create an object for the given class and initialize it from conf
+   * @param theClass class of which an object is created
+   * @param conf Configuration
+   * @return a new object
+   */
+  @SuppressWarnings("unchecked")
+  public static <T> T newInstance(Class<T> theClass, Configuration conf) {
+    T result;
+    try {
+      Constructor<?> ctor = CONSTRUCTOR_CACHE.getIfPresent(theClass);
+      if (ctor == null) {
+        ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY);
+        ctor.setAccessible(true);
+        CONSTRUCTOR_CACHE.put(theClass, ctor);
+      }
+      result = (T)ctor.newInstance();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    setConf(result, conf);
+    return result;
+  }
+
+
+  /**
+   * Check and set 'configuration' if necessary.
+   *
+   * @param theObject object for which to set configuration
+   * @param conf Configuration
+   */
+  public static void setConf(Object theObject, Configuration conf) {
+    if (conf != null) {
+      if (theObject instanceof Configurable) {
+        ((Configurable) theObject).setConf(conf);
+      }
+      setJobConf(theObject, conf);
+    }
+  }
+
+  private static void setJobConf(Object theObject, Configuration conf) {
+    if (configureMethod == null) return;
+    try {
+      if (jobConfClass.isAssignableFrom(conf.getClass()) &&
+            jobConfigurableClass.isAssignableFrom(theObject.getClass())) {
+        configureMethod.invoke(theObject, conf);
+      }
+    } catch (Exception e) {
+      throw new RuntimeException("Error in configuring object", e);
+    }
+  }
+}

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Wed Apr 15 18:41:11 2015
@@ -80,7 +80,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 import javax.annotation.Nullable;
 
@@ -372,7 +372,7 @@ public class MetaStoreUtils {
       return null;
     }
     try {
-      Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(lib).
+      Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib).
               asSubclass(Deserializer.class), conf);
       if (skipConfError) {
         SerDeUtils.initializeSerDeWithoutErrorCheck(deserializer, conf,
@@ -419,7 +419,7 @@ public class MetaStoreUtils {
       org.apache.hadoop.hive.metastore.api.Table table) throws MetaException {
     String lib = part.getSd().getSerdeInfo().getSerializationLib();
     try {
-      Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(lib).
+      Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib).
         asSubclass(Deserializer.class), conf);
       SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table),
                                  MetaStoreUtils.getPartitionMetadata(part, table));

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Wed Apr 15 18:41:11 2015
@@ -207,9 +207,9 @@ import org.apache.hadoop.hive.shims.Hado
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.tools.HadoopArchives;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hive.common.util.AnnotationUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 import org.stringtemplate.v4.ST;
 
 /**
@@ -3786,7 +3786,7 @@ public class DDLTask extends Task<DDLWor
   private void validateSerDe(String serdeName) throws HiveException {
     try {
 
-      Deserializer d = ReflectionUtils.newInstance(conf.getClassByName(serdeName).
+      Deserializer d = ReflectionUtil.newInstance(conf.getClassByName(serdeName).
           asSubclass(Deserializer.class), conf);
       if (d != null) {
         LOG.debug("Found class for " + serdeName);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DefaultFetchFormatter.java Wed Apr 15 18:41:11 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.serde2.Del
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * serialize row by user specified serde and call toString() to make string type result
@@ -50,12 +50,13 @@ public class DefaultFetchFormatter<T> im
     }
   }
 
+//TODO#: THIS
   private SerDe initializeSerde(Configuration conf, Properties props) throws Exception {
     String serdeName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE);
     Class<? extends SerDe> serdeClass = Class.forName(serdeName, true,
         Utilities.getSessionSpecifiedClassLoader()).asSubclass(SerDe.class);
     // cast only needed for Hadoop 0.17 compatibility
-    SerDe serde = ReflectionUtils.newInstance(serdeClass, null);
+    SerDe serde = ReflectionUtil.newInstance(serdeClass, null);
 
     Properties serdeProps = new Properties();
     if (serde instanceof DelimitedJSONSerDe) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java Wed Apr 15 18:41:11 2015
@@ -39,7 +39,7 @@ import org.apache.hadoop.hive.serde2.Des
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * DemuxOperator is an operator used by MapReduce Jobs optimized by
@@ -134,12 +134,12 @@ public class DemuxOperator extends Opera
         cntrs[newTag] = 0;
         nextCntrs[newTag] = 0;
         TableDesc keyTableDesc = conf.getKeysSerializeInfos().get(newTag);
-        Deserializer inputKeyDeserializer = ReflectionUtils.newInstance(keyTableDesc
+        Deserializer inputKeyDeserializer = ReflectionUtil.newInstance(keyTableDesc
             .getDeserializerClass(), null);
         SerDeUtils.initializeSerDe(inputKeyDeserializer, null, keyTableDesc.getProperties(), null);
 
         TableDesc valueTableDesc = conf.getValuesSerializeInfos().get(newTag);
-        Deserializer inputValueDeserializer = ReflectionUtils.newInstance(valueTableDesc
+        Deserializer inputValueDeserializer = ReflectionUtil.newInstance(valueTableDesc
             .getDeserializerClass(), null);
         SerDeUtils.initializeSerDe(inputValueDeserializer, null, valueTableDesc.getProperties(),
                                    null);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Wed Apr 15 18:41:11 2015
@@ -69,9 +69,9 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobConfigurable;
 import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hive.common.util.AnnotationUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 import com.google.common.collect.Iterators;
 
@@ -204,13 +204,12 @@ public class FetchOperator implements Se
        JobConf conf) throws IOException {
     if (Configurable.class.isAssignableFrom(inputFormatClass) ||
         JobConfigurable.class.isAssignableFrom(inputFormatClass)) {
-      return ReflectionUtils
-          .newInstance(inputFormatClass, conf);
+      return ReflectionUtil.newInstance(inputFormatClass, conf);
     }
     InputFormat format = inputFormats.get(inputFormatClass.getName());
     if (format == null) {
       try {
-        format = ReflectionUtils.newInstance(inputFormatClass, conf);
+        format = ReflectionUtil.newInstance(inputFormatClass, conf);
         inputFormats.put(inputFormatClass.getName(), format);
       } catch (Exception e) {
         throw new IOException("Cannot create an instance of InputFormat class "

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java Wed Apr 15 18:41:11 2015
@@ -44,7 +44,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 public class JoinUtil {
 
@@ -268,7 +268,7 @@ public class JoinUtil {
     if (desc == null) {
       return null;
     }
-    SerDe sd = (SerDe) ReflectionUtils.newInstance(desc.getDeserializerClass(),
+    SerDe sd = (SerDe) ReflectionUtil.newInstance(desc.getDeserializerClass(),
         null);
     try {
       SerDeUtils.initializeSerDe(sd, null, desc.getProperties(), null);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Wed Apr 15 18:41:11 2015
@@ -61,7 +61,7 @@ import org.apache.hadoop.hive.serde2.Ser
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 import static org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition;
 import static org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer.KeyValueHelper;
@@ -229,8 +229,8 @@ public class MapJoinOperator extends Abs
 
     try {
       TableDesc keyTableDesc = conf.getKeyTblDesc();
-      SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
- null);
+      SerDe keySerializer = (SerDe) ReflectionUtil.newInstance(
+          keyTableDesc.getDeserializerClass(), null);
       SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null);
       MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false);
       for (int pos = 0; pos < order.length; pos++) {
@@ -243,8 +243,8 @@ public class MapJoinOperator extends Abs
         } else {
           valueTableDesc = conf.getValueFilteredTblDescs().get(pos);
         }
-        SerDe valueSerDe =
-            (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
+        SerDe valueSerDe = (SerDe) ReflectionUtil.newInstance(
+            valueTableDesc.getDeserializerClass(), null);
         SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
         MapJoinObjectSerDeContext valueContext =
             new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos));

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Wed Apr 15 18:41:11 2015
@@ -51,7 +51,7 @@ import org.apache.hadoop.io.WritableComp
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.util.PriorityQueue;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * Sorted Merge Map Join Operator.
@@ -525,7 +525,7 @@ public class SMBMapJoinOperator extends
 
     BucketMapJoinContext bucketMatcherCxt = localWork.getBucketMapjoinContext();
     Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
-    BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls, null);
+    BucketMatcher bucketMatcher = ReflectionUtil.newInstance(bucketMatcherCls, null);
 
     getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputPath.toString()));
     if (isLogInfoEnabled) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java Wed Apr 15 18:41:11 2015
@@ -52,8 +52,8 @@ import org.apache.hadoop.mapred.InputFor
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.MapMaker;
@@ -150,7 +150,7 @@ public class StatsNoJobTask extends Task
         boolean statsAvailable = false;
         for(FileStatus file: fileList) {
           if (!file.isDir()) {
-            InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
+            InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtil.newInstance(
                 partn.getInputFormatClass(), jc);
             InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0,
                 new String[] { partn.getLocation() });
@@ -248,7 +248,7 @@ public class StatsNoJobTask extends Task
           boolean statsAvailable = false;
           for(FileStatus file: fileList) {
             if (!file.isDir()) {
-              InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
+              InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtil.newInstance(
                   table.getInputFormatClass(), jc);
               InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { table
                   .getDataLocation().toString() });

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Apr 15 18:41:11 2015
@@ -181,8 +181,8 @@ import org.apache.hadoop.mapred.Reporter
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Shell;
+import org.apache.hive.common.util.ReflectionUtil;
 
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.io.Input;
@@ -1375,7 +1375,7 @@ public final class Utilities {
     if (isCompressed) {
       Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
           DefaultCodec.class);
-      CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jc);
+      CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc);
       return codec.createOutputStream(out);
     } else {
       return (out);
@@ -1424,7 +1424,7 @@ public final class Utilities {
     if ((hiveOutputFormat instanceof HiveIgnoreKeyTextOutputFormat) && isCompressed) {
       Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc,
           DefaultCodec.class);
-      CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jc);
+      CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc);
       return codec.getDefaultExtension();
     }
     return "";
@@ -1476,7 +1476,7 @@ public final class Utilities {
     if (isCompressed) {
       compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc);
       codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
-      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
+      codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc);
     }
     return SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec,
       progressable);
@@ -1500,7 +1500,7 @@ public final class Utilities {
     CompressionCodec codec = null;
     if (isCompressed) {
       Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
-      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
+      codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc);
     }
     return new RCFile.Writer(fs, jc, file, progressable, codec);
   }
@@ -2948,7 +2948,7 @@ public final class Utilities {
 
         if (reworkInputFormats.size() > 0) {
           for (Class<? extends InputFormat> inputFormatCls : reworkInputFormats) {
-            ReworkMapredInputFormat inst = (ReworkMapredInputFormat) ReflectionUtils
+            ReworkMapredInputFormat inst = (ReworkMapredInputFormat) ReflectionUtil
                 .newInstance(inputFormatCls, null);
             inst.rework(conf, mapredWork);
           }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java Wed Apr 15 18:41:11 2015
@@ -46,7 +46,7 @@ import org.apache.hadoop.mapred.InputFor
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * Simple persistent container for rows.
@@ -213,7 +213,7 @@ public class RowContainer<ROW extends Li
         JobConf localJc = getLocalFSJobConfClone(jc);
         if (inputSplits == null) {
           if (this.inputFormat == null) {
-            inputFormat = ReflectionUtils.newInstance(
+            inputFormat = ReflectionUtil.newInstance(
                 tblDesc.getInputFileFormatClass(), localJc);
           }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputSplit.java Wed Apr 15 18:41:11 2015
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.io.HiveInputFormat.HiveInputSplit;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * HiveInputSplit encapsulates an InputSplit with its corresponding
@@ -137,12 +137,11 @@ public class BucketizedHiveInputSplit ex
   @Override
   public void readFields(DataInput in) throws IOException {
     String inputSplitClassName = in.readUTF();
-
     int numSplits = in.readInt();
     inputSplits = new InputSplit[numSplits];
     for (int i = 0; i < numSplits; i++) {
       try {
-        inputSplits[i] = (InputSplit) ReflectionUtils.newInstance(conf
+        inputSplits[i] = (InputSplit) ReflectionUtil.newInstance(conf
             .getClassByName(inputSplitClassName), conf);
       } catch (Exception e) {
         throw new IOException(

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Wed Apr 15 18:41:11 2015
@@ -59,7 +59,7 @@ import org.apache.hadoop.mapred.Sequence
 import org.apache.hadoop.mapred.TaskAttemptContext;
 import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * An util class for various Hive file format tasks.
@@ -274,7 +274,7 @@ public final class HiveFileFormatUtils {
 
   private static HiveOutputFormat<?, ?> getHiveOutputFormat(
       Configuration conf, Class<? extends OutputFormat> outputClass) throws HiveException {
-    OutputFormat<?, ?> outputFormat = ReflectionUtils.newInstance(outputClass, conf);
+    OutputFormat<?, ?> outputFormat = ReflectionUtil.newInstance(outputClass, conf);
     if (!(outputFormat instanceof HiveOutputFormat)) {
       outputFormat = new HivePassThroughOutputFormat(outputFormat);
     }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Wed Apr 15 18:41:11 2015
@@ -23,7 +23,6 @@ import java.io.DataOutput;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -61,7 +60,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobConfigurable;
 import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * HiveInputFormat is a parameterized InputFormat which looks at the path name
@@ -156,11 +155,12 @@ public class HiveInputFormat<K extends W
       return inputSplit.getLocations();
     }
 
+ // TODO#: THIS
     @Override
     public void readFields(DataInput in) throws IOException {
       String inputSplitClassName = in.readUTF();
       try {
-        inputSplit = (InputSplit) ReflectionUtils.newInstance(conf
+        inputSplit = (InputSplit) ReflectionUtil.newInstance(conf
             .getClassByName(inputSplitClassName), conf);
       } catch (Exception e) {
         throw new IOException(
@@ -200,7 +200,7 @@ public class HiveInputFormat<K extends W
     InputFormat<WritableComparable, Writable> instance = inputFormats.get(inputFormatClass);
     if (instance == null) {
       try {
-        instance = (InputFormat<WritableComparable, Writable>) ReflectionUtils
+        instance = (InputFormat<WritableComparable, Writable>) ReflectionUtil
             .newInstance(inputFormatClass, job);
         // HBase input formats are not thread safe today. See HIVE-8808.
         String inputFormatName = inputFormatClass.getName().toLowerCase();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Wed Apr 15 18:41:11 2015
@@ -40,7 +40,7 @@ import org.apache.hadoop.hive.serde2.Des
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 import org.apache.hive.common.util.HiveStringUtils;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 
@@ -138,7 +138,7 @@ public class PartitionDesc implements Se
   public Deserializer getDeserializer(Configuration conf) throws Exception {
     Properties schema = getProperties();
     String clazzName = getDeserializerClassName();
-    Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(clazzName)
+    Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName)
         .asSubclass(Deserializer.class), conf);
     SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema);
     return deserializer;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java Wed Apr 15 18:41:11 2015
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.plan;
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -40,7 +39,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * ReduceWork represents all the information used to run a reduce task on the cluster.
@@ -114,7 +113,7 @@ public class ReduceWork extends BaseWork
   private ObjectInspector getObjectInspector(TableDesc desc) {
     ObjectInspector objectInspector;
     try {
-      Deserializer deserializer = ReflectionUtils.newInstance(desc
+      Deserializer deserializer = ReflectionUtil.newInstance(desc
                 .getDeserializerClass(), null);
       SerDeUtils.initializeSerDe(deserializer, null, desc.getProperties(), null);
       objectInspector = deserializer.getObjectInspector();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java?rev=1673926&r1=1673925&r2=1673926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java Wed Apr 15 18:41:11 2015
@@ -34,8 +34,9 @@ import org.apache.hadoop.hive.serde2.Des
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.util.ReflectionUtils;
+
 import org.apache.hive.common.util.HiveStringUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * TableDesc.
@@ -90,7 +91,7 @@ public class TableDesc implements Serial
   }
 
   public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception {
-    Deserializer de = ReflectionUtils.newInstance(
+    Deserializer de = ReflectionUtil.newInstance(
         getDeserializerClass().asSubclass(Deserializer.class), conf);
     if (ignoreError) {
       SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null);