You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/04/17 01:14:05 UTC

svn commit: r1674166 - in /hive/trunk: hbase-handler/src/java/org/apache/hadoop/hive/hbase/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ ql/src/java/org/apache/hadoop/hive/ql/plan/

Author: sershe
Date: Thu Apr 16 23:14:04 2015
New Revision: 1674166

URL: http://svn.apache.org/r1674166
Log:
HIVE-10346 : Tez on HBase has problems with settings again (Sergey Shelukhin, reviewed by Gunther Hagleitner)

Modified:
    hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java Thu Apr 16 23:14:04 2015
@@ -380,6 +380,7 @@ public class HBaseStorageHandler extends
     // do this for reconciling HBaseStorageHandler for use in HCatalog
     // check to see if this an input job or an outputjob
     if (this.configureInputJobProps) {
+      LOG.info("Configuring input job properties");
       String snapshotName = HiveConf.getVar(jobConf, HiveConf.ConfVars.HIVE_HBASE_SNAPSHOT_NAME);
       if (snapshotName != null) {
         HBaseTableSnapshotInputFormatUtil.assertSupportsTableSnapshots();
@@ -428,6 +429,7 @@ public class HBaseStorageHandler extends
       } //input job properties
     }
     else {
+      LOG.info("Configuring output job properties");
       if (isHBaseGenerateHFiles(jobConf)) {
         // only support bulkload when a hfile.family.path has been specified.
         // TODO: support detecting cf's from column mapping

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Thu Apr 16 23:14:04 2015
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hive.con
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
+import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -30,6 +31,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.Future;
 
@@ -59,6 +61,7 @@ import org.apache.hadoop.hive.ql.plan.Fi
 import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
 import org.apache.hadoop.hive.ql.plan.PlanUtils;
 import org.apache.hadoop.hive.ql.plan.SkewedColumnPositionPair;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.ql.stats.StatsCollectionTaskIndependent;
 import org.apache.hadoop.hive.ql.stats.StatsPublisher;
@@ -339,7 +342,12 @@ public class FileSinkOperator extends Te
       taskId = Utilities.getTaskId(hconf);
       initializeSpecPath();
       fs = specPath.getFileSystem(hconf);
-      hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(hconf, conf.getTableInfo());
+      try {
+        createHiveOutputFormat(hconf);
+      } catch (HiveException ex) {
+        logOutputFormatError(hconf, ex);
+        throw ex;
+      }
       isCompressed = conf.getCompressed();
       parent = Utilities.toTempPath(conf.getDirName());
       statsCollectRawDataSize = conf.isStatsCollectRawDataSize();
@@ -440,6 +448,26 @@ public class FileSinkOperator extends Te
     return result;
   }
 
+  private void logOutputFormatError(Configuration hconf, HiveException ex) {
+    StringWriter errorWriter = new StringWriter();
+    errorWriter.append("Failed to create output format; configuration: ");
+    try {
+      Configuration.dumpConfiguration(hconf, errorWriter);
+    } catch (IOException ex2) {
+      errorWriter.append("{ failed to dump configuration: " + ex2.getMessage() + " }");
+    }
+    Properties tdp = null;
+    if (this.conf.getTableInfo() != null
+        && (tdp = this.conf.getTableInfo().getProperties()) != null) {
+      errorWriter.append(";\n table properties: { ");
+      for (Map.Entry<Object, Object> e : tdp.entrySet()) {
+        errorWriter.append(e.getKey() + ": " + e.getValue() + ", ");
+      }
+      errorWriter.append('}');
+    }
+    LOG.error(errorWriter.toString(), ex);
+  }
+
   /**
    * Initialize list bucketing information
    */
@@ -1082,10 +1110,10 @@ public class FileSinkOperator extends Te
 
   public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
     if (hiveOutputFormat == null) {
-      Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), job);
       try {
-        hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(job, getConf().getTableInfo());
-      } catch (Exception ex) {
+        createHiveOutputFormat(job);
+      } catch (HiveException ex) {
+        logOutputFormatError(job, ex);
         throw new IOException(ex);
       }
     }
@@ -1101,6 +1129,17 @@ public class FileSinkOperator extends Te
     }
   }
 
+  private void createHiveOutputFormat(Configuration hconf) throws HiveException {
+    if (hiveOutputFormat == null) {
+      Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), hconf);
+    }
+    try {
+      hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(hconf, getConf().getTableInfo());
+    } catch (Throwable t) {
+      throw (t instanceof HiveException) ? (HiveException)t : new HiveException(t);
+    }
+  }
+
   private void publishStats() throws HiveException {
     boolean isStatsReliable = conf.isStatsReliable();
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu Apr 16 23:14:04 2015
@@ -2398,7 +2398,7 @@ public final class Utilities {
    * @param job
    *          configuration which receives configured properties
    */
-  public static void copyTableJobPropertiesToConf(TableDesc tbl, JobConf job) {
+  public static void copyTableJobPropertiesToConf(TableDesc tbl, Configuration job) {
     Properties tblProperties = tbl.getProperties();
     for(String name: tblProperties.stringPropertyNames()) {
       if (job.get(name) == null) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java Thu Apr 16 23:14:04 2015
@@ -74,7 +74,6 @@ public class MetadataReader {
       // filter and combine the io to read row index and bloom filters for that column together
       if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
         boolean readBloomFilter = false;
-        // TODO#: HERE
         if (sargColumns != null && sargColumns[col] &&
             nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
           len += nextStream.getLength();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Thu Apr 16 23:14:04 2015
@@ -812,7 +812,7 @@ public final class PlanUtils {
                   tableDesc,
                   jobProperties);
             } catch(AbstractMethodError e) {
-                LOG.debug("configureInputJobProperties not found "+
+                LOG.info("configureInputJobProperties not found "+
                     "using configureTableJobProperties",e);
                 storageHandler.configureTableJobProperties(tableDesc, jobProperties);
             }
@@ -823,7 +823,7 @@ public final class PlanUtils {
                   tableDesc,
                   jobProperties);
             } catch(AbstractMethodError e) {
-                LOG.debug("configureOutputJobProperties not found"+
+                LOG.info("configureOutputJobProperties not found"+
                     "using configureTableJobProperties",e);
                 storageHandler.configureTableJobProperties(tableDesc, jobProperties);
             }