You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/04/17 01:14:05 UTC
svn commit: r1674166 - in /hive/trunk:
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/io/orc/
ql/src/java/org/apache/hadoop/hive/ql/plan/
Author: sershe
Date: Thu Apr 16 23:14:04 2015
New Revision: 1674166
URL: http://svn.apache.org/r1674166
Log:
HIVE-10346 : Tez on HBase has problems with settings again (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Modified:
hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java Thu Apr 16 23:14:04 2015
@@ -380,6 +380,7 @@ public class HBaseStorageHandler extends
// do this for reconciling HBaseStorageHandler for use in HCatalog
// check to see if this an input job or an outputjob
if (this.configureInputJobProps) {
+ LOG.info("Configuring input job properties");
String snapshotName = HiveConf.getVar(jobConf, HiveConf.ConfVars.HIVE_HBASE_SNAPSHOT_NAME);
if (snapshotName != null) {
HBaseTableSnapshotInputFormatUtil.assertSupportsTableSnapshots();
@@ -428,6 +429,7 @@ public class HBaseStorageHandler extends
} //input job properties
}
else {
+ LOG.info("Configuring output job properties");
if (isHBaseGenerateHFiles(jobConf)) {
// only support bulkload when a hfile.family.path has been specified.
// TODO: support detecting cf's from column mapping
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Thu Apr 16 23:14:04 2015
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hive.con
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
+import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -30,6 +31,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Future;
@@ -59,6 +61,7 @@ import org.apache.hadoop.hive.ql.plan.Fi
import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.SkewedColumnPositionPair;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.ql.stats.StatsCollectionTaskIndependent;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
@@ -339,7 +342,12 @@ public class FileSinkOperator extends Te
taskId = Utilities.getTaskId(hconf);
initializeSpecPath();
fs = specPath.getFileSystem(hconf);
- hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(hconf, conf.getTableInfo());
+ try {
+ createHiveOutputFormat(hconf);
+ } catch (HiveException ex) {
+ logOutputFormatError(hconf, ex);
+ throw ex;
+ }
isCompressed = conf.getCompressed();
parent = Utilities.toTempPath(conf.getDirName());
statsCollectRawDataSize = conf.isStatsCollectRawDataSize();
@@ -440,6 +448,26 @@ public class FileSinkOperator extends Te
return result;
}
+ private void logOutputFormatError(Configuration hconf, HiveException ex) {
+ StringWriter errorWriter = new StringWriter();
+ errorWriter.append("Failed to create output format; configuration: ");
+ try {
+ Configuration.dumpConfiguration(hconf, errorWriter);
+ } catch (IOException ex2) {
+ errorWriter.append("{ failed to dump configuration: " + ex2.getMessage() + " }");
+ }
+ Properties tdp = null;
+ if (this.conf.getTableInfo() != null
+ && (tdp = this.conf.getTableInfo().getProperties()) != null) {
+ errorWriter.append(";\n table properties: { ");
+ for (Map.Entry<Object, Object> e : tdp.entrySet()) {
+ errorWriter.append(e.getKey() + ": " + e.getValue() + ", ");
+ }
+ errorWriter.append('}');
+ }
+ LOG.error(errorWriter.toString(), ex);
+ }
+
/**
* Initialize list bucketing information
*/
@@ -1082,10 +1110,10 @@ public class FileSinkOperator extends Te
public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
if (hiveOutputFormat == null) {
- Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), job);
try {
- hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(job, getConf().getTableInfo());
- } catch (Exception ex) {
+ createHiveOutputFormat(job);
+ } catch (HiveException ex) {
+ logOutputFormatError(job, ex);
throw new IOException(ex);
}
}
@@ -1101,6 +1129,17 @@ public class FileSinkOperator extends Te
}
}
+ private void createHiveOutputFormat(Configuration hconf) throws HiveException {
+ if (hiveOutputFormat == null) {
+ Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), hconf);
+ }
+ try {
+ hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(hconf, getConf().getTableInfo());
+ } catch (Throwable t) {
+ throw (t instanceof HiveException) ? (HiveException)t : new HiveException(t);
+ }
+ }
+
private void publishStats() throws HiveException {
boolean isStatsReliable = conf.isStatsReliable();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu Apr 16 23:14:04 2015
@@ -2398,7 +2398,7 @@ public final class Utilities {
* @param job
* configuration which receives configured properties
*/
- public static void copyTableJobPropertiesToConf(TableDesc tbl, JobConf job) {
+ public static void copyTableJobPropertiesToConf(TableDesc tbl, Configuration job) {
Properties tblProperties = tbl.getProperties();
for(String name: tblProperties.stringPropertyNames()) {
if (job.get(name) == null) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java Thu Apr 16 23:14:04 2015
@@ -74,7 +74,6 @@ public class MetadataReader {
// filter and combine the io to read row index and bloom filters for that column together
if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
boolean readBloomFilter = false;
- // TODO#: HERE
if (sargColumns != null && sargColumns[col] &&
nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
len += nextStream.getLength();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1674166&r1=1674165&r2=1674166&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Thu Apr 16 23:14:04 2015
@@ -812,7 +812,7 @@ public final class PlanUtils {
tableDesc,
jobProperties);
} catch(AbstractMethodError e) {
- LOG.debug("configureInputJobProperties not found "+
+ LOG.info("configureInputJobProperties not found "+
"using configureTableJobProperties",e);
storageHandler.configureTableJobProperties(tableDesc, jobProperties);
}
@@ -823,7 +823,7 @@ public final class PlanUtils {
tableDesc,
jobProperties);
} catch(AbstractMethodError e) {
- LOG.debug("configureOutputJobProperties not found"+
+ LOG.info("configureOutputJobProperties not found"+
"using configureTableJobProperties",e);
storageHandler.configureTableJobProperties(tableDesc, jobProperties);
}