You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2015/01/15 09:45:17 UTC
svn commit: r1651972 - in /hive/trunk: ./ hbase-handler/ itests/
itests/qtest-spark/ itests/src/test/resources/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/
ql/src/java/org/apache/hadoop/hive/ql/optimize...
Author: szehon
Date: Thu Jan 15 08:45:16 2015
New Revision: 1651972
URL: http://svn.apache.org/r1651972
Log:
HIVE-9352 : Merge from spark to trunk (follow-up of HIVE-9257) (Szehon, reviewed by Brock)
Removed:
hive/trunk/ql/src/test/queries/clientpositive/spark_test.q
hive/trunk/ql/src/test/results/clientpositive/spark/spark_test.q.out
hive/trunk/ql/src/test/results/clientpositive/spark_test.q.out
Modified:
hive/trunk/ (props changed)
hive/trunk/hbase-handler/pom.xml (props changed)
hive/trunk/itests/pom.xml
hive/trunk/itests/qtest-spark/pom.xml
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/pom.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Propchange: hive/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 15 08:45:16 2015
@@ -1,5 +1,5 @@
/hive/branches/branch-0.11:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184
/hive/branches/cbo:1605012-1627125
-/hive/branches/spark:1608589-1650662
+/hive/branches/spark:1608589-1651242
/hive/branches/tez:1494760-1622766
/hive/branches/vectorization:1466908-1527856
Propchange: hive/trunk/hbase-handler/pom.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 15 08:45:16 2015
@@ -1,6 +1,6 @@
/hive/branches/branch-0.11/hbase-handler/pom.xml:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184
/hive/branches/cbo/hbase-handler/pom.xml:1605012-1627125
-/hive/branches/spark/hbase-handler/pom.xml:1608589-1650662
+/hive/branches/spark/hbase-handler/pom.xml:1608589-1651242
/hive/branches/tez/hbase-handler/pom.xml:1494760-1622766
/hive/branches/vectorization/hbase-handler/pom.xml:1466908-1527856
/hive/trunk/hbase-handler/pom.xml:1494760-1537575
Modified: hive/trunk/itests/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/pom.xml?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/itests/pom.xml (original)
+++ hive/trunk/itests/pom.xml Thu Jan 15 08:45:16 2015
@@ -90,7 +90,7 @@
mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName
}
mkdir -p $DOWNLOAD_DIR
- download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark"
+ download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark"
cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
</echo>
</target>
Modified: hive/trunk/itests/qtest-spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest-spark/pom.xml?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/itests/qtest-spark/pom.xml (original)
+++ hive/trunk/itests/qtest-spark/pom.xml Thu Jan 15 08:45:16 2015
@@ -54,38 +54,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-api</artifactId>
- <version>${tez.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>${spark.jetty.version}</version>
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Thu Jan 15 08:45:16 2015
@@ -842,7 +842,6 @@ spark.query.files=add_part_multiple.q, \
smb_mapjoin_8.q, \
smb_mapjoin_9.q, \
sort.q, \
- spark_test.q, \
stats0.q, \
stats1.q, \
stats10.q, \
Modified: hive/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/pom.xml (original)
+++ hive/trunk/pom.xml Thu Jan 15 08:45:16 2015
@@ -154,7 +154,7 @@
<ST4.version>4.0.4</ST4.version>
<tez.version>0.5.2</tez.version>
<super-csv.version>2.2.0</super-csv.version>
- <spark.version>1.2.1-SNAPSHOT</spark.version>
+ <spark.version>1.2.0</spark.version>
<scala.binary.version>2.10</scala.binary.version>
<scala.version>2.10.4</scala.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
@@ -212,17 +212,7 @@
<snapshots>
<enabled>false</enabled>
</snapshots>
- </repository>
- <repository>
- <id>spark-snapshot</id>
- <url>http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark_2.10-1.2-SNAPSHOT/</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
+ </repository>
</repositories>
<!-- Hadoop dependency management is done at the bottom under profiles -->
@@ -833,6 +823,7 @@
<LANG>en_US.UTF-8</LANG>
<HADOOP_CLASSPATH>${test.tmp.dir}/conf:${basedir}/${hive.path.to.root}/conf</HADOOP_CLASSPATH>
<HIVE_HADOOP_TEST_CLASSPATH>${test.hive.hadoop.classpath}</HIVE_HADOOP_TEST_CLASSPATH>
+ <SPARK_SUBMIT_CLASSPATH>${spark.home}/lib/spark-assembly-${spark.version}-hadoop2.4.0.jar:${test.hive.hadoop.classpath}</SPARK_SUBMIT_CLASSPATH>
<PATH>${env.PATH}${test.extra.path}</PATH>
</environmentVariables>
<systemPropertyVariables>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Thu Jan 15 08:45:16 2015
@@ -32,13 +32,13 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
-import java.util.LinkedHashMap;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -48,7 +48,12 @@ import org.apache.hadoop.hive.ql.hooks.R
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.plan.*;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.ExplainWork;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.SparkWork;
+import org.apache.hadoop.hive.ql.plan.TezWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.security.authorization.AuthorizationFactory;
import org.apache.hadoop.hive.ql.session.SessionState;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Thu Jan 15 08:45:16 2015
@@ -96,8 +96,8 @@ public class HashTableSinkOperator exten
protected transient MapJoinPersistableTableContainer[] mapJoinTables;
protected transient MapJoinTableContainerSerDe[] mapJoinTableSerdes;
- private final Object[] EMPTY_OBJECT_ARRAY = new Object[0];
- private final MapJoinEagerRowContainer EMPTY_ROW_CONTAINER = new MapJoinEagerRowContainer();
+ private final Object[] emptyObjectArray = new Object[0];
+ private final MapJoinEagerRowContainer emptyRowContainer = new MapJoinEagerRowContainer();
private long rowNumber = 0;
protected transient LogHelper console;
@@ -118,7 +118,7 @@ public class HashTableSinkOperator exten
boolean isSilent = HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVESESSIONSILENT);
console = new LogHelper(LOG, isSilent);
memoryExhaustionHandler = new MapJoinMemoryExhaustionHandler(console, conf.getHashtableMemoryUsage());
- EMPTY_ROW_CONTAINER.addRow(EMPTY_OBJECT_ARRAY);
+ emptyRowContainer.addRow(emptyObjectArray);
// for small tables only; so get the big table position first
posBigTableAlias = conf.getPosBigTable();
@@ -229,7 +229,7 @@ public class HashTableSinkOperator exten
MapJoinKeyObject key = new MapJoinKeyObject();
key.readFromRow(currentKey, joinKeysObjectInspectors[alias]);
- Object[] value = EMPTY_OBJECT_ARRAY;
+ Object[] value = emptyObjectArray;
if((hasFilter(alias) && filterMaps[alias].length > 0) || joinValues[alias].size() > 0) {
value = JoinUtil.computeMapJoinValues(row, joinValues[alias],
joinValuesObjectInspectors[alias], joinFilters[alias], joinFilterObjectInspectors[alias],
@@ -242,14 +242,14 @@ public class HashTableSinkOperator exten
rowContainer = new MapJoinEagerRowContainer();
rowContainer.addRow(value);
} else {
- rowContainer = EMPTY_ROW_CONTAINER;
+ rowContainer = emptyRowContainer;
}
rowNumber++;
if (rowNumber > hashTableScale && rowNumber % hashTableScale == 0) {
memoryExhaustionHandler.checkMemoryStatus(tableContainer.size(), rowNumber);
}
tableContainer.put(key, rowContainer);
- } else if (rowContainer == EMPTY_ROW_CONTAINER) {
+ } else if (rowContainer == emptyRowContainer) {
rowContainer = rowContainer.copy();
rowContainer.addRow(value);
tableContainer.put(key, rowContainer);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java Thu Jan 15 08:45:16 2015
@@ -26,13 +26,13 @@ import org.apache.spark.api.java.functio
public abstract class HivePairFlatMapFunction<T, K, V> implements PairFlatMapFunction<T, K, V> {
- private static final NumberFormat TASK_ID_FORMAT = NumberFormat.getInstance();
- private static final NumberFormat STAGE_ID_FORMAT = NumberFormat.getInstance();
- static {
- TASK_ID_FORMAT.setGroupingUsed(false);
- TASK_ID_FORMAT.setMinimumIntegerDigits(6);
- STAGE_ID_FORMAT.setGroupingUsed(false);
- STAGE_ID_FORMAT.setMinimumIntegerDigits(4);
+ private final NumberFormat taskIdFormat = NumberFormat.getInstance();
+ private final NumberFormat stageIdFormat = NumberFormat.getInstance();
+ {
+ taskIdFormat.setGroupingUsed(false);
+ taskIdFormat.setMinimumIntegerDigits(6);
+ stageIdFormat.setGroupingUsed(false);
+ stageIdFormat.setMinimumIntegerDigits(4);
}
protected transient JobConf jobConf;
@@ -60,7 +60,7 @@ public abstract class HivePairFlatMapFun
StringBuilder taskAttemptIdBuilder = new StringBuilder("attempt_");
taskAttemptIdBuilder.append(System.currentTimeMillis())
.append("_")
- .append(STAGE_ID_FORMAT.format(TaskContext.get().stageId()))
+ .append(stageIdFormat.format(TaskContext.get().stageId()))
.append("_");
if (isMap()) {
@@ -71,7 +71,7 @@ public abstract class HivePairFlatMapFun
// Spark task attempt id is increased by Spark context instead of task, which may introduce
// unstable qtest output, since non Hive features depends on this, we always set it to 0 here.
- taskAttemptIdBuilder.append(TASK_ID_FORMAT.format(TaskContext.get().partitionId()))
+ taskAttemptIdBuilder.append(taskIdFormat.format(TaskContext.get().partitionId()))
.append("_0");
String taskAttemptIdStr = taskAttemptIdBuilder.toString();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java Thu Jan 15 08:45:16 2015
@@ -55,13 +55,13 @@ public class KryoSerializer {
try {
jobConf.write(new DataOutputStream(out));
} catch (IOException e) {
- LOG.error("Error serializing job configuration", e);
+ LOG.error("Error serializing job configuration: " + e, e);
return null;
} finally {
try {
out.close();
} catch (IOException e) {
- LOG.error("Error closing output stream", e);
+ LOG.error("Error closing output stream: " + e, e);
}
}
@@ -74,8 +74,8 @@ public class KryoSerializer {
try {
conf.readFields(new DataInputStream(new ByteArrayInputStream(buffer)));
} catch (IOException e) {
- LOG.error("Error de-serializing job configuration");
- return null;
+ String msg = "Error de-serializing job configuration: " + e;
+ throw new IllegalStateException(msg, e);
}
return conf;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java Thu Jan 15 08:45:16 2015
@@ -39,7 +39,6 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
import java.util.Iterator;
@@ -57,9 +56,9 @@ import java.util.List;
*
*/
public class SparkMapRecordHandler extends SparkRecordHandler {
+ private static final Log LOG = LogFactory.getLog(SparkMapRecordHandler.class);
private static final String PLAN_KEY = "__MAP_PLAN__";
private MapOperator mo;
- public static final Log LOG = LogFactory.getLog(SparkMapRecordHandler.class);
private MapredLocalWork localWork = null;
private boolean isLogInfoEnabled = false;
private ExecMapperContext execContext;
@@ -125,7 +124,7 @@ public class SparkMapRecordHandler exten
// Don't create a new object if we are already out of memory
throw (OutOfMemoryError) e;
} else {
- throw new RuntimeException("Map operator initialization failed", e);
+ throw new RuntimeException("Map operator initialization failed: " + e, e);
}
}
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS);
@@ -149,8 +148,9 @@ public class SparkMapRecordHandler exten
// Don't create a new object if we are already out of memory
throw (OutOfMemoryError) e;
} else {
- LOG.fatal(StringUtils.stringifyException(e));
- throw new RuntimeException(e);
+ String msg = "Error processing row: " + e;
+ LOG.fatal(msg, e);
+ throw new RuntimeException(msg, e);
}
}
}
@@ -196,8 +196,9 @@ public class SparkMapRecordHandler exten
} catch (Exception e) {
if (!abort) {
// signal new failure to map-reduce
- LOG.error("Hit error while closing operators - failing tree");
- throw new IllegalStateException("Error while closing operators", e);
+ String msg = "Hit error while closing operators - failing tree: " + e;
+ LOG.error(msg, e);
+ throw new IllegalStateException(msg, e);
}
} finally {
MapredContext.close();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java Thu Jan 15 08:45:16 2015
@@ -322,10 +322,6 @@ public class GenSparkSkewJoinProcessor {
}
}
- // keep it as reference in case we need fetch work
-// localPlan.getAliasToFetchWork().put(small_alias.toString(),
-// new FetchWork(tblDir, tableDescList.get(small_alias)));
-
listWorks.add(skewJoinMapJoinTask.getWork());
listTasks.add(skewJoinMapJoinTask);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java Thu Jan 15 08:45:16 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.plan.Op
/**
* Walks the operator tree in DFS fashion.
- *
+ *
* Cloned from GenTezWorkWarlker.
*/
public class GenSparkWorkWalker extends DefaultGraphWalker {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java Thu Jan 15 08:45:16 2015
@@ -98,11 +98,6 @@ public class SparkCompiler extends TaskC
}
@Override
- public void init(HiveConf conf, LogHelper console, Hive db) {
- super.init(conf, console, db);
- }
-
- @Override
protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs,
Set<WriteEntity> outputs) throws SemanticException {
PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java Thu Jan 15 08:45:16 2015
@@ -42,7 +42,7 @@ public class SparkEdgeProperty {
public boolean isShuffleNone() {
return edgeType == SHUFFLE_NONE;
}
-
+
public void setShuffleNone() {
edgeType = SHUFFLE_NONE;
}
@@ -80,7 +80,7 @@ public class SparkEdgeProperty {
if (isShuffleNone()) {
return "NONE";
}
-
+
StringBuilder sb = new StringBuilder();
if (isShuffleGroup()) {
sb.append("GROUP");
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java Thu Jan 15 08:45:16 2015
@@ -53,7 +53,8 @@ public class CounterStatsPublisher imple
try {
reporter.incrCounter(fileID, entry.getKey(), Long.valueOf(entry.getValue()));
} catch (Exception e) {
- LOG.error("Failed to increment counter value " + entry.getValue() + " for " + entry.getKey());
+ LOG.error("Failed to increment counter value " + entry.getValue() + " for " + entry.getKey()
+ + ": " + e, e);
return false;
}
}
Modified: hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Thu Jan 15 08:45:16 2015
@@ -71,7 +71,6 @@ import org.apache.hadoop.mapreduce.TaskI
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.hadoop.mapreduce.util.ResourceBundles;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.UserGroupInformation;
Modified: hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java (original)
+++ hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java Thu Jan 15 08:45:16 2015
@@ -49,6 +49,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
@@ -63,6 +64,7 @@ class SparkClientImpl implements SparkCl
private static final String DRIVER_OPTS_KEY = "spark.driver.extraJavaOptions";
private static final String EXECUTOR_OPTS_KEY = "spark.executor.extraJavaOptions";
+ private static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath";
private final Map<String, String> conf;
private final AtomicInteger childIdGenerator;
@@ -231,6 +233,17 @@ class SparkClientImpl implements SparkCl
allProps.put(DRIVER_OPTS_KEY, driverJavaOpts);
allProps.put(EXECUTOR_OPTS_KEY, executorJavaOpts);
+ String hiveHadoopTestClasspath = Strings.nullToEmpty(System.getenv("HIVE_HADOOP_TEST_CLASSPATH"));
+ if (!hiveHadoopTestClasspath.isEmpty()) {
+ String extraClasspath = Strings.nullToEmpty((String)allProps.get(DRIVER_EXTRA_CLASSPATH));
+ if (extraClasspath.isEmpty()) {
+ allProps.put(DRIVER_EXTRA_CLASSPATH, hiveHadoopTestClasspath);
+ } else {
+ extraClasspath = extraClasspath.endsWith(File.pathSeparator) ? extraClasspath : extraClasspath + File.pathSeparator;
+ allProps.put(DRIVER_EXTRA_CLASSPATH, extraClasspath + hiveHadoopTestClasspath);
+ }
+ }
+
Writer writer = new OutputStreamWriter(new FileOutputStream(properties), Charsets.UTF_8);
try {
allProps.store(writer, "Spark Context configuration");