You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/10/30 17:34:17 UTC
svn commit: r1635538 - in /hive/branches/spark: hcatalog/webhcat/svr/
itests/qtest-spark/ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/...
Author: brock
Date: Thu Oct 30 16:34:17 2014
New Revision: 1635538
URL: http://svn.apache.org/r1635538
Log:
HIVE-8502 - Merge trunk into spark 10/17/14 (post merge fixes)
Added:
hive/branches/spark/hcatalog/webhcat/svr/.gitignore
Modified:
hive/branches/spark/itests/qtest-spark/pom.xml
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
hive/branches/spark/shims/0.23/pom.xml
hive/branches/spark/spark-client/pom.xml
Added: hive/branches/spark/hcatalog/webhcat/svr/.gitignore
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/.gitignore?rev=1635538&view=auto
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/svr/.gitignore (added)
+++ hive/branches/spark/hcatalog/webhcat/svr/.gitignore Thu Oct 30 16:34:17 2014
@@ -0,0 +1 @@
+dependency-reduced-pom.xml
Modified: hive/branches/spark/itests/qtest-spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/qtest-spark/pom.xml?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/itests/qtest-spark/pom.xml (original)
+++ hive/branches/spark/itests/qtest-spark/pom.xml Thu Oct 30 16:34:17 2014
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java Thu Oct 30 16:34:17 2014
@@ -197,7 +197,7 @@ public class SparkMapRecordHandler exten
logCloseInfo();
}
- ReportStats rps = new ReportStats(rp);
+ ReportStats rps = new ReportStats(rp, jc);
mo.preorderMap(rps);
return;
} catch (Exception e) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java Thu Oct 30 16:34:17 2014
@@ -410,7 +410,7 @@ public class SparkReduceRecordHandler ex
}
reducer.close(abort);
- ReportStats rps = new ReportStats(rp);
+ ReportStats rps = new ReportStats(rp, jc);
reducer.preorderMap(rps);
} catch (Exception e) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java Thu Oct 30 16:34:17 2014
@@ -25,9 +25,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapOperator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ScriptOperator;
import org.apache.hadoop.mapreduce.util.ResourceBundles;
@@ -64,11 +66,18 @@ public class SparkCounters implements Se
* pre-define all needed Counters here.
*/
private void initializeSparkCounters() {
- createCounter(HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVECOUNTERGROUP),
- Operator.HIVECOUNTERCREATEDFILES);
+ String groupName = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVECOUNTERGROUP);
+ createCounter(groupName, Operator.HIVECOUNTERCREATEDFILES);
+ createCounter(groupName, MapOperator.Counter.DESERIALIZE_ERRORS);
+ createCounter(groupName, MapOperator.Counter.RECORDS_IN);
+ createCounter(groupName, FileSinkOperator.Counter.RECORDS_OUT);
+ createCounter(groupName, ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE);
+ createCounter(groupName, ScriptOperator.Counter.DESERIALIZE_ERRORS);
+ createCounter(groupName, ScriptOperator.Counter.SERIALIZE_ERRORS);
+ createCounter(groupName, JoinOperator.SkewkeyTableCounter.SKEWJOINFOLLOWUPJOBS);
+ // TODO remove? changed due to HIVE-8429
createCounter(MapOperator.Counter.DESERIALIZE_ERRORS);
- createCounter(FilterOperator.Counter.FILTERED);
- createCounter(FilterOperator.Counter.PASSED);
+ createCounter(MapOperator.Counter.RECORDS_IN);
createCounter(ScriptOperator.Counter.DESERIALIZE_ERRORS);
createCounter(ScriptOperator.Counter.SERIALIZE_ERRORS);
createCounter(JoinOperator.SkewkeyTableCounter.SKEWJOINFOLLOWUPJOBS);
@@ -78,6 +87,10 @@ public class SparkCounters implements Se
createCounter(key.getDeclaringClass().getName(), key.name());
}
+ public void createCounter(String groupName, Enum<?> key) {
+ createCounter(groupName, key.name(), 0L);
+ }
+
public void createCounter(String groupName, String counterName) {
createCounter(groupName, counterName, 0L);
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java Thu Oct 30 16:34:17 2014
@@ -85,7 +85,6 @@ public class SetSparkReducerParallelism
maxReducers, false);
LOG.info("Set parallelism for reduce sink " + sink + " to: " + numReducers);
desc.setNumReducers(numReducers);
- desc.setAutoParallel(true);
}
} else {
LOG.info("Number of reducers determined to be: " + desc.getNumReducers());
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Thu Oct 30 16:34:17 2014
@@ -140,11 +140,6 @@ public abstract class BaseWork extends A
return returnSet;
}
- public Map<String, Map<Integer, String>> getScratchColumnVectorTypes() {
- return scratchColumnVectorTypes;
-
- }
-
public Map<String, Map<Integer, String>> getAllScratchColumnVectorTypeMaps() {
return allScratchColumnVectorTypeMaps;
}
Modified: hive/branches/spark/shims/0.23/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/0.23/pom.xml?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/shims/0.23/pom.xml (original)
+++ hive/branches/spark/shims/0.23/pom.xml Thu Oct 30 16:34:17 2014
@@ -123,6 +123,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<version>${hadoop-23.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
Modified: hive/branches/spark/spark-client/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/pom.xml?rev=1635538&r1=1635537&r2=1635538&view=diff
==============================================================================
--- hive/branches/spark/spark-client/pom.xml (original)
+++ hive/branches/spark/spark-client/pom.xml Thu Oct 30 16:34:17 2014
@@ -22,14 +22,14 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.hive</groupId>
<artifactId>spark-client</artifactId>
<packaging>jar</packaging>
<name>Spark Remote Client</name>
- <version>0.14.0-SNAPSHOT</version>
+ <version>0.15.0-SNAPSHOT</version>
<properties>
<hive.path.to.root>..</hive.path.to.root>