You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/01/09 00:24:01 UTC
svn commit: r1650417 - in /hive/branches/spark/ql/src:
java/org/apache/hadoop/hive/ql/Context.java
test/results/clientnegative/fs_default_name2.q.out
test/results/clientpositive/udf_example_add.q.out
Author: xuefu
Date: Thu Jan 8 23:24:00 2015
New Revision: 1650417
URL: http://svn.apache.org/r1650417
Log:
HIVE-9306: Let Context.isLocalOnlyExecutionMode() return false if execution engine is Spark [Spark Branch] (reviewed by szehon)
Added:
hive/branches/spark/ql/src/test/results/clientpositive/udf_example_add.q.out
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Context.java
hive/branches/spark/ql/src/test/results/clientnegative/fs_default_name2.q.out
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1650417&r1=1650416&r2=1650417&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Context.java Thu Jan 8 23:24:00 2015
@@ -539,12 +539,14 @@ public class Context {
* Today this translates into running hadoop jobs locally
*/
public boolean isLocalOnlyExecutionMode() {
- // TODO: hack to allow spark to run in a cluster mode. Without this, depending on
- // user's local hadoop settings, true may be returned, which causes plan to be
- // stored in local path. This will be addressed in subsquent patches, where false is
- // returned for spark always.
- return false;
-// return ShimLoader.getHadoopShims().isLocalMode(conf);
+ // Always allow spark to run in a cluster mode. Without this, depending on
+ // user's local hadoop settings, true may be returned, which causes plan to be
+ // stored in local path.
+ if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
+ return false;
+ }
+
+ return ShimLoader.getHadoopShims().isLocalMode(conf);
}
public List<HiveLock> getHiveLocks() {
Modified: hive/branches/spark/ql/src/test/results/clientnegative/fs_default_name2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientnegative/fs_default_name2.q.out?rev=1650417&r1=1650416&r2=1650417&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientnegative/fs_default_name2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientnegative/fs_default_name2.q.out Thu Jan 8 23:24:00 2015
@@ -1 +1 @@
-FAILED: RuntimeException Error while making MR scratch directory - check filesystem config (java.net.URISyntaxException: Illegal character in scheme name at index 0: 'http://www.example.com)
+FAILED: IllegalArgumentException Illegal character in scheme name at index 0: 'http://www.example.com
Added: hive/branches/spark/ql/src/test/results/clientpositive/udf_example_add.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/udf_example_add.q.out?rev=1650417&view=auto
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/udf_example_add.q.out (added)
+++ hive/branches/spark/ql/src/test/results/clientpositive/udf_example_add.q.out Thu Jan 8 23:24:00 2015
@@ -0,0 +1,75 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION example_add AS 'org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: example_add
+POSTHOOK: query: CREATE TEMPORARY FUNCTION example_add AS 'org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: example_add
+PREHOOK: query: EXPLAIN
+SELECT example_add(1, 2),
+ example_add(1, 2, 3),
+ example_add(1, 2, 3, 4),
+ example_add(1.1, 2.2),
+ example_add(1.1, 2.2, 3.3),
+ example_add(1.1, 2.2, 3.3, 4.4),
+ example_add(1, 2, 3, 4.4)
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT example_add(1, 2),
+ example_add(1, 2, 3),
+ example_add(1, 2, 3, 4),
+ example_add(1.1, 2.2),
+ example_add(1.1, 2.2, 3.3),
+ example_add(1.1, 2.2, 3.3, 4.4),
+ example_add(1, 2, 3, 4.4)
+FROM src LIMIT 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 3 (type: int), 6 (type: int), 10 (type: int), 3.3000000000000003 (type: double), 6.6 (type: double), 11.0 (type: double), 10.4 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE
+ ListSink
+
+PREHOOK: query: SELECT example_add(1, 2),
+ example_add(1, 2, 3),
+ example_add(1, 2, 3, 4),
+ example_add(1.1, 2.2),
+ example_add(1.1, 2.2, 3.3),
+ example_add(1.1, 2.2, 3.3, 4.4),
+ example_add(1, 2, 3, 4.4)
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT example_add(1, 2),
+ example_add(1, 2, 3),
+ example_add(1, 2, 3, 4),
+ example_add(1.1, 2.2),
+ example_add(1.1, 2.2, 3.3),
+ example_add(1.1, 2.2, 3.3, 4.4),
+ example_add(1, 2, 3, 4.4)
+FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+3 6 10 3.3000000000000003 6.6 11.0 10.4
+PREHOOK: query: DROP TEMPORARY FUNCTION example_add
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: example_add
+POSTHOOK: query: DROP TEMPORARY FUNCTION example_add
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: example_add