You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2014/07/23 17:11:54 UTC
svn commit: r1612847 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/conf/
itests/hive-unit/src/test/java/org/apache/hive/jdbc/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/test/queries/clientpositive/ ql/src/test/r...
Author: gopalv
Date: Wed Jul 23 15:11:53 2014
New Revision: 1612847
URL: http://svn.apache.org/r1612847
Log:
HIVE-7397 : Set the simple fetch task conversion to more with a default threshold of 1Gb
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/data/conf/hive-site.xml
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q
hive/trunk/ql/src/test/results/clientnegative/authorization_fail_8.q.out
hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/bucket2.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/bucket3.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/bucket4.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Jul 23 15:11:53 2014
@@ -1261,14 +1261,14 @@ public class HiveConf extends Configurat
HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true, ""),
- HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "minimal", new StringSet("minimal", "more"),
+ HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "more", new StringSet("minimal", "more"),
"Some select queries can be converted to single FETCH task minimizing latency.\n" +
"Currently the query should be single sourced not having any subquery and should not have\n" +
"any aggregations or distincts (which incurs RS), lateral views and joins.\n" +
"1. minimal : SELECT STAR, FILTER on partition columns, LIMIT only\n" +
"2. more : SELECT, FILTER, LIMIT only (support TABLESAMPLE and virtual columns)\n"
),
- HIVEFETCHTASKCONVERSIONTHRESHOLD("hive.fetch.task.conversion.threshold", -1l,
+ HIVEFETCHTASKCONVERSIONTHRESHOLD("hive.fetch.task.conversion.threshold", 1073741824L,
"Input threshold for applying hive.fetch.task.conversion. If target table is native, input length\n" +
"is calculated by summation of file lengths. If it's not native, storage handler for the table\n" +
"can optionally implement org.apache.hadoop.hive.ql.metadata.InputEstimator interface."),
Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Wed Jul 23 15:11:53 2014
@@ -1,7 +1,5 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
@@ -16,8 +14,7 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--->
-<configuration>
+--><configuration>
<!-- WARNING!!! This file is auto generated for documentation purposes ONLY! -->
<!-- WARNING!!! Any changes you make to this file will be ignored by Hive. -->
<!-- WARNING!!! You must make your changes in hive-site.xml instead. -->
@@ -2224,7 +2221,7 @@
</property>
<property>
<key>hive.fetch.task.conversion</key>
- <value>minimal</value>
+ <value>more</value>
<description>
Some select queries can be converted to single FETCH task minimizing latency.
Currently the query should be single sourced not having any subquery and should not have
@@ -2235,7 +2232,7 @@
</property>
<property>
<key>hive.fetch.task.conversion.threshold</key>
- <value>-1</value>
+ <value>1073741824</value>
<description>
Input threshold for applying hive.fetch.task.conversion. If target table is native, input length
is calculated by summation of file lengths. If it's not native, storage handler for the table
Modified: hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/hive-site.xml?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/data/conf/hive-site.xml (original)
+++ hive/trunk/data/conf/hive-site.xml Wed Jul 23 15:11:53 2014
@@ -235,5 +235,9 @@
</property>
+<property>
+ <name>hive.fetch.task.conversion</name>
+ <value>minimal</value>
+</property>
</configuration>
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Wed Jul 23 15:11:53 2014
@@ -275,22 +275,22 @@ public class TestJdbcDriver2 {
Statement s = this.con.createStatement();
ResultSet rs = s.executeQuery("SELECT * FROM " + dataTypeTableName);
- rs.close();
- s.close();
-
assertTrue(s.getConnection() == this.con);
assertTrue(rs.getStatement() == s);
+ rs.close();
+ s.close();
+
/* Test parent references from PreparedStatement */
PreparedStatement ps = this.con.prepareStatement("SELECT * FROM " + dataTypeTableName);
rs = ps.executeQuery();
- rs.close();
- ps.close();
-
assertTrue(ps.getConnection() == this.con);
assertTrue(rs.getStatement() == ps);
+ rs.close();
+ ps.close();
+
/* Test DatabaseMetaData queries which do not have a parent Statement */
DatabaseMetaData md = this.con.getMetaData();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java Wed Jul 23 15:11:53 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
@@ -27,6 +28,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FetchTask;
@@ -106,9 +108,9 @@ public class SimpleFetchOptimizer implem
pctx.getConf(), HiveConf.ConfVars.HIVEFETCHTASKCONVERSION);
boolean aggressive = "more".equals(mode);
+ final int limit = pctx.getQB().getParseInfo().getOuterQueryLimit();
FetchData fetch = checkTree(aggressive, pctx, alias, source);
- if (fetch != null && checkThreshold(fetch, pctx)) {
- int limit = pctx.getQB().getParseInfo().getOuterQueryLimit();
+ if (fetch != null && checkThreshold(fetch, limit, pctx)) {
FetchWork fetchWork = fetch.convertToWork();
FetchTask fetchTask = (FetchTask) TaskFactory.get(fetchWork, pctx.getConf());
fetchWork.setSink(fetch.completed(pctx, fetchWork));
@@ -119,7 +121,10 @@ public class SimpleFetchOptimizer implem
return null;
}
- private boolean checkThreshold(FetchData data, ParseContext pctx) throws Exception {
+ private boolean checkThreshold(FetchData data, int limit, ParseContext pctx) throws Exception {
+ if (limit > 0 && data.hasOnlyPruningFilter()) {
+ return true;
+ }
long threshold = HiveConf.getLongVar(pctx.getConf(),
HiveConf.ConfVars.HIVEFETCHTASKCONVERSIONTHRESHOLD);
if (threshold < 0) {
@@ -169,7 +174,7 @@ public class SimpleFetchOptimizer implem
PrunedPartitionList pruned = pctx.getPrunedPartitions(alias, ts);
if (aggressive || !pruned.hasUnknownPartitions()) {
bypassFilter &= !pruned.hasUnknownPartitions();
- return checkOperators(new FetchData(parent, table, pruned, splitSample), ts,
+ return checkOperators(new FetchData(parent, table, pruned, splitSample, bypassFilter), ts,
aggressive, bypassFilter);
}
}
@@ -211,6 +216,7 @@ public class SimpleFetchOptimizer implem
private final SplitSample splitSample;
private final PrunedPartitionList partsList;
private final HashSet<ReadEntity> inputs = new HashSet<ReadEntity>();
+ private final boolean onlyPruningFilter;
// source table scan
private TableScanOperator scanOp;
@@ -223,14 +229,23 @@ public class SimpleFetchOptimizer implem
this.table = table;
this.partsList = null;
this.splitSample = splitSample;
+ this.onlyPruningFilter = false;
}
private FetchData(ReadEntity parent, Table table, PrunedPartitionList partsList,
- SplitSample splitSample) {
+ SplitSample splitSample, boolean bypassFilter) {
this.parent = parent;
this.table = table;
this.partsList = partsList;
this.splitSample = splitSample;
+ this.onlyPruningFilter = bypassFilter;
+ }
+
+ /*
+ * all filters were executed during partition pruning
+ */
+ public boolean hasOnlyPruningFilter() {
+ return this.onlyPruningFilter;
}
private FetchWork convertToWork() throws HiveException {
@@ -317,7 +332,12 @@ public class SimpleFetchOptimizer implem
InputFormat input = HiveInputFormat.getInputFormatFromCache(clazz, conf);
summary = ((ContentSummaryInputFormat)input).getContentSummary(path, conf);
} else {
- summary = path.getFileSystem(conf).getContentSummary(path);
+ FileSystem fs = path.getFileSystem(conf);
+ try {
+ summary = fs.getContentSummary(path);
+ } catch (FileNotFoundException e) {
+ return 0;
+ }
}
return summary.getLength();
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q Wed Jul 23 15:11:53 2014
@@ -5,5 +5,6 @@ explain select cast(key as int) * 10, up
set hive.fetch.task.conversion.threshold=100;
+-- from HIVE-7397, limit + partition pruning filter
explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10;
explain select cast(key as int) * 10, upper(value) from src limit 10;
Modified: hive/trunk/ql/src/test/results/clientnegative/authorization_fail_8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/authorization_fail_8.q.out?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/authorization_fail_8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/authorization_fail_8.q.out Wed Jul 23 15:11:53 2014
@@ -8,4 +8,41 @@ POSTHOOK: Output: default@authorization_
PREHOOK: query: GRANT SELECT ON authorization_fail TO USER user2 WITH GRANT OPTION
PREHOOK: type: GRANT_PRIVILEGE
PREHOOK: Output: default@authorization_fail
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. null
+POSTHOOK: query: GRANT SELECT ON authorization_fail TO USER user2 WITH GRANT OPTION
+POSTHOOK: type: GRANT_PRIVILEGE
+POSTHOOK: Output: default@authorization_fail
+PREHOOK: query: SHOW GRANT USER user2 ON TABLE authorization_fail
+PREHOOK: type: SHOW_GRANT
+POSTHOOK: query: SHOW GRANT USER user2 ON TABLE authorization_fail
+POSTHOOK: type: SHOW_GRANT
+default authorization_fail user2 USER SELECT true -1 user1
+PREHOOK: query: -- user2 current has grant option, this should work
+GRANT SELECT ON authorization_fail TO USER user3
+PREHOOK: type: GRANT_PRIVILEGE
+PREHOOK: Output: default@authorization_fail
+POSTHOOK: query: -- user2 current has grant option, this should work
+GRANT SELECT ON authorization_fail TO USER user3
+POSTHOOK: type: GRANT_PRIVILEGE
+POSTHOOK: Output: default@authorization_fail
+PREHOOK: query: REVOKE SELECT ON authorization_fail FROM USER user3
+PREHOOK: type: REVOKE_PRIVILEGE
+PREHOOK: Output: default@authorization_fail
+POSTHOOK: query: REVOKE SELECT ON authorization_fail FROM USER user3
+POSTHOOK: type: REVOKE_PRIVILEGE
+POSTHOOK: Output: default@authorization_fail
+PREHOOK: query: REVOKE GRANT OPTION FOR SELECT ON authorization_fail FROM USER user2
+PREHOOK: type: REVOKE_PRIVILEGE
+PREHOOK: Output: default@authorization_fail
+POSTHOOK: query: REVOKE GRANT OPTION FOR SELECT ON authorization_fail FROM USER user2
+POSTHOOK: type: REVOKE_PRIVILEGE
+POSTHOOK: Output: default@authorization_fail
+PREHOOK: query: SHOW GRANT USER user2 ON TABLE authorization_fail
+PREHOOK: type: SHOW_GRANT
+POSTHOOK: query: SHOW GRANT USER user2 ON TABLE authorization_fail
+POSTHOOK: type: SHOW_GRANT
+default authorization_fail user2 USER SELECT false -1 user1
+PREHOOK: query: -- Now that grant option has been revoked, granting to other users should fail
+GRANT SELECT ON authorization_fail TO USER user3
+PREHOOK: type: GRANT_PRIVILEGE
+PREHOOK: Output: default@authorization_fail
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Permission denied. Principal [name=user2, type=USER] does not have following privileges on Object [type=TABLE_OR_VIEW, name=default.authorization_fail] for operation GRANT_PRIVILEGE : [SELECT with grant]
Modified: hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out Wed Jul 23 15:11:53 2014
@@ -46,41 +46,31 @@ STAGE PLANS:
Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
ListSink
-PREHOOK: query: explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10
+PREHOOK: query: -- from HIVE-7397, limit + partition pruning filter
+explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10
PREHOOK: type: QUERY
-POSTHOOK: query: explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10
+POSTHOOK: query: -- from HIVE-7397, limit + partition pruning filter
+explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: srcpart
- Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string), ds (type: string), hr (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
limit: 10
Processor Tree:
- ListSink
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string), ds (type: string), hr (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE
+ ListSink
PREHOOK: query: explain select cast(key as int) * 10, upper(value) from src limit 10
PREHOOK: type: QUERY
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/bucket2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/bucket2.q.out?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/tez/bucket2.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/tez/bucket2.q.out Wed Jul 23 15:11:53 2014 differ
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/bucket3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/bucket3.q.out?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/tez/bucket3.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/tez/bucket3.q.out Wed Jul 23 15:11:53 2014 differ
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/bucket4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/bucket4.q.out?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/tez/bucket4.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/tez/bucket4.q.out Wed Jul 23 15:11:53 2014 differ
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out?rev=1612847&r1=1612846&r2=1612847&view=diff
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out Wed Jul 23 15:11:53 2014 differ