You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@drill.apache.org by "ASF GitHub Bot (JIRA)" <ji...@apache.org> on 2018/05/31 23:46:00 UTC
[jira] [Commented] (DRILL-4309) Make this option
store.hive.optimize_scan_with_native_readers=true default
[ https://issues.apache.org/jira/browse/DRILL-4309?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16497345#comment-16497345 ]
ASF GitHub Bot commented on DRILL-4309:
---------------------------------------
ilooner closed pull request #350: DRILL-4309: Make this option `store.hive.optimize_scan_with_native_re…
URL: https://github.com/apache/drill/pull/350
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
index 7ac189636f..7457478a92 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
@@ -115,21 +115,16 @@ public void pruneDataTypeSupport() throws Exception {
@Test
public void pruneDataTypeSupportNativeReaders() throws Exception {
- try {
- test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- final String query = "EXPLAIN PLAN FOR " +
- "SELECT * FROM hive.readtest_parquet WHERE tinyint_part = 64";
+ final String query = "EXPLAIN PLAN FOR " +
+ "SELECT * FROM hive.readtest_parquet WHERE tinyint_part = 64";
- final String plan = getPlanInString(query, OPTIQ_FORMAT);
+ final String plan = getPlanInString(query, OPTIQ_FORMAT);
- // Check and make sure that Filter is not present in the plan
- assertFalse(plan.contains("Filter"));
+ // Check and make sure that Filter is not present in the plan
+ assertFalse(plan.contains("Filter"));
- // Make sure the plan contains the Hive scan utilizing native parquet reader
- assertTrue(plan.contains("groupscan=[HiveDrillNativeParquetScan"));
- } finally {
- test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- }
+ // Make sure the plan contains the Hive scan utilizing native parquet reader
+ assertTrue(plan.contains("groupscan=[HiveDrillNativeParquetScan"));
}
@Test // DRILL-3579
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
index ce9996172c..d093d2e0a4 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
@@ -107,14 +107,9 @@ public void testHiveCountStar() throws Exception {
@Test
public void projectPushDownOnHiveParquetTable() throws Exception {
- try {
- test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- String query = "SELECT boolean_field, boolean_part, int_field, int_part FROM hive.readtest_parquet";
- String expectedColNames = "\"columns\" : [ \"`boolean_field`\", \"`dir0`\", \"`int_field`\", \"`dir9`\" ]";
-
- testHelper(query, 2, expectedColNames, "hive-drill-native-parquet-scan");
- } finally {
- test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- }
+ String query = "SELECT boolean_field, boolean_part, int_field, int_part FROM hive.readtest_parquet";
+ String expectedColNames = "\"columns\" : [ \"`boolean_field`\", \"`dir0`\", \"`int_field`\", \"`dir9`\" ]";
+
+ testHelper(query, 2, expectedColNames, "hive-drill-native-parquet-scan");
}
}
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index c2e367da67..659293ab36 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -41,16 +41,8 @@ public static void setupOptions() throws Exception {
@Test // DRILL-4083
public void testNativeScanWhenNoColumnIsRead() throws Exception {
- try {
- test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
-
- String query = "SELECT count(*) FROM hive.readtest_parquet";
- testPhysicalPlan(query, "hive-drill-native-parquet-scan");
- } finally {
- test(String.format("alter session set `%s` = %s",
- ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS,
- ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR.getDefault().bool_val ? "true" : "false"));
- }
+ String query = "SELECT count(*) FROM hive.readtest_parquet";
+ testPhysicalPlan(query, "hive-drill-native-parquet-scan");
}
@Test
@@ -188,109 +180,104 @@ public void readAllSupportedHiveDataTypes() throws Exception {
*/
@Test
public void readAllSupportedHiveDataTypesNativeParquet() throws Exception {
- try {
- test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- final String query = "SELECT * FROM hive.readtest_parquet";
+ final String query = "SELECT * FROM hive.readtest_parquet";
- // Make sure the plan has Hive scan with native parquet reader
- testPhysicalPlan(query, "hive-drill-native-parquet-scan");
+ // Make sure the plan has Hive scan with native parquet reader
+ testPhysicalPlan(query, "hive-drill-native-parquet-scan");
- testBuilder().sqlQuery(query)
- .unOrdered()
- .baselineColumns(
- "binary_field",
- "boolean_field",
- "tinyint_field",
- "decimal0_field",
- "decimal9_field",
- "decimal18_field",
- "decimal28_field",
- "decimal38_field",
- "double_field",
- "float_field",
- "int_field",
- "bigint_field",
- "smallint_field",
- "string_field",
- "varchar_field",
- "timestamp_field",
- // There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
- //"binary_part",
- "boolean_part",
- "tinyint_part",
- "decimal0_part",
- "decimal9_part",
- "decimal18_part",
- "decimal28_part",
- "decimal38_part",
- "double_part",
- "float_part",
- "int_part",
- "bigint_part",
- "smallint_part",
- "string_part",
- "varchar_part",
- "timestamp_part",
- "date_part")
- .baselineValues(
- "binaryfield",
- false,
- 34,
- new BigDecimal("66"),
- new BigDecimal("2347.92"),
- new BigDecimal("2758725827.99990"),
- new BigDecimal("29375892739852.8"),
- new BigDecimal("89853749534593985.783"),
- 8.345d,
- 4.67f,
- 123456,
- 234235L,
- 3455,
- "stringfield",
- "varcharfield",
- new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
- // There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
- //"binary",
- true,
- 64,
- new BigDecimal("37"),
- new BigDecimal("36.90"),
- new BigDecimal("3289379872.94565"),
- new BigDecimal("39579334534534.4"),
- new BigDecimal("363945093845093890.900"),
- 8.345d,
- 4.67f,
- 123456,
- 234235L,
- 3455,
- "string",
- "varchar",
- new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
- new DateTime(Date.valueOf("2013-07-05").getTime()))
- .baselineValues( // All fields are null, but partition fields have non-null values
- null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
- // There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
- //"binary",
- true,
- 64,
- new BigDecimal("37"),
- new BigDecimal("36.90"),
- new BigDecimal("3289379872.94565"),
- new BigDecimal("39579334534534.4"),
- new BigDecimal("363945093845093890.900"),
- 8.345d,
- 4.67f,
- 123456,
- 234235L,
- 3455,
- "string",
- "varchar",
- new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
- new DateTime(Date.valueOf("2013-07-05").getTime()))
- .go();
- } finally {
- test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- }
+ testBuilder().sqlQuery(query)
+ .unOrdered()
+ .baselineColumns(
+ "binary_field",
+ "boolean_field",
+ "tinyint_field",
+ "decimal0_field",
+ "decimal9_field",
+ "decimal18_field",
+ "decimal28_field",
+ "decimal38_field",
+ "double_field",
+ "float_field",
+ "int_field",
+ "bigint_field",
+ "smallint_field",
+ "string_field",
+ "varchar_field",
+ "timestamp_field",
+ // There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
+ //"binary_part",
+ "boolean_part",
+ "tinyint_part",
+ "decimal0_part",
+ "decimal9_part",
+ "decimal18_part",
+ "decimal28_part",
+ "decimal38_part",
+ "double_part",
+ "float_part",
+ "int_part",
+ "bigint_part",
+ "smallint_part",
+ "string_part",
+ "varchar_part",
+ "timestamp_part",
+ "date_part")
+ .baselineValues(
+ "binaryfield",
+ false,
+ 34,
+ new BigDecimal("66"),
+ new BigDecimal("2347.92"),
+ new BigDecimal("2758725827.99990"),
+ new BigDecimal("29375892739852.8"),
+ new BigDecimal("89853749534593985.783"),
+ 8.345d,
+ 4.67f,
+ 123456,
+ 234235L,
+ 3455,
+ "stringfield",
+ "varcharfield",
+ new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
+ // There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
+ //"binary",
+ true,
+ 64,
+ new BigDecimal("37"),
+ new BigDecimal("36.90"),
+ new BigDecimal("3289379872.94565"),
+ new BigDecimal("39579334534534.4"),
+ new BigDecimal("363945093845093890.900"),
+ 8.345d,
+ 4.67f,
+ 123456,
+ 234235L,
+ 3455,
+ "string",
+ "varchar",
+ new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
+ new DateTime(Date.valueOf("2013-07-05").getTime()))
+ .baselineValues( // All fields are null, but partition fields have non-null values
+ null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ // There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
+ //"binary",
+ true,
+ 64,
+ new BigDecimal("37"),
+ new BigDecimal("36.90"),
+ new BigDecimal("3289379872.94565"),
+ new BigDecimal("39579334534534.4"),
+ new BigDecimal("363945093845093890.900"),
+ 8.345d,
+ 4.67f,
+ 123456,
+ 234235L,
+ 3455,
+ "string",
+ "varchar",
+ new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
+ new DateTime(Date.valueOf("2013-07-05").getTime()))
+ .go();
}
@Test
@@ -364,16 +351,11 @@ public void readFromAlteredPartitionedTable() throws Exception {
@Test // DRILL-3938
public void nativeReaderIsDisabledForAlteredPartitionedTable() throws Exception {
- try {
- test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- final String query = "EXPLAIN PLAN FOR SELECT key, `value`, newcol FROM hive.kv_parquet ORDER BY key LIMIT 1";
+ final String query = "EXPLAIN PLAN FOR SELECT key, `value`, newcol FROM hive.kv_parquet ORDER BY key LIMIT 1";
- // Make sure the HiveScan in plan has no native parquet reader
- final String planStr = getPlanInString(query, JSON_FORMAT);
- assertFalse("Hive native is not expected in the plan", planStr.contains("hive-drill-native-parquet-scan"));
- } finally {
- test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- }
+ // Make sure the HiveScan in plan has no native parquet reader
+ final String planStr = getPlanInString(query, JSON_FORMAT);
+ assertFalse("Hive native is not expected in the plan", planStr.contains("hive-drill-native-parquet-scan"));
}
@Test // DRILL-3739
@@ -388,18 +370,12 @@ public void readingFromStorageHandleBasedTable() throws Exception {
@Test // DRILL-3739
public void readingFromStorageHandleBasedTable2() throws Exception {
- try {
- test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
-
- testBuilder()
- .sqlQuery("SELECT * FROM hive.kv_sh ORDER BY key LIMIT 2")
- .ordered()
- .baselineColumns("key", "value")
- .expectsEmptyResultSet()
- .go();
- } finally {
- test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
- }
+ testBuilder()
+ .sqlQuery("SELECT * FROM hive.kv_sh ORDER BY key LIMIT 2")
+ .ordered()
+ .baselineColumns("key", "value")
+ .expectsEmptyResultSet()
+ .go();
}
@AfterClass
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index c5a506339a..6dd811a7f5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -163,7 +163,7 @@
// in core which is not right. Move this option and above two mongo plugin related options once we have the feature.
String HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS = "store.hive.optimize_scan_with_native_readers";
OptionValidator HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR =
- new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS, false);
+ new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS, true);
String SLICE_TARGET = "planner.slice_target";
long SLICE_TARGET_DEFAULT = 100000l;
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
> Make this option store.hive.optimize_scan_with_native_readers=true default
> --------------------------------------------------------------------------
>
> Key: DRILL-4309
> URL: https://issues.apache.org/jira/browse/DRILL-4309
> Project: Apache Drill
> Issue Type: Improvement
> Components: Query Planning & Optimization
> Affects Versions: 1.8.0
> Reporter: Sean Hsuan-Yi Chu
> Assignee: Vitalii Diravka
> Priority: Major
> Labels: doc-impacting
> Fix For: Future
>
>
> This new feature has been around and used/tests in many scenarios.
> We should enable this feature by default.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)