You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/10/03 23:45:00 UTC
[3/5] hive git commit: HIVE-14830. Move a majority of the
MiniLlapCliDriver tests to use an inline AM. (Siddharth Seth,
reviewed by Prasanth Jayachandran)
HIVE-14830. Move a majority of the MiniLlapCliDriver tests to use an inline AM. (Siddharth Seth, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0a4b3d8f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0a4b3d8f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0a4b3d8f
Branch: refs/heads/hive-14535
Commit: 0a4b3d8ff673f6f6670293a7491873c229cb0f40
Parents: b86342f
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Oct 3 13:05:34 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Oct 3 13:05:34 2016 -0700
----------------------------------------------------------------------
.../hive/cli/TestMiniLlapLocalCliDriver.java | 62 ++++++++++
.../test/resources/testconfiguration.properties | 117 ++++++++++---------
.../hadoop/hive/cli/control/CliConfigs.java | 29 +++++
.../hive/ql/optimizer/physical/LlapDecider.java | 1 +
4 files changed, 152 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
new file mode 100644
index 0000000..5b1caf6
--- /dev/null
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli;
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.hadoop.hive.cli.control.CliAdapter;
+import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestMiniLlapLocalCliDriver {
+
+ static CliAdapter adapter = new CliConfigs.MiniLlapLocalCliConfig().getCliAdapter();
+
+ @Parameters(name = "{0}")
+ public static List<Object[]> getParameters() throws Exception {
+ return adapter.getParameters();
+ }
+
+ @ClassRule
+ public static TestRule cliClassRule = adapter.buildClassRule();
+
+ @Rule
+ public TestRule cliTestRule = adapter.buildTestRule();
+
+ private String name;
+ private File qfile;
+
+ public TestMiniLlapLocalCliDriver(String name, File qfile) {
+ this.name = name;
+ this.qfile = qfile;
+ }
+
+ @Test
+ public void testCliDriver() throws Exception {
+ adapter.runTest(name, qfile);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index e810a58..fbba0cd 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -54,8 +54,40 @@ minitez.query.files=explainuser_3.q,\
stats_filemetadata.q,\
tez_union_with_udf.q
+
minillap.shared.query.files=acid_globallimit.q,\
- alter_merge_2_orc.q,\
+ insert_into1.q,\
+ insert_into2.q,\
+ insert_values_orig_table.,\
+ llapdecider.q,\
+ mapreduce1.q,\
+ mapreduce2.q,\
+ orc_merge1.q,\
+ orc_merge10.q,\
+ orc_merge2.q,\
+ orc_merge3.q,\
+ orc_merge4.q,\
+ orc_merge_diff_fs.q,\
+ script_env_var1.q,\
+ script_env_var2.q,\
+ script_pipe.q,\
+ tez_union_view.q,\
+ transform1.q,\
+ transform2.q,\
+ transform_ppr1.q,\
+ transform_ppr2.q,\
+ unionDistinct_1.q,\
+ union_type_chk.q,\
+ orc_ppd_basic.q,\
+ orc_ppd_schema_evol_3a.q,\
+ cte_2.q,\
+ cte_4.q,\
+ llap_nullscan.q,\
+ dynamic_partition_pruning_2.q,\
+ tez_union_dynamic_partition.q,\
+ union_fast_stats.q
+
+minillaplocal.shared.query.files=alter_merge_2_orc.q,\
alter_merge_orc.q,\
alter_merge_stats_orc.q,\
auto_join0.q,\
@@ -111,9 +143,7 @@ minillap.shared.query.files=acid_globallimit.q,\
cross_product_check_2.q,\
ctas.q,\
cte_1.q,\
- cte_2.q,\
cte_3.q,\
- cte_4.q,\
cte_5.q,\
cte_mat_1.q,\
cte_mat_2.q,\
@@ -131,7 +161,6 @@ minillap.shared.query.files=acid_globallimit.q,\
delete_whole_partition.q,\
disable_merge_for_bucketing.q,\
dynamic_partition_pruning.q,\
- dynamic_partition_pruning_2.q,\
dynpart_sort_opt_vectorization.q,\
dynpart_sort_optimization.q,\
dynpart_sort_optimization2.q,\
@@ -145,13 +174,10 @@ minillap.shared.query.files=acid_globallimit.q,\
having.q,\
identity_project_remove_skip.q,\
insert1.q,\
- insert_into1.q,\
- insert_into2.q,\
insert_orig_table.q,\
insert_update_delete.q,\
insert_values_dynamic_partitioned.q,\
insert_values_non_partitioned.q,\
- insert_values_orig_table.,\
insert_values_partitioned.q,\
insert_values_tmp_table.q,\
join0.q,\
@@ -159,8 +185,6 @@ minillap.shared.query.files=acid_globallimit.q,\
join_nullsafe.q,\
leftsemijoin.q,\
limit_pushdown.q,\
- llap_nullscan.q,\
- llapdecider.q,\
load_dyn_part1.q,\
load_dyn_part2.q,\
load_dyn_part3.q,\
@@ -168,8 +192,6 @@ minillap.shared.query.files=acid_globallimit.q,\
mapjoin2.q,\
mapjoin_decimal.q,\
mapjoin_mapjoin.q,\
- mapreduce1.q,\
- mapreduce2.q,\
merge1.q,\
merge2.q,\
mergejoin.q,\
@@ -180,36 +202,25 @@ minillap.shared.query.files=acid_globallimit.q,\
nonmr_fetch_threshold.q,\
optimize_nullscan.q,\
orc_analyze.q,\
- orc_merge1.q,\
- orc_merge10.q,\
orc_merge11.q,\
- orc_merge2.q,\
- orc_merge3.q,\
- orc_merge4.q,\
orc_merge5.q,\
orc_merge6.q,\
orc_merge7.q,\
orc_merge8.q,\
orc_merge9.q,\
- orc_merge_diff_fs.q,\
orc_merge_incompat1.q,\
orc_merge_incompat2.q,\
orc_merge_incompat3.q,\
- orc_ppd_basic.q,\
orc_ppd_schema_evol_1a.q,\
orc_ppd_schema_evol_1b.q,\
orc_ppd_schema_evol_2a.q,\
orc_ppd_schema_evol_2b.q,\
- orc_ppd_schema_evol_3a.q,\
order_null.q,\
parallel.q,\
ptf.q,\
ptf_matchpath.q,\
ptf_streaming.q,\
sample1.q,\
- script_env_var1.q,\
- script_env_var2.q,\
- script_pipe.q,\
selectDistinctStar.q,\
select_dummy_source.q,\
skewjoin.q,\
@@ -237,16 +248,10 @@ minillap.shared.query.files=acid_globallimit.q,\
tez_union.q,\
tez_union2.q,\
tez_union_decimal.q,\
- tez_union_dynamic_partition.q,\
tez_union_group_by.q,\
tez_union_multiinsert.q,\
- tez_union_view.q,\
tez_vector_dynpart_hashjoin_1.q,\
tez_vector_dynpart_hashjoin_2.q,\
- transform1.q,\
- transform2.q,\
- transform_ppr1.q,\
- transform_ppr2.q,\
union2.q,\
union3.q,\
union4.q,\
@@ -255,10 +260,7 @@ minillap.shared.query.files=acid_globallimit.q,\
union7.q,\
union8.q,\
union9.q,\
- unionDistinct_1.q,\
- union_fast_stats.q,\
union_stats.q,\
- union_type_chk.q,\
update_after_multiple_inserts.q,\
update_all_non_partitioned.q,\
update_all_partitioned.q,\
@@ -403,11 +405,35 @@ minillap.shared.query.files=acid_globallimit.q,\
vectorized_timestamp_ints_casts.q
minillap.query.files=acid_bucket_pruning.q,\
- acid_vectorization_missing_cols.q,\
- auto_sortmerge_join_16.q,\
- bucket4.q,\
bucket5.q,\
bucket6.q,\
+ explainuser_2.q,\
+ empty_dir_in_table.q,\
+ llap_udf.q,\
+ llapdecider.q,\
+ reduce_deduplicate.q,\
+ remote_script.q,\
+ tez_aggr_part_stats.q,\
+ tez_union_view.q,\
+ file_with_header_footer.q,\
+ external_table_with_space_in_location_path.q,\
+ import_exported_table.q,\
+ orc_llap_counters.q,\
+ orc_llap_counters1.q,\
+ load_hdfs_file_with_space_in_the_name.q,\
+ orc_ppd_basic.q,\
+ schemeAuthority.q,\
+ schemeAuthority2.q,\
+ temp_table_external.q,\
+ table_nonprintable.q,\
+ llap_nullscan.q,\
+ dynamic_partition_pruning_2.q,\
+ tez_union_dynamic_partition.q,\
+ load_fs2.q
+
+minillaplocal.query.files=acid_vectorization_missing_cols.q,\
+ auto_sortmerge_join_16.q,\
+ bucket4.q,\
bucket_many.q,\
bucket_map_join_tez1.q,\
bucket_map_join_tez2.q,\
@@ -418,18 +444,12 @@ minillap.query.files=acid_bucket_pruning.q,\
constprog_dpp.q,\
disable_merge_for_bucketing.q,\
dynamic_partition_pruning.q,\
- dynamic_partition_pruning_2.q,\
- empty_dir_in_table.q,\
exchgpartition2lel.q,\
explainuser_1.q,\
- explainuser_2.q,\
explainuser_4.q,\
- external_table_with_space_in_location_path.q,\
- file_with_header_footer.q,\
groupby2.q,\
hybridgrace_hashjoin_1.q,\
hybridgrace_hashjoin_2.q,\
- import_exported_table.q,\
infer_bucket_sort_bucketed_table.q,\
input16_cc.q,\
insert_dir_distcp.q,\
@@ -437,24 +457,14 @@ minillap.query.files=acid_bucket_pruning.q,\
join_acid_non_acid.q,\
leftsemijoin_mr.q,\
list_bucket_dml_10.q,\
- llap_nullscan.q,\
- llap_udf.q,\
- llapdecider.q,\
- load_fs2.q,\
- load_hdfs_file_with_space_in_the_name.q,\
lvj_mapjoin.q,\
mapjoin_decimal.q,\
mergejoin_3way.q,\
mrr.q,\
non_native_window_udf.q,\
orc_llap.q,\
- orc_llap_counters.q,\
- orc_llap_counters1.q,\
orc_llap_nonvector.q,\
- orc_ppd_basic.q,\
quotedid_smb.q,\
- reduce_deduplicate.q,\
- remote_script.q,\
schema_evol_orc_acid_part.q,\
schema_evol_orc_acid_part_update.q,\
schema_evol_orc_acid_table.q,\
@@ -484,12 +494,7 @@ minillap.query.files=acid_bucket_pruning.q,\
schema_evol_text_vecrow_part_all_complex.q,\
schema_evol_text_vecrow_part_all_primitive.q,\
schema_evol_text_vecrow_table.q,\
- schemeAuthority.q,\
- schemeAuthority2.q,\
smb_cache.q,\
- table_nonprintable.q,\
- temp_table_external.q,\
- tez_aggr_part_stats.q,\
tez_bmj_schema_evolution.q,\
tez_dml.q,\
tez_dynpart_hashjoin_1.q,\
@@ -510,10 +515,8 @@ minillap.query.files=acid_bucket_pruning.q,\
tez_union.q,\
tez_union2.q,\
tez_union_decimal.q,\
- tez_union_dynamic_partition.q,\
tez_union_group_by.q,\
tez_union_multiinsert.q,\
- tez_union_view.q,\
tez_vector_dynpart_hashjoin_1.q,\
tez_vector_dynpart_hashjoin_2.q,\
uber_reduce.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index ca72282..201c3d3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -47,6 +47,7 @@ public class CliConfigs {
setQueryDir("ql/src/test/queries/clientpositive");
excludesFrom(testConfigProps, "minillap.query.files");
+ excludesFrom(testConfigProps, "minillaplocal.query.files");
excludesFrom(testConfigProps, "minimr.query.files");
excludesFrom(testConfigProps, "minitez.query.files");
excludesFrom(testConfigProps, "encrypted.query.files");
@@ -129,6 +130,7 @@ public class CliConfigs {
setHiveConfDir("data/conf/tez");
setClusterType(MiniClusterType.tez);
setMetastoreType(MetastoreType.hbase);
+ setFsType(QTestUtil.FsType.hdfs);
} catch (Exception e) {
throw new RuntimeException("can't construct cliconfig", e);
}
@@ -152,6 +154,33 @@ public class CliConfigs {
setHiveConfDir("data/conf/llap");
setClusterType(MiniClusterType.llap);
+ setMetastoreType(MetastoreType.sql);
+ } catch (Exception e) {
+ throw new RuntimeException("can't construct cliconfig", e);
+ }
+ }
+ }
+
+ public static class MiniLlapLocalCliConfig extends AbstractCliConfig {
+
+ public MiniLlapLocalCliConfig() {
+ super(CoreCliDriver.class);
+ try {
+ setQueryDir("ql/src/test/queries/clientpositive");
+
+ includesFrom(testConfigProps, "minillaplocal.query.files");
+ includesFrom(testConfigProps, "minillaplocal.shared.query.files");
+
+ setResultsDir("ql/src/test/results/clientpositive/llap");
+ setLogDir("itests/qtest/target/qfile-results/clientpositive");
+
+ setInitScript("q_test_init.sql");
+ setCleanupScript("q_test_cleanup.sql");
+
+ setHiveConfDir("data/conf/llap");
+ setClusterType(MiniClusterType.llap_local);
+ setMetastoreType(MetastoreType.sql);
+ setFsType(QTestUtil.FsType.local);
} catch (Exception e) {
throw new RuntimeException("can't construct cliconfig", e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0a4b3d8f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index 1ab7137..f46e2d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -156,6 +156,7 @@ public class LlapDecider implements PhysicalPlanResolver {
if (tezWork.getChildren(work).isEmpty()
&& work instanceof ReduceWork
&& ((ReduceWork) work).getNumReduceTasks() == 1) {
+ LOG.info("Converting work to uber: {}", work);
work.setUberMode(true);
}
}