You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/11/19 20:56:01 UTC
svn commit: r1543549 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/conf/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/ itests/qtest/
itests/util/src/main/java/org/apache/...
Author: hashutosh
Date: Tue Nov 19 19:56:00 2013
New Revision: 1543549
URL: http://svn.apache.org/r1543549
Log:
HIVE-4632 : Use hadoop counter as a stat publisher (Navis via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java
hive/trunk/ql/src/test/queries/clientpositive/stats_counter.q
hive/trunk/ql/src/test/results/clientpositive/stats_counter.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/data/conf/hive-site.xml
hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java
hive/trunk/itests/qtest/pom.xml
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/DummyStatsAggregator.java
hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/KeyVerifyingStatsAggregator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestStatsPublisherEnhanced.java
hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_1.q
hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_2.q
hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_1.q
hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_2.q
hive/trunk/ql/src/test/queries/clientpositive/stats19.q
hive/trunk/ql/src/test/queries/clientpositive/stats_aggregator_error_1.q
hive/trunk/ql/src/test/queries/clientpositive/stats_publisher_error_1.q
hive/trunk/ql/src/test/results/clientpositive/stats_aggregator_error_1.q.out
hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java Tue Nov 19 19:56:00 2013
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hive.common;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+
import java.util.Map;
@@ -27,15 +30,34 @@ import java.util.Map;
public class StatsSetupConst {
- /**
- * The value of the user variable "hive.stats.dbclass" to use HBase implementation.
- */
- public static final String HBASE_IMPL_CLASS_VAL = "hbase";
-
- /**
- * The value of the user variable "hive.stats.dbclass" to use JDBC implementation.
- */
- public static final String JDBC_IMPL_CLASS_VAL = "jdbc";
+ public enum StatDB {
+ hbase {
+ public String getPublisher(Configuration conf) {
+ return "org.apache.hadoop.hive.hbase.HBaseStatsPublisher"; }
+ public String getAggregator(Configuration conf) {
+ return "org.apache.hadoop.hive.hbase.HBaseStatsAggregator"; }
+ },
+ jdbc {
+ public String getPublisher(Configuration conf) {
+ return "org.apache.hadoop.hive.ql.stats.jdbc.JDBCStatsPublisher"; }
+ public String getAggregator(Configuration conf) {
+ return "org.apache.hadoop.hive.ql.stats.jdbc.JDBCStatsAggregator"; }
+ },
+ counter {
+ public String getPublisher(Configuration conf) {
+ return "org.apache.hadoop.hive.ql.stats.CounterStatsPublisher"; }
+ public String getAggregator(Configuration conf) {
+ return "org.apache.hadoop.hive.ql.stats.CounterStatsAggregator"; }
+ },
+ custom {
+ public String getPublisher(Configuration conf) {
+ return HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_DEFAULT_PUBLISHER); }
+ public String getAggregator(Configuration conf) {
+ return HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_DEFAULT_AGGREGATOR); }
+ };
+ public abstract String getPublisher(Configuration conf);
+ public abstract String getAggregator(Configuration conf);
+ }
// statistics stored in metastore
/**
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Nov 19 19:56:00 2013
@@ -603,8 +603,8 @@ public class HiveConf extends Configurat
// Statistics
HIVESTATSAUTOGATHER("hive.stats.autogather", true),
- HIVESTATSDBCLASS("hive.stats.dbclass",
- "jdbc:derby"), // other options are jdbc:mysql and hbase as defined in StatsSetupConst.java
+ HIVESTATSDBCLASS("hive.stats.dbclass", "counter",
+ new PatternValidator("jdbc(:.*)", "hbase", "counter", "custom")), // StatsSetupConst.StatDB
HIVESTATSJDBCDRIVER("hive.stats.jdbcdriver",
"org.apache.derby.jdbc.EmbeddedDriver"), // JDBC driver specific to the dbclass
HIVESTATSDBCONNECTIONSTRING("hive.stats.dbconnectionstring",
@@ -1367,6 +1367,27 @@ public class HiveConf extends Configurat
}
}
+ public static class PatternValidator implements Validator {
+ private final List<Pattern> expected = new ArrayList<Pattern>();
+ private PatternValidator(String... values) {
+ for (String value : values) {
+ expected.add(Pattern.compile(value));
+ }
+ }
+ @Override
+ public String validate(String value) {
+ if (value == null) {
+ return "Invalid value.. expects one of patterns " + expected;
+ }
+ for (Pattern pattern : expected) {
+ if (pattern.matcher(value).matches()) {
+ return null;
+ }
+ }
+ return "Invalid value.. expects one of patterns " + expected;
+ }
+ }
+
public static class RatioValidator implements Validator {
@Override
public String validate(String value) {
Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Tue Nov 19 19:56:00 2013
@@ -1203,8 +1203,8 @@
<property>
<name>hive.stats.dbclass</name>
- <value>jdbc:derby</value>
- <description>The default database that stores temporary hive statistics.</description>
+ <value>counter</value>
+ <description>The storage that stores temporary hive statistics. Currently, jdbc, hbase, counter and custom type is supported</description>
</property>
<property>
@@ -1228,13 +1228,13 @@
<property>
<name>hive.stats.default.publisher</name>
<value></value>
- <description>The Java class (implementing the StatsPublisher interface) that is used by default if hive.stats.dbclass is not JDBC or HBase.</description>
+ <description>The Java class (implementing the StatsPublisher interface) that is used by default if hive.stats.dbclass is custom type.</description>
</property>
<property>
<name>hive.stats.default.aggregator</name>
<value></value>
- <description>The Java class (implementing the StatsAggregator interface) that is used by default if hive.stats.dbclass is not JDBC or HBase.</description>
+ <description>The Java class (implementing the StatsAggregator interface) that is used by default if hive.stats.dbclass is custom type.</description>
</property>
<property>
Modified: hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/hive-site.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/data/conf/hive-site.xml (original)
+++ hive/trunk/data/conf/hive-site.xml Tue Nov 19 19:56:00 2013
@@ -186,4 +186,9 @@
<description>The default SerDe hive will use for the rcfile format</description>
</property>
+<property>
+ <name>hive.stats.dbclass</name>
+ <value>jdbc:derby</value>
+ <description>The default storatge that stores temporary hive statistics. Currently, jdbc, hbase and counter type is supported</description>
+</property>
</configuration>
Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java Tue Nov 19 19:56:00 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.stats.StatsAggregator;
@@ -46,7 +47,7 @@ public class HBaseStatsAggregator implem
/**
* Does the necessary HBase initializations.
*/
- public boolean connect(Configuration hiveconf) {
+ public boolean connect(Configuration hiveconf, MapRedTask sourceTask) {
try {
htable = new HTable(HBaseConfiguration.create(hiveconf),
Modified: hive/trunk/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest/pom.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/itests/qtest/pom.xml (original)
+++ hive/trunk/itests/qtest/pom.xml Tue Nov 19 19:56:00 2013
@@ -36,7 +36,7 @@
<run_disabled>false</run_disabled>
<clustermode></clustermode>
<execute.beeline.tests>false</execute.beeline.tests>
- <minimr.query.files>list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q</minimr.query.files>
+ <minimr.query.files>list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q</minimr.query.files>
<minimr.query.negative.files>cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q</minimr.query.negative.files>
<beeline.positive.exclude>add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rena
me.q,exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_o
verwrite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q</beeline.positive.exclude>
</properties>
Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/DummyStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/DummyStatsAggregator.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/DummyStatsAggregator.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/DummyStatsAggregator.java Tue Nov 19 19:56:00 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.stats;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
/**
* An test implementation for StatsAggregator.
@@ -32,7 +33,7 @@ public class DummyStatsAggregator implem
// This is a test. The parameter hive.test.dummystats.aggregator's value
// denotes the method which needs to throw an error.
- public boolean connect(Configuration hconf) {
+ public boolean connect(Configuration hconf, MapRedTask sourceTask) {
errorMethod = hconf.get("hive.test.dummystats.aggregator", "");
if (errorMethod.equalsIgnoreCase("connect")) {
return false;
Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/KeyVerifyingStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/KeyVerifyingStatsAggregator.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/KeyVerifyingStatsAggregator.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/stats/KeyVerifyingStatsAggregator.java Tue Nov 19 19:56:00 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.stats;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.session.SessionState;
/**
@@ -29,7 +30,7 @@ import org.apache.hadoop.hive.ql.session
public class KeyVerifyingStatsAggregator implements StatsAggregator {
- public boolean connect(Configuration hconf) {
+ public boolean connect(Configuration hconf, MapRedTask sourceTask) {
return true;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Tue Nov 19 19:56:00 2013
@@ -28,9 +28,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.Warehouse;
@@ -178,11 +176,11 @@ public class StatsTask extends Task<Stat
if (!this.getWork().getNoStatsAggregator()) {
String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
- StatsFactory.setImplementation(statsImplementationClass, conf);
- if (work.isNoScanAnalyzeCommand()){
+ StatsFactory factory = StatsFactory.newFactory(statsImplementationClass, conf);
+ if (factory != null && work.isNoScanAnalyzeCommand()){
// initialize stats publishing table for noscan which has only stats task
// the rest of MR task following stats task initializes it in ExecDriver.java
- StatsPublisher statsPublisher = StatsFactory.getStatsPublisher();
+ StatsPublisher statsPublisher = factory.getStatsPublisher();
if (!statsPublisher.init(conf)) { // creating stats table if not exists
if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) {
throw
@@ -190,10 +188,12 @@ public class StatsTask extends Task<Stat
}
}
}
- statsAggregator = StatsFactory.getStatsAggregator();
- // manufacture a StatsAggregator
- if (!statsAggregator.connect(conf)) {
- throw new HiveException("StatsAggregator connect failed " + statsImplementationClass);
+ if (factory != null) {
+ statsAggregator = factory.getStatsAggregator();
+ // manufacture a StatsAggregator
+ if (!statsAggregator.connect(conf, getWork().getSourceTask())) {
+ throw new HiveException("StatsAggregator connect failed " + statsImplementationClass);
+ }
}
}
@@ -445,19 +445,4 @@ public class StatsTask extends Task<Stat
}
return list;
}
-
- /**
- * This method is static as it is called from the shutdown hook at the ExecDriver.
- */
- public static void cleanUp(String jobID, Configuration config) {
- StatsAggregator statsAggregator;
- String statsImplementationClass = HiveConf.getVar(config, HiveConf.ConfVars.HIVESTATSDBCLASS);
- StatsFactory.setImplementation(statsImplementationClass, config);
- statsAggregator = StatsFactory.getStatsAggregator();
- if (statsAggregator.connect(config)) {
- statsAggregator.cleanUp(jobID + Path.SEPARATOR); // Adding the path separator to avoid an Id
- // being a prefix of another ID
- statsAggregator.closeConnection();
- }
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Nov 19 19:56:00 2013
@@ -2276,12 +2276,8 @@ public final class Utilities {
}
public static StatsPublisher getStatsPublisher(JobConf jc) {
- String statsImplementationClass = HiveConf.getVar(jc, HiveConf.ConfVars.HIVESTATSDBCLASS);
- if (StatsFactory.setImplementation(statsImplementationClass, jc)) {
- return StatsFactory.getStatsPublisher();
- } else {
- return null;
- }
+ StatsFactory factory = StatsFactory.newFactory(jc);
+ return factory == null ? null : factory.getStatsPublisher();
}
/**
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Tue Nov 19 19:56:00 2013
@@ -82,7 +82,6 @@ import org.apache.hadoop.mapred.Counters
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.log4j.Appender;
@@ -414,9 +413,9 @@ public class ExecDriver extends Task<Map
if (mWork.isGatheringStats() || (rWork != null && rWork.isGatheringStats())) {
// initialize stats publishing table
StatsPublisher statsPublisher;
- String statsImplementationClass = HiveConf.getVar(job, HiveConf.ConfVars.HIVESTATSDBCLASS);
- if (StatsFactory.setImplementation(statsImplementationClass, job)) {
- statsPublisher = StatsFactory.getStatsPublisher();
+ StatsFactory factory = StatsFactory.newFactory(job);
+ if (factory != null) {
+ statsPublisher = factory.getStatsPublisher();
if (!statsPublisher.init(job)) { // creating stats table if not exists
if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) {
throw
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java Tue Nov 19 19:56:00 2013
@@ -205,9 +205,9 @@ public class PartialScanTask extends Tas
if (work.isGatheringStats()) {
// initialize stats publishing table
StatsPublisher statsPublisher;
- String statsImplementationClass = HiveConf.getVar(job, HiveConf.ConfVars.HIVESTATSDBCLASS);
- if (StatsFactory.setImplementation(statsImplementationClass, job)) {
- statsPublisher = StatsFactory.getStatsPublisher();
+ StatsFactory factory = StatsFactory.newFactory(job);
+ if (factory != null) {
+ statsPublisher = factory.getStatsPublisher();
if (!statsPublisher.init(job)) { // creating stats table if not exists
if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) {
throw
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Tue Nov 19 19:56:00 2013
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.UnionOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork;
import org.apache.hadoop.hive.ql.lib.Node;
@@ -221,7 +222,7 @@ public class GenMRFileSink1 implements N
private void addStatsTask(FileSinkOperator nd, MoveTask mvTask,
Task<? extends Serializable> currTask, HiveConf hconf) {
- MoveWork mvWork = ((MoveTask) mvTask).getWork();
+ MoveWork mvWork = mvTask.getWork();
StatsWork statsWork = null;
if (mvWork.getLoadTableWork() != null) {
statsWork = new StatsWork(mvWork.getLoadTableWork());
@@ -229,12 +230,14 @@ public class GenMRFileSink1 implements N
statsWork = new StatsWork(mvWork.getLoadFileWork());
}
assert statsWork != null : "Error when genereting StatsTask";
+
+ statsWork.setSourceTask((MapRedTask)currTask);
statsWork.setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE));
MapredWork mrWork = (MapredWork) currTask.getWork();
// AggKey in StatsWork is used for stats aggregation while StatsAggPrefix
// in FileSinkDesc is used for stats publishing. They should be consistent.
- statsWork.setAggKey(((FileSinkOperator) nd).getConf().getStatsAggPrefix());
+ statsWork.setAggKey(nd.getConf().getStatsAggPrefix());
Task<? extends Serializable> statsTask = TaskFactory.get(statsWork, hconf);
// mark the MapredWork and FileSinkOperator for gathering stats
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java Tue Nov 19 19:56:00 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
/**
@@ -49,6 +50,8 @@ public class StatsWork implements Serial
private boolean isPartialScanAnalyzeCommand = false;
+ private transient MapRedTask sourceTask;
+
public StatsWork() {
}
@@ -140,4 +143,12 @@ public class StatsWork implements Serial
public void setPartialScanAnalyzeCommand(boolean isPartialScanAnalyzeCommand) {
this.isPartialScanAnalyzeCommand = isPartialScanAnalyzeCommand;
}
+
+ public MapRedTask getSourceTask() {
+ return sourceTask;
+ }
+
+ public void setSourceTask(MapRedTask sourceTask) {
+ this.sourceTask = sourceTask;
+ }
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java?rev=1543549&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java Tue Nov 19 19:56:00 2013
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.stats;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
+
+public class CounterStatsAggregator implements StatsAggregator {
+
+ private static final Log LOG = LogFactory.getLog(CounterStatsAggregator.class.getName());
+
+ private Counters counters;
+ private JobClient jc;
+
+ @Override
+ public boolean connect(Configuration hconf, MapRedTask sourceTask) {
+ try {
+ jc = new JobClient(toJobConf(hconf));
+ RunningJob job = jc.getJob(sourceTask.getJobID());
+ if (job != null) {
+ counters = job.getCounters();
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to get Job instance for " + sourceTask.getJobID());
+ }
+ return counters != null;
+ }
+
+ private JobConf toJobConf(Configuration hconf) {
+ return hconf instanceof JobConf ? (JobConf)hconf : new JobConf(hconf, ExecDriver.class);
+ }
+
+ @Override
+ public String aggregateStats(String keyPrefix, String statType) {
+ long value = 0;
+ for (String groupName : counters.getGroupNames()) {
+ if (groupName.startsWith(keyPrefix)) {
+ value += counters.getGroup(groupName).getCounter(statType);
+ }
+ }
+ return String.valueOf(value);
+ }
+
+ @Override
+ public boolean closeConnection() {
+ try {
+ jc.close();
+ } catch (IOException e) {
+ LOG.error("Error closing job client for stats aggregator.", e);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean cleanUp(String keyPrefix) {
+ return true;
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java?rev=1543549&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java Tue Nov 19 19:56:00 2013
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.stats;
+
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.MapredContext;
+import org.apache.hadoop.mapred.Reporter;
+
+public class CounterStatsPublisher implements StatsPublisher {
+
+ private static final Log LOG = LogFactory.getLog(CounterStatsPublisher.class.getName());
+
+ private Reporter reporter;
+
+ @Override
+ public boolean init(Configuration hconf) {
+ return true;
+ }
+
+ public boolean connect(Configuration hconf) {
+ MapredContext context = MapredContext.get();
+ if (context == null || context.getReporter() == null) {
+ return false;
+ }
+ reporter = context.getReporter();
+ return true;
+ }
+
+ @Override
+ public boolean publishStat(String fileID, Map<String, String> stats) {
+ for (Map.Entry<String, String> entry : stats.entrySet()) {
+ try {
+ reporter.incrCounter(fileID, entry.getKey(), Long.valueOf(entry.getValue()));
+ } catch (NumberFormatException e) {
+ LOG.error("Invalid counter value " + entry.getValue() + " for " + entry.getKey());
+ }
+ }
+ return true;
+ }
+ @Override
+ public boolean closeConnection() {
+ return true;
+ }
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java Tue Nov 19 19:56:00 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.stats;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
/**
* An interface for any possible implementation for gathering statistics.
@@ -31,9 +32,10 @@ public interface StatsAggregator {
*
* @param hconf
* HiveConf that contains the connection parameters.
+ * @param sourceTask
* @return true if connection is successful, false otherwise.
*/
- public boolean connect(Configuration hconf);
+ public boolean connect(Configuration hconf, MapRedTask sourceTask);
/**
* This method aggregates a given statistic from all tasks (partial stats).
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java Tue Nov 19 19:56:00 2013
@@ -24,7 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.StatsSetupConst.StatDB;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.util.ReflectionUtils;
@@ -36,9 +36,13 @@ public final class StatsFactory {
static final private Log LOG = LogFactory.getLog(StatsFactory.class.getName());
- private static Class <? extends Serializable> publisherImplementation;
- private static Class <? extends Serializable> aggregatorImplementation;
- private static Configuration jobConf;
+ private Class <? extends Serializable> publisherImplementation;
+ private Class <? extends Serializable> aggregatorImplementation;
+ private Configuration jobConf;
+
+ public static StatsFactory newFactory(Configuration conf) {
+ return newFactory(HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS), conf);
+ }
/**
* Sets the paths of the implementation classes of publishing
@@ -46,53 +50,30 @@ public final class StatsFactory {
* The paths are determined according to a configuration parameter which
* is passed as the user input for choosing the implementation as MySQL, HBase, ...
*/
- public static boolean setImplementation(String configurationParam, Configuration conf) {
+ public static StatsFactory newFactory(String configurationParam, Configuration conf) {
+ StatsFactory factory = new StatsFactory(conf);
+ if (factory.initialize(configurationParam.toLowerCase())) {
+ return factory;
+ }
+ return null;
+ }
+ private StatsFactory(Configuration conf) {
+ this.jobConf = conf;
+ }
+
+ private boolean initialize(String type) {
ClassLoader classLoader = JavaUtils.getClassLoader();
- if (configurationParam.equals(StatsSetupConst.HBASE_IMPL_CLASS_VAL)) {
- // Case: hbase
- try {
- publisherImplementation = (Class<? extends Serializable>)
- Class.forName("org.apache.hadoop.hive.hbase.HBaseStatsPublisher", true, classLoader);
-
- aggregatorImplementation = (Class<? extends Serializable>)
- Class.forName("org.apache.hadoop.hive.hbase.HBaseStatsAggregator", true, classLoader);
- } catch (ClassNotFoundException e) {
- LOG.error("HBase Publisher/Aggregator classes cannot be loaded.", e);
- return false;
- }
- } else if (configurationParam.contains(StatsSetupConst.JDBC_IMPL_CLASS_VAL)) {
- // Case: jdbc:mysql or jdbc:derby
- try {
- publisherImplementation = (Class<? extends Serializable>)
- Class.forName("org.apache.hadoop.hive.ql.stats.jdbc.JDBCStatsPublisher", true, classLoader);
-
- aggregatorImplementation = (Class<? extends Serializable>)
- Class.forName("org.apache.hadoop.hive.ql.stats.jdbc.JDBCStatsAggregator", true, classLoader);
- } catch (ClassNotFoundException e) {
- LOG.error("JDBC Publisher/Aggregator classes cannot be loaded.", e);
- return false;
- }
- } else {
- // try default stats publisher/aggregator
- String defPublisher = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_DEFAULT_PUBLISHER);
- String defAggregator = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_DEFAULT_AGGREGATOR);
- // ERROR no default publisher/aggregator is defined
- if (defPublisher == null || defAggregator == null) {
- return false;
- }
- try{
- publisherImplementation = (Class<? extends Serializable>)
- Class.forName(defPublisher, true, classLoader);
- aggregatorImplementation = (Class<? extends Serializable>)
- Class.forName(defAggregator, true, classLoader);
- } catch (ClassNotFoundException e) {
- LOG.error("JDBC Publisher/Aggregator classes cannot be loaded.", e);
- return false;
- }
+ try {
+ StatDB statDB = type.startsWith("jdbc") ? StatDB.jdbc : StatDB.valueOf(type);
+ publisherImplementation = (Class<? extends Serializable>)
+ Class.forName(statDB.getPublisher(jobConf), true, classLoader);
+ aggregatorImplementation = (Class<? extends Serializable>)
+ Class.forName(statDB.getAggregator(jobConf), true, classLoader);
+ } catch (Exception e) {
+ LOG.error(type + " Publisher/Aggregator classes cannot be loaded.", e);
+ return false;
}
-
- jobConf = conf;
return true;
}
@@ -100,7 +81,7 @@ public final class StatsFactory {
* Returns a Stats publisher implementation class for the IStatsPublisher interface
* For example HBaseStatsPublisher for the HBase implementation
*/
- public static StatsPublisher getStatsPublisher() {
+ public StatsPublisher getStatsPublisher() {
return (StatsPublisher) ReflectionUtils.newInstance(publisherImplementation, jobConf);
}
@@ -109,7 +90,7 @@ public final class StatsFactory {
* Returns a Stats Aggregator implementation class for the IStatsAggregator interface
* For example HBaseStatsAggregator for the HBase implementation
*/
- public static StatsAggregator getStatsAggregator() {
+ public StatsAggregator getStatsAggregator() {
return (StatsAggregator) ReflectionUtils.newInstance(aggregatorImplementation, jobConf);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Tue Nov 19 19:56:00 2013
@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.stats.StatsAggregator;
@@ -40,6 +41,7 @@ public class JDBCStatsAggregator impleme
private Connection conn;
private String connectionString;
private Configuration hiveconf;
+ private MapRedTask sourceTask;
private final Map<String, PreparedStatement> columnMapping;
private final Log LOG = LogFactory.getLog(this.getClass().getName());
private int timeout = 30;
@@ -53,7 +55,7 @@ public class JDBCStatsAggregator impleme
}
@Override
- public boolean connect(Configuration hiveconf) {
+ public boolean connect(Configuration hiveconf, MapRedTask sourceTask) {
this.hiveconf = hiveconf;
timeout = HiveConf.getIntVar(hiveconf, HiveConf.ConfVars.HIVE_STATS_JDBC_TIMEOUT);
connectionString = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING);
@@ -157,7 +159,7 @@ public class JDBCStatsAggregator impleme
} catch (InterruptedException iex) {
}
// getting a new connection
- if (!connect(hiveconf)) {
+ if (!connect(hiveconf, sourceTask)) {
// if cannot reconnect, just fail because connect() already handles retries.
LOG.error("Error during publishing aggregation. " + e);
return null;
@@ -235,7 +237,7 @@ public class JDBCStatsAggregator impleme
} catch (InterruptedException iex) {
}
// getting a new connection
- if (!connect(hiveconf)) {
+ if (!connect(hiveconf, sourceTask)) {
LOG.error("Error during clean-up. " + e);
return false;
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestStatsPublisherEnhanced.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestStatsPublisherEnhanced.java?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestStatsPublisherEnhanced.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestStatsPublisherEnhanced.java Tue Nov 19 19:56:00 2013
@@ -41,12 +41,14 @@ public class TestStatsPublisherEnhanced
protected String statsImplementationClass;
protected Map<String, String> stats;
+ protected StatsFactory factory;
+
public TestStatsPublisherEnhanced(String name) {
super(name);
conf = new JobConf(TestStatsPublisherEnhanced.class);
conf.set("hive.stats.dbclass", "jdbc:derby");
- statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
- StatsFactory.setImplementation(statsImplementationClass, conf);
+ factory = StatsFactory.newFactory(conf);
+ assert factory != null;
}
@Override
@@ -56,9 +58,9 @@ public class TestStatsPublisherEnhanced
@Override
protected void tearDown() {
- StatsAggregator sa = StatsFactory.getStatsAggregator();
+ StatsAggregator sa = factory.getStatsAggregator();
assertNotNull(sa);
- assertTrue(sa.connect(conf));
+ assertTrue(sa.connect(conf, null));
assertTrue(sa.cleanUp("file_0"));
assertTrue(sa.closeConnection());
}
@@ -82,9 +84,9 @@ public class TestStatsPublisherEnhanced
assertTrue(statsPublisher.connect(conf));
// instantiate stats aggregator
- StatsAggregator statsAggregator = StatsFactory.getStatsAggregator();
+ StatsAggregator statsAggregator = factory.getStatsAggregator();
assertNotNull(statsAggregator);
- assertTrue(statsAggregator.connect(conf));
+ assertTrue(statsAggregator.connect(conf, null));
// publish stats
fillStatMap("200", "1000");
@@ -130,9 +132,9 @@ public class TestStatsPublisherEnhanced
assertTrue(statsPublisher.connect(conf));
// instantiate stats aggregator
- StatsAggregator statsAggregator = StatsFactory.getStatsAggregator();
+ StatsAggregator statsAggregator = factory.getStatsAggregator();
assertNotNull(statsAggregator);
- assertTrue(statsAggregator.connect(conf));
+ assertTrue(statsAggregator.connect(conf, null));
// statsAggregator.cleanUp("file_0000");
// assertTrue(statsAggregator.connect(conf));
@@ -191,9 +193,9 @@ public class TestStatsPublisherEnhanced
assertTrue(statsPublisher.connect(conf));
// instantiate stats aggregator
- StatsAggregator statsAggregator = StatsFactory.getStatsAggregator();
+ StatsAggregator statsAggregator = factory.getStatsAggregator();
assertNotNull(statsAggregator);
- assertTrue(statsAggregator.connect(conf));
+ assertTrue(statsAggregator.connect(conf, null));
// publish stats
fillStatMap("200", "1000");
@@ -256,9 +258,9 @@ public class TestStatsPublisherEnhanced
assertTrue(statsPublisher.connect(conf));
// instantiate stats aggregator
- StatsAggregator statsAggregator = StatsFactory.getStatsAggregator();
+ StatsAggregator statsAggregator = factory.getStatsAggregator();
assertNotNull(statsAggregator);
- assertTrue(statsAggregator.connect(conf));
+ assertTrue(statsAggregator.connect(conf, null));
// publish stats
fillStatMap("200", "");
@@ -327,9 +329,9 @@ public class TestStatsPublisherEnhanced
assertTrue(statsPublisher.connect(conf));
// instantiate stats aggregator
- StatsAggregator statsAggregator = StatsFactory.getStatsAggregator();
+ StatsAggregator statsAggregator = factory.getStatsAggregator();
assertNotNull(statsAggregator);
- assertTrue(statsAggregator.connect(conf));
+ assertTrue(statsAggregator.connect(conf, null));
// publish stats
fillStatMap("200", "1000");
Modified: hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_1.q?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_1.q (original)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_1.q Tue Nov 19 19:56:00 2013
@@ -6,7 +6,7 @@
create table tmptable(key string, value string);
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher=org.apache.hadoop.hive.ql.stats.DummyStatsPublisher;
set hive.stats.default.aggregator=org.apache.hadoop.hive.ql.stats.DummyStatsAggregator;
set hive.test.dummystats.aggregator=connect;
Modified: hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_2.q?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_2.q (original)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_aggregator_error_2.q Tue Nov 19 19:56:00 2013
@@ -5,7 +5,7 @@
create table tmptable(key string, value string);
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher=org.apache.hadoop.hive.ql.stats.DummyStatsPublisher;
set hive.stats.default.aggregator="";
Modified: hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_1.q?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_1.q (original)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_1.q Tue Nov 19 19:56:00 2013
@@ -6,7 +6,7 @@
create table tmptable(key string, value string);
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher=org.apache.hadoop.hive.ql.stats.DummyStatsPublisher;
set hive.stats.default.aggregator=org.apache.hadoop.hive.ql.stats.DummyStatsAggregator;
set hive.test.dummystats.publisher=connect;
Modified: hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_2.q?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_2.q (original)
+++ hive/trunk/ql/src/test/queries/clientnegative/stats_publisher_error_2.q Tue Nov 19 19:56:00 2013
@@ -5,7 +5,7 @@
create table tmptable(key string, value string);
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher="";
set hive.stats.default.aggregator=org.apache.hadoop.hive.ql.stats.DummyStatsAggregator;
Modified: hive/trunk/ql/src/test/queries/clientpositive/stats19.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats19.q?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats19.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats19.q Tue Nov 19 19:56:00 2013
@@ -1,7 +1,7 @@
set datanucleus.cache.collections=false;
set hive.stats.autogather=true;
set hive.stats.reliable=true;
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher=org.apache.hadoop.hive.ql.stats.DummyStatsPublisher;
set hive.stats.default.aggregator=org.apache.hadoop.hive.ql.stats.KeyVerifyingStatsAggregator;
@@ -56,7 +56,7 @@ insert overwrite table stats_part partit
desc formatted stats_part partition (ds='2010-04-08', hr = '13');
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher=org.apache.hadoop.hive.ql.stats.DummyStatsPublisher;
set hive.stats.default.aggregator=org.apache.hadoop.hive.ql.stats.KeyVerifyingStatsAggregator;
set hive.stats.key.prefix.max.length=0;
Modified: hive/trunk/ql/src/test/queries/clientpositive/stats_aggregator_error_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats_aggregator_error_1.q?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats_aggregator_error_1.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats_aggregator_error_1.q Tue Nov 19 19:56:00 2013
@@ -1,12 +1,12 @@
-- In this test, there is a dummy stats aggregator which throws an error when various
--- methods are called (as indicated by the parameter hive.test.dummystats.agregator)
+-- methods are called (as indicated by the parameter hive.test.dummystats.aggregator)
-- Since stats need not be reliable (by setting hive.stats.reliable to false), the
-- insert statements succeed. The insert statement succeeds even if the stats aggregator
-- is set to null, since stats need not be reliable.
create table tmptable(key string, value string);
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher=org.apache.hadoop.hive.ql.stats.DummyStatsPublisher;
set hive.stats.default.aggregator=org.apache.hadoop.hive.ql.stats.DummyStatsAggregator;
set hive.stats.reliable=false;
Added: hive/trunk/ql/src/test/queries/clientpositive/stats_counter.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats_counter.q?rev=1543549&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats_counter.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats_counter.q Tue Nov 19 19:56:00 2013
@@ -0,0 +1,6 @@
+set hive.stats.autogather=true;
+set hive.stats.dbclass=counter;
+
+create table dummy as select * from src;
+
+desc formatted dummy;
Modified: hive/trunk/ql/src/test/queries/clientpositive/stats_publisher_error_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats_publisher_error_1.q?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats_publisher_error_1.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats_publisher_error_1.q Tue Nov 19 19:56:00 2013
@@ -6,7 +6,7 @@
create table tmptable(key string, value string);
-set hive.stats.dbclass=dummy;
+set hive.stats.dbclass=custom;
set hive.stats.default.publisher=org.apache.hadoop.hive.ql.stats.DummyStatsPublisher;
set hive.stats.default.aggregator=org.apache.hadoop.hive.ql.stats.DummyStatsAggregator;
set hive.stats.reliable=false;
Modified: hive/trunk/ql/src/test/results/clientpositive/stats_aggregator_error_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats_aggregator_error_1.q.out?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/stats_aggregator_error_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/stats_aggregator_error_1.q.out Tue Nov 19 19:56:00 2013
@@ -1,5 +1,5 @@
PREHOOK: query: -- In this test, there is a dummy stats aggregator which throws an error when various
--- methods are called (as indicated by the parameter hive.test.dummystats.agregator)
+-- methods are called (as indicated by the parameter hive.test.dummystats.aggregator)
-- Since stats need not be reliable (by setting hive.stats.reliable to false), the
-- insert statements succeed. The insert statement succeeds even if the stats aggregator
-- is set to null, since stats need not be reliable.
@@ -7,7 +7,7 @@ PREHOOK: query: -- In this test, there i
create table tmptable(key string, value string)
PREHOOK: type: CREATETABLE
POSTHOOK: query: -- In this test, there is a dummy stats aggregator which throws an error when various
--- methods are called (as indicated by the parameter hive.test.dummystats.agregator)
+-- methods are called (as indicated by the parameter hive.test.dummystats.aggregator)
-- Since stats need not be reliable (by setting hive.stats.reliable to false), the
-- insert statements succeed. The insert statement succeeds even if the stats aggregator
-- is set to null, since stats need not be reliable.
Added: hive/trunk/ql/src/test/results/clientpositive/stats_counter.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats_counter.q.out?rev=1543549&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/stats_counter.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/stats_counter.q.out Tue Nov 19 19:56:00 2013
@@ -0,0 +1,41 @@
+PREHOOK: query: create table dummy as select * from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+POSTHOOK: query: create table dummy as select * from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dummy
+PREHOOK: query: desc formatted dummy
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: desc formatted dummy
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+key string None
+value string None
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ totalSize 5812
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
Modified: hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml Tue Nov 19 19:56:00 2013
@@ -24,6 +24,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
@@ -557,6 +560,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
@@ -1075,6 +1081,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
@@ -557,6 +560,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
@@ -1075,6 +1081,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml Tue Nov 19 19:56:00 2013
@@ -24,6 +24,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml Tue Nov 19 19:56:00 2013
@@ -24,6 +24,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml Tue Nov 19 19:56:00 2013
@@ -24,6 +24,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml Tue Nov 19 19:56:00 2013
@@ -28,6 +28,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask1"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml Tue Nov 19 19:56:00 2013
@@ -24,6 +24,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml?rev=1543549&r1=1543548&r2=1543549&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml Tue Nov 19 19:56:00 2013
@@ -35,6 +35,9 @@
<void property="aggKey">
#### A masked pattern was here ####
</void>
+ <void property="sourceTask">
+ <object idref="MapRedTask0"/>
+ </void>
</object>
</void>
</object>