You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/04/10 02:35:40 UTC
svn commit: r1311533 - in /hive/branches/branch-0.9: ./
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/io/
ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/ ql...
Author: hashutosh
Date: Tue Apr 10 00:35:40 2012
New Revision: 1311533
URL: http://svn.apache.org/viewvc?rev=1311533&view=rev
Log:
Merged HIVE-2764 into hive9
Added:
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormatImpl.java
- copied unchanged from r1311418, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormatImpl.java
Modified:
hive/branches/branch-0.9/ (props changed)
hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java
hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java
hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Propchange: hive/branches/branch-0.9/
------------------------------------------------------------------------------
svn:mergeinfo = /hive/trunk:1311418
Modified: hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original)
+++ hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Tue Apr 10 00:35:40 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.HT
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.util.Bytes;
@@ -414,6 +415,9 @@ public class HiveHBaseTableInputFormat e
@Override
public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException {
+ //obtain delegation tokens for the job
+ TableMapReduceUtil.initCredentials(jobConf);
+
String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME);
setHTable(new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName)));
String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING);
Modified: hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java (original)
+++ hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java Tue Apr 10 00:35:40 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HBaseConf
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
@@ -102,6 +103,9 @@ public class HiveHBaseTableOutputFormat
@Override
public void checkOutputSpecs(FileSystem fs, JobConf jc) throws IOException {
+ //obtain delegation tokens for the job
+ TableMapReduceUtil.initCredentials(jc);
+
String hbaseTableName = jc.get(HBaseSerDe.HBASE_TABLE_NAME);
jc.set(TableOutputFormat.OUTPUT_TABLE, hbaseTableName);
Job job = new Job(jc);
Modified: hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java (original)
+++ hive/branches/branch-0.9/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java Tue Apr 10 00:35:40 2012
@@ -24,6 +24,7 @@ import java.util.Properties;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
@@ -40,8 +41,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Progressable;
@@ -203,4 +203,19 @@ public class HiveHFileOutputFormat exten
}
};
}
+
+ @Override
+ public void checkOutputSpecs(FileSystem ignored, JobConf jc) throws IOException {
+ //delegate to the new api
+ Job job = new Job(jc);
+ JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job);
+
+ checkOutputSpecs(jobContext);
+ }
+
+ @Override
+ public org.apache.hadoop.mapred.RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(
+ FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
+ throw new NotImplementedException("This will not be invoked");
+ }
}
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Apr 10 00:35:40 2012
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.ql.QueryPl
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
import org.apache.hadoop.hive.ql.io.IOPrepareCache;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.FetchWork;
@@ -264,7 +265,9 @@ public class ExecDriver extends Task<Map
return 5;
}
- ShimLoader.getHadoopShims().setNullOutputFormat(job);
+ ShimLoader.getHadoopShims().prepareJobOutput(job);
+ //See the javadoc on HiveOutputFormatImpl and HadoopShims.prepareJobOutput()
+ job.setOutputFormat(HiveOutputFormatImpl.class);
job.setMapperClass(ExecMapper.class);
job.setMapOutputKeyClass(HiveKey.class);
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Tue Apr 10 00:35:40 2012
@@ -35,7 +35,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
-import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.HivePartitioner;
@@ -52,10 +51,10 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.Serializer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.SubStructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
@@ -766,6 +765,28 @@ public class FileSinkOperator extends Te
getConf().getTableInfo());
}
+ public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
+ if (hiveOutputFormat == null) {
+ try {
+ hiveOutputFormat = conf.getTableInfo().getOutputFileFormatClass().newInstance();
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
+ Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), job);
+
+ if (conf.getTableInfo().isNonNative()) {
+ //check the ouput specs only if it is a storage handler (native tables's outputformats does
+ //not set the job's output properties correctly)
+ try {
+ hiveOutputFormat.checkOutputSpecs(ignored, job);
+ } catch (NoSuchMethodError e) {
+ //For BC, ignore this for now, but leave a log message
+ LOG.warn("HiveOutputFormat should implement checkOutputSpecs() method`");
+ }
+ }
+ }
+
private void publishStats() {
// Initializing a stats publisher
StatsPublisher statsPublisher = Utilities.getStatsPublisher(jc);
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java Tue Apr 10 00:35:40 2012
@@ -23,14 +23,13 @@ import java.util.Properties;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.util.Progressable;
@@ -39,9 +38,9 @@ import org.apache.hadoop.util.Progressab
* A {@link HiveOutputFormat} that writes {@link SequenceFile}s with the
* content saved in the keys, and null in the values.
*/
-public class HiveNullValueSequenceFileOutputFormat
- extends SequenceFileOutputFormat
- implements HiveOutputFormat<WritableComparable, Writable> {
+public class HiveNullValueSequenceFileOutputFormat<K,V>
+ extends SequenceFileOutputFormat<K,V>
+ implements HiveOutputFormat<K,V> {
private static final Writable NULL_WRITABLE = NullWritable.get();
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveOutputFormat.java Tue Apr 10 00:35:40 2012
@@ -24,8 +24,8 @@ import java.util.Properties;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.util.Progressable;
/**
@@ -34,16 +34,16 @@ import org.apache.hadoop.util.Progressab
* {@link #getHiveRecordWriter(JobConf, Path, Class, boolean, Properties, Progressable)}
* , with various parameters used to create the final out file and get some
* specific settings.
- *
+ *
* @see org.apache.hadoop.mapred.OutputFormat
* @see RecordWriter
* @see JobConf
*/
-public interface HiveOutputFormat<K extends WritableComparable, V extends Writable> {
+public interface HiveOutputFormat<K, V> extends OutputFormat<K, V> {
/**
* create the final out file and get some specific settings.
- *
+ *
* @param jc
* the job configuration file
* @param finalOutPath
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java Tue Apr 10 00:35:40 2012
@@ -23,25 +23,24 @@ import java.util.Properties;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.util.Progressable;
/** A {@link HiveOutputFormat} that writes {@link SequenceFile}s. */
-public class HiveSequenceFileOutputFormat extends SequenceFileOutputFormat
- implements HiveOutputFormat<WritableComparable, Writable> {
+public class HiveSequenceFileOutputFormat<K,V> extends SequenceFileOutputFormat<K,V>
+ implements HiveOutputFormat<K, V> {
BytesWritable EMPTY_KEY = new BytesWritable();
/**
* create the final out file, and output an empty key as the key.
- *
+ *
* @param jc
* the job configuration file
* @param finalOutPath
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java Tue Apr 10 00:35:40 2012
@@ -44,16 +44,16 @@ import org.apache.hadoop.util.Reflection
*/
public class RCFileOutputFormat extends
FileOutputFormat<WritableComparable, BytesRefArrayWritable> implements
- HiveOutputFormat<WritableComparable, Writable> {
+ HiveOutputFormat<WritableComparable, BytesRefArrayWritable> {
/**
* set number of columns into the given configuration.
- *
+ *
* @param conf
* configuration instance which need to set the column number
* @param columnNum
* column number for RCFile's Writer
- *
+ *
*/
public static void setColumnNumber(Configuration conf, int columnNum) {
assert columnNum > 0;
@@ -62,7 +62,7 @@ public class RCFileOutputFormat extends
/**
* Returns the number of columns set in the conf for writers.
- *
+ *
* @param conf
* @return number of columns for RCFile's writer
*/
@@ -102,7 +102,7 @@ public class RCFileOutputFormat extends
/**
* create the final out file.
- *
+ *
* @param jc
* the job configuration file
* @param finalOutPath
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Tue Apr 10 00:35:40 2012
@@ -37,13 +37,13 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.exec.ExecDriver;
import org.apache.hadoop.hive.ql.exec.HadoopJobExecHelper;
import org.apache.hadoop.hive.ql.exec.HadoopJobExecHook;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Throttle;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -91,7 +91,8 @@ public class BlockMergeTask extends Task
HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT,
CombineHiveInputFormat.class.getName());
success = true;
- ShimLoader.getHadoopShims().setNullOutputFormat(job);
+ ShimLoader.getHadoopShims().prepareJobOutput(job);
+ job.setOutputFormat(HiveOutputFormatImpl.class);
job.setMapperClass(work.getMapperClass());
Context ctx = driverContext.getCtx();
Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java Tue Apr 10 00:35:40 2012
@@ -454,4 +454,36 @@ public class MapredWork implements Seria
pathToPartitionInfo.put(path.toString(), partDesc);
}
+ public List<Operator<?>> getAllOperators() {
+ ArrayList<Operator<?>> opList = new ArrayList<Operator<?>>();
+ ArrayList<Operator<?>> returnList = new ArrayList<Operator<?>>();
+
+ if (getReducer() != null) {
+ opList.add(getReducer());
+ }
+
+ Map<String, ArrayList<String>> pa = getPathToAliases();
+ if (pa != null) {
+ for (List<String> ls : pa.values()) {
+ for (String a : ls) {
+ Operator<?> op = getAliasToWork().get(a);
+ if (op != null ) {
+ opList.add(op);
+ }
+ }
+ }
+ }
+
+ //recursively add all children
+ while (!opList.isEmpty()) {
+ Operator<?> op = opList.remove(0);
+ if (op.getChildOperators() != null) {
+ opList.addAll(op.getChildOperators());
+ }
+ returnList.add(op);
+ }
+
+ return returnList;
+ }
+
}
Modified: hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Tue Apr 10 00:35:40 2012
@@ -53,7 +53,6 @@ import org.apache.hadoop.mapred.TaskComp
import org.apache.hadoop.mapred.TaskID;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileSplit;
-import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.security.SecurityUtil;
@@ -474,8 +473,7 @@ public class Hadoop20Shims implements Ha
public void abortTask(TaskAttemptContext taskContext) { }
}
- public void setNullOutputFormat(JobConf conf) {
- conf.setOutputFormat(NullOutputFormat.class);
+ public void prepareJobOutput(JobConf conf) {
conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class);
// option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463)
Modified: hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Tue Apr 10 00:35:40 2012
@@ -50,7 +50,6 @@ import org.apache.hadoop.mapred.TaskComp
import org.apache.hadoop.mapred.TaskID;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileSplit;
-import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -473,8 +472,7 @@ public abstract class HadoopShimsSecure
public void abortTask(TaskAttemptContext taskContext) { }
}
- public void setNullOutputFormat(JobConf conf) {
- conf.setOutputFormat(NullOutputFormat.class);
+ public void prepareJobOutput(JobConf conf) {
conf.setOutputCommitter(NullOutputCommitter.class);
// option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463)
Modified: hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1311533&r1=1311532&r2=1311533&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Tue Apr 10 00:35:40 2012
@@ -152,10 +152,10 @@ public interface HadoopShims {
* the setup/cleanup/commit of output from the hive client. As a result it does
* not need support for the same inside the MR framework
*
- * This routine sets the appropriate options to set the output format and any
- * options related to bypass setup/cleanup/commit support in the MR framework
+ * This routine sets the appropriate options related to bypass setup/cleanup/commit
+ * support in the MR framework, but does not set the OutputFormat class.
*/
- void setNullOutputFormat(JobConf conf);
+ void prepareJobOutput(JobConf conf);
/**
* Get the UGI that the given job configuration will run as.