You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/06/04 03:21:35 UTC

svn commit: r781633 [1/13] - in /hadoop/hive/trunk: ./ common/src/java/org/apache/hadoop/hive/conf/ data/scripts/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ...

Author: zshao
Date: Thu Jun  4 01:21:30 2009
New Revision: 781633

URL: http://svn.apache.org/viewvc?rev=781633&view=rev
Log:
HIVE-195. Map-side join. (Namit Jain via zshao)

Added:
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinObject.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink4.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapJoinDesc.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredLocalWork.java
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/join2.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join25.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join26.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join27.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join28.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join29.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join30.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join31.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join32.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join33.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join34.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join35.q
    hadoop/hive/trunk/ql/src/test/results/clientnegative/join2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join25.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join26.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join27.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join28.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join29.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join30.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join31.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join32.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join33.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join34.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join35.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hadoop/hive/trunk/data/scripts/dumpdata_script.py
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/joinDesc.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/selectDesc.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerFactory.java
    hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
    hadoop/hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_noskew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_noskew.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby9.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join0.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join13.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join15.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join20.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join21.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join22.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/no_hooks.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup4.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_clusterby.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_gby_join.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_join2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_udf_case.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/regex_col.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_case_column_pruning.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union17.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union20.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/cast1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input8.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_part1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testxpath.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testxpath2.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml
    hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Jun  4 01:21:30 2009
@@ -39,6 +39,8 @@
     HIVE-509. Add create table like.
     (Zheng Shao via namit)
 
+    HIVE-195. Map-side join. (Namit Jain via zshao)
+
   IMPROVEMENTS
     HIVE-389. Option to build without ivy (jssarma)
 

Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jun  4 01:21:30 2009
@@ -113,6 +113,9 @@
     HIVEMAPSIDEAGGREGATE("hive.map.aggr", "true"),
     HIVEGROUPBYSKEW("hive.groupby.skewindata", "false"),
     HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000),
+    HIVEMAPJOINNUMROWSCACHE("hive.mapjoin.numrows", 1000),
+    HIVEMAPJOINROWSIZE("hive.mapjoin.size.key", 10000),
+    HIVEMAPJOINCACHEROWS("hive.mapjoin.cache.numrows", 10000),
     HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", 100000),
     HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", (float)0.5),
     

Modified: hadoop/hive/trunk/data/scripts/dumpdata_script.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/scripts/dumpdata_script.py?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/data/scripts/dumpdata_script.py (original)
+++ hadoop/hive/trunk/data/scripts/dumpdata_script.py Thu Jun  4 01:21:30 2009
@@ -1,4 +1,9 @@
+import sys
+
 for i in xrange(50):
    for j in xrange(5):
       for k in xrange(20022):      
          print 20000 * i + k
+
+for line in sys.stdin:
+  pass

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java Thu Jun  4 01:21:30 2009
@@ -39,10 +39,10 @@
   transient protected ObjectInspector standardRowInspector;
   transient int maxSize;
 
-  public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
-    super.initialize(hconf, reporter, inputObjInspector);
+  public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
     rowList = new ArrayList<Object> ();
     maxSize = conf.getBufferSize().intValue();
+    initializeChildren(hconf, reporter, inputObjInspector);
   }
 
   boolean firstRow = true;

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=781633&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Thu Jun  4 01:21:30 2009
@@ -0,0 +1,585 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+import java.util.Vector;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.joinCond;
+import org.apache.hadoop.hive.ql.plan.joinDesc;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * Join operator implementation.
+ */
+public class CommonJoinOperator<T extends joinDesc> extends Operator<T> implements Serializable {
+  private static final long serialVersionUID = 1L;
+  static final protected Log LOG = LogFactory.getLog(CommonJoinOperator.class.getName());
+
+  public static class IntermediateObject {
+    ArrayList<Object>[] objs;
+    int curSize;
+
+    public IntermediateObject(ArrayList<Object>[] objs, int curSize) {
+      this.objs = objs;
+      this.curSize = curSize;
+    }
+
+    public ArrayList<Object>[] getObjs() {
+      return objs;
+    }
+
+    public int getCurSize() {
+      return curSize;
+    }
+
+    public void pushObj(ArrayList<Object> obj) {
+      objs[curSize++] = obj;
+    }
+
+    public void popObj() {
+      curSize--;
+    }
+  }
+
+  transient protected int numValues; // number of aliases
+  transient protected Map<Byte, List<ExprNodeEvaluator>> joinValues;
+  transient protected Map<Byte, List<ObjectInspector>>   joinValuesObjectInspectors;
+  
+  transient static protected Byte[] order; // order in which the results should be output
+  transient protected joinCond[] condn;
+  transient protected boolean noOuterJoin;
+  transient private Object[] dummyObj; // for outer joins, contains the
+                                       // potential nulls for the concerned
+                                       // aliases
+  transient private Vector<ArrayList<Object>>[] dummyObjVectors;
+  transient private Stack<Iterator<ArrayList<Object>>> iterators;
+  transient protected int totalSz; // total size of the composite object
+  transient ObjectInspector joinOutputObjectInspector;
+  
+  // keys are the column names. basically this maps the position of the column in 
+  // the output of the CommonJoinOperator to the input columnInfo.
+  transient private Map<Integer, Set<String>> posToAliasMap;
+
+  HashMap<Byte, Vector<ArrayList<Object>>> storage;
+  int joinEmitInterval = -1;
+  int nextSz = 0;
+  transient Byte lastAlias = null;
+  
+  protected int populateJoinKeyValue(Map<Byte, List<ExprNodeEvaluator>> outMap,
+      Map<Byte, List<exprNodeDesc>> inputMap) {
+
+    int total = 0;
+
+    Iterator<Map.Entry<Byte, List<exprNodeDesc>>> entryIter = inputMap.entrySet().iterator();
+    while (entryIter.hasNext()) {
+      Map.Entry<Byte, List<exprNodeDesc>> e = (Map.Entry<Byte, List<exprNodeDesc>>) entryIter.next();
+      Byte key = (Byte) e.getKey();
+
+      List<exprNodeDesc> expr = (List<exprNodeDesc>) e.getValue();
+      int sz = expr.size();
+      total += sz;
+
+      List<ExprNodeEvaluator> valueFields = new ArrayList<ExprNodeEvaluator>();
+
+      for (int j = 0; j < sz; j++)
+        valueFields.add(ExprNodeEvaluatorFactory.get(expr.get(j)));
+
+      outMap.put(key, valueFields);
+    }
+    
+    return total;
+  }
+
+  public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+    LOG.info("COMMONJOIN " + ((StructObjectInspector)inputObjInspector[0]).getTypeName());   
+    totalSz = 0;
+    // Map that contains the rows for each alias
+    storage = new HashMap<Byte, Vector<ArrayList<Object>>>();
+
+    numValues = conf.getExprs().size();
+    
+    joinValues = new HashMap<Byte, List<ExprNodeEvaluator>>();
+    joinValuesObjectInspectors = new HashMap<Byte, List<ObjectInspector>>();
+
+    if (order == null) {
+      order = new Byte[numValues];
+      for (int i = 0; i < numValues; i++)
+        order[i] = (byte) i;
+    }
+    condn = conf.getConds();
+    noOuterJoin = conf.getNoOuterJoin();
+
+    totalSz = populateJoinKeyValue(joinValues, conf.getExprs());
+
+    dummyObj = new Object[numValues];
+    dummyObjVectors = new Vector[numValues];
+
+    int pos = 0;
+    for (Byte alias : order) {
+      int sz = conf.getExprs().get(alias).size();
+      ArrayList<Object> nr = new ArrayList<Object>(sz);
+
+      for (int j = 0; j < sz; j++)
+        nr.add(null);
+
+      dummyObj[pos] = nr;
+      Vector<ArrayList<Object>> values = new Vector<ArrayList<Object>>();
+      values.add((ArrayList<Object>) dummyObj[pos]);
+      dummyObjVectors[pos] = values;
+      pos++;
+    }
+
+    iterators = new Stack<Iterator<ArrayList<Object>>>();
+    
+    joinEmitInterval = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEJOINEMITINTERVAL);
+    
+    forwardCache = new Object[totalSz];
+  }
+
+  public void startGroup() throws HiveException {
+    LOG.trace("Join: Starting new group");
+    storage.clear();
+    for (Byte alias : order)
+      storage.put(alias, new Vector<ArrayList<Object>>());
+  }
+
+  private int getNextSize(int sz) {
+    // A very simple counter to keep track of join entries for a key
+    if (sz >= 100000)
+      return sz + 100000;
+    
+    return 2 * sz;
+  }
+
+  transient protected Byte alias;
+  
+  protected ArrayList<Object> computeValues(Object row, ObjectInspector rowInspector,
+    List<ExprNodeEvaluator> valueFields, Map<Byte, List<ObjectInspector>> joinExprsObjectInspectors) throws HiveException {
+    
+    // Get the valueFields Object Inspectors
+    List<ObjectInspector> valueFieldOI = joinExprsObjectInspectors.get(alias);
+    if (valueFieldOI == null) {
+      // Initialize the ExprEvaluator if necessary
+      valueFieldOI = new ArrayList<ObjectInspector>();
+      for (int i=0; i<valueFields.size(); i++) {
+        valueFieldOI.add(valueFields.get(i).initialize(rowInspector));
+      }
+      joinExprsObjectInspectors.put(alias, valueFieldOI);
+    }
+    
+    // Compute the values
+    ArrayList<Object> nr = new ArrayList<Object>(valueFields.size());
+    for (int i=0; i<valueFields.size(); i++) {
+      nr.add(ObjectInspectorUtils.copyToStandardObject(
+          valueFields.get(i).evaluate(row),
+          valueFieldOI.get(i)));
+    }
+    
+    return nr;
+  }
+  
+  public void process(Object row, ObjectInspector rowInspector, int tag)
+      throws HiveException {
+    try {
+      // get alias
+      alias = (byte)tag;
+
+      if ((lastAlias == null) || (!lastAlias.equals(alias)))
+        nextSz = joinEmitInterval;
+      
+      ArrayList<Object> nr = computeValues(row, rowInspector, joinValues.get(alias), joinValuesObjectInspectors);
+      
+      // number of rows for the key in the given table
+      int sz = storage.get(alias).size();
+
+      // Are we consuming too much memory
+      if (alias == numValues - 1) {
+        if (sz == joinEmitInterval) {
+          // The input is sorted by alias, so if we are already in the last join operand,
+          // we can emit some results now.
+          // Note this has to be done before adding the current row to the storage,
+          // to preserve the correctness for outer joins.
+          checkAndGenObject();
+          storage.get(alias).clear();
+        }
+      } else {
+        if (sz == nextSz) {
+          // Output a warning if we reached at least 1000 rows for a join operand
+          // We won't output a warning for the last join operand since the size
+          // will never goes to joinEmitInterval.
+          StructObjectInspector soi = (StructObjectInspector)rowInspector;
+          StructField sf = soi.getStructFieldRef(Utilities.ReduceField.KEY.toString());
+          Object keyObject = soi.getStructFieldData(row, sf);
+          LOG.warn("table " + alias + " has " + sz + " rows for join key " + keyObject);
+          nextSz = getNextSize(nextSz);
+        }
+      }
+
+      // Add the value to the vector
+      storage.get(alias).add(nr);
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw new HiveException(e);
+    }
+  }
+
+  transient Object[] forwardCache;
+  
+  private void createForwardJoinObject(IntermediateObject intObj,
+      boolean[] nullsArr) throws HiveException {
+    int p = 0;
+    for (int i = 0; i < numValues; i++) {
+      Byte alias = order[i];
+      int sz = joinValues.get(alias).size();
+      if (nullsArr[i]) {
+        for (int j = 0; j < sz; j++) {
+          forwardCache[p++] = null;
+        }
+      } else {
+        ArrayList<Object> obj = intObj.getObjs()[i];
+        for (int j = 0; j < sz; j++) {
+          forwardCache[p++] = obj.get(j);
+        }
+      }
+    }
+    forward(forwardCache, joinOutputObjectInspector);
+  }
+
+  private void copyOldArray(boolean[] src, boolean[] dest) {
+    for (int i = 0; i < src.length; i++)
+      dest[i] = src[i];
+  }
+
+  private Vector<boolean[]> joinObjectsInnerJoin(Vector<boolean[]> resNulls,
+      Vector<boolean[]> inputNulls, ArrayList<Object> newObj,
+      IntermediateObject intObj, int left, boolean newObjNull) {
+    if (newObjNull)
+      return resNulls;
+    Iterator<boolean[]> nullsIter = inputNulls.iterator();
+    while (nullsIter.hasNext()) {
+      boolean[] oldNulls = nullsIter.next();
+      boolean oldObjNull = oldNulls[left];
+      if (!oldObjNull) {
+        boolean[] newNulls = new boolean[intObj.getCurSize()];
+        copyOldArray(oldNulls, newNulls);
+        newNulls[oldNulls.length] = false;
+        resNulls.add(newNulls);
+      }
+    }
+    return resNulls;
+  }
+
+  private Vector<boolean[]> joinObjectsLeftOuterJoin(
+      Vector<boolean[]> resNulls, Vector<boolean[]> inputNulls,
+      ArrayList<Object> newObj, IntermediateObject intObj, int left,
+      boolean newObjNull) {
+    Iterator<boolean[]> nullsIter = inputNulls.iterator();
+    while (nullsIter.hasNext()) {
+      boolean[] oldNulls = nullsIter.next();
+      boolean oldObjNull = oldNulls[left];
+      boolean[] newNulls = new boolean[intObj.getCurSize()];
+      copyOldArray(oldNulls, newNulls);
+      if (oldObjNull)
+        newNulls[oldNulls.length] = true;
+      else
+        newNulls[oldNulls.length] = newObjNull;
+      resNulls.add(newNulls);
+    }
+    return resNulls;
+  }
+
+  private Vector<boolean[]> joinObjectsRightOuterJoin(
+      Vector<boolean[]> resNulls, Vector<boolean[]> inputNulls,
+      ArrayList<Object> newObj, IntermediateObject intObj, int left,
+      boolean newObjNull, boolean firstRow) {
+    if (newObjNull)
+      return resNulls;
+
+    if (inputNulls.isEmpty() && firstRow) {
+      boolean[] newNulls = new boolean[intObj.getCurSize()];
+      for (int i = 0; i < intObj.getCurSize() - 1; i++)
+        newNulls[i] = true;
+      newNulls[intObj.getCurSize()-1] = newObjNull;
+      resNulls.add(newNulls);
+      return resNulls;
+    }
+
+    boolean allOldObjsNull = firstRow;
+
+    Iterator<boolean[]> nullsIter = inputNulls.iterator();
+    while (nullsIter.hasNext()) {
+      boolean[] oldNulls = nullsIter.next();
+      if (!oldNulls[left]) {
+        allOldObjsNull = false;
+        break;
+      }
+    }
+
+    nullsIter = inputNulls.iterator();
+    while (nullsIter.hasNext()) {
+      boolean[] oldNulls = nullsIter.next();
+      boolean oldObjNull = oldNulls[left];
+
+      if (!oldObjNull) {
+        boolean[] newNulls = new boolean[intObj.getCurSize()];
+        copyOldArray(oldNulls, newNulls);
+        newNulls[oldNulls.length] = newObjNull;
+        resNulls.add(newNulls);
+      } else if (allOldObjsNull) {
+        boolean[] newNulls = new boolean[intObj.getCurSize()];
+        for (int i = 0; i < intObj.getCurSize() - 1; i++)
+          newNulls[i] = true;
+        newNulls[oldNulls.length] = newObjNull;
+        resNulls.add(newNulls);
+        return resNulls;
+      }
+    }
+    return resNulls;
+  }
+
+  private Vector<boolean[]> joinObjectsFullOuterJoin(
+      Vector<boolean[]> resNulls, Vector<boolean[]> inputNulls,
+      ArrayList<Object> newObj, IntermediateObject intObj, int left,
+      boolean newObjNull, boolean firstRow) {
+    if (newObjNull) {
+      Iterator<boolean[]> nullsIter = inputNulls.iterator();
+      while (nullsIter.hasNext()) {
+        boolean[] oldNulls = nullsIter.next();
+        boolean[] newNulls = new boolean[intObj.getCurSize()];
+        copyOldArray(oldNulls, newNulls);
+        newNulls[oldNulls.length] = newObjNull;
+        resNulls.add(newNulls);
+      }
+      return resNulls;
+    }
+
+    if (inputNulls.isEmpty() && firstRow) {
+      boolean[] newNulls = new boolean[intObj.getCurSize()];
+      for (int i = 0; i < intObj.getCurSize() - 1; i++)
+        newNulls[i] = true;
+      newNulls[intObj.getCurSize()-1] = newObjNull;
+      resNulls.add(newNulls);
+      return resNulls;
+    }
+
+    boolean allOldObjsNull = firstRow;
+
+    Iterator<boolean[]> nullsIter = inputNulls.iterator();
+    while (nullsIter.hasNext()) {
+      boolean[] oldNulls = nullsIter.next();
+      if (!oldNulls[left]) {
+        allOldObjsNull = false;
+        break;
+      }
+    }
+    boolean rhsPreserved = false;
+
+    nullsIter = inputNulls.iterator();
+    while (nullsIter.hasNext()) {
+      boolean[] oldNulls = nullsIter.next();
+      boolean oldObjNull = oldNulls[left];
+
+      if (!oldObjNull) {
+        boolean[] newNulls = new boolean[intObj.getCurSize()];
+        copyOldArray(oldNulls, newNulls);
+        newNulls[oldNulls.length] = newObjNull;
+        resNulls.add(newNulls);
+      } else if (oldObjNull) {
+        boolean[] newNulls = new boolean[intObj.getCurSize()];
+        copyOldArray(oldNulls, newNulls);
+        newNulls[oldNulls.length] = true;
+        resNulls.add(newNulls);
+
+        if (allOldObjsNull && !rhsPreserved) {
+          newNulls = new boolean[intObj.getCurSize()];
+          for (int i = 0; i < oldNulls.length; i++)
+            newNulls[i] = true;
+          newNulls[oldNulls.length] = false;
+          resNulls.add(newNulls);
+          rhsPreserved = true;
+        }
+      }
+    }
+    return resNulls;
+  }
+
+  /*
+   * The new input is added to the list of existing inputs. Each entry in the
+   * array of inputNulls denotes the entries in the intermediate object to be
+   * used. The intermediate object is augmented with the new object, and list of
+   * nulls is changed appropriately. The list will contain all non-nulls for a
+   * inner join. The outer joins are processed appropriately.
+   */
+  private Vector<boolean[]> joinObjects(Vector<boolean[]> inputNulls,
+                                        ArrayList<Object> newObj, IntermediateObject intObj, 
+                                        int joinPos, boolean firstRow) {
+    Vector<boolean[]> resNulls = new Vector<boolean[]>();
+    boolean newObjNull = newObj == dummyObj[joinPos] ? true : false;
+    if (joinPos == 0) {
+      if (newObjNull)
+        return null;
+      boolean[] nulls = new boolean[1];
+      nulls[0] = newObjNull;
+      resNulls.add(nulls);
+      return resNulls;
+    }
+
+    int left = condn[joinPos - 1].getLeft();
+    int type = condn[joinPos - 1].getType();
+
+    // process all nulls for RIGHT and FULL OUTER JOINS
+    if (((type == joinDesc.RIGHT_OUTER_JOIN) || (type == joinDesc.FULL_OUTER_JOIN))
+        && !newObjNull && (inputNulls == null) && firstRow) {
+      boolean[] newNulls = new boolean[intObj.getCurSize()];
+      for (int i = 0; i < newNulls.length - 1; i++)
+        newNulls[i] = true;
+      newNulls[newNulls.length - 1] = false;
+      resNulls.add(newNulls);
+      return resNulls;
+    }
+
+    if (inputNulls == null)
+      return null;
+
+    if (type == joinDesc.INNER_JOIN)
+      return joinObjectsInnerJoin(resNulls, inputNulls, newObj, intObj, left,
+          newObjNull);
+    else if (type == joinDesc.LEFT_OUTER_JOIN)
+      return joinObjectsLeftOuterJoin(resNulls, inputNulls, newObj, intObj,
+          left, newObjNull);
+    else if (type == joinDesc.RIGHT_OUTER_JOIN)
+      return joinObjectsRightOuterJoin(resNulls, inputNulls, newObj, intObj,
+                                       left, newObjNull, firstRow);
+    assert (type == joinDesc.FULL_OUTER_JOIN);
+    return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left,
+                                    newObjNull, firstRow);
+  }
+
+  /*
+   * genObject is a recursive function. For the inputs, a array of bitvectors is
+   * maintained (inputNulls) where each entry denotes whether the element is to
+   * be used or not (whether it is null or not). The size of the bitvector is
+   * same as the number of inputs under consideration currently. When all inputs
+   * are accounted for, the output is forwared appropriately.
+   */
+  private void genObject(Vector<boolean[]> inputNulls, int aliasNum,
+                         IntermediateObject intObj, boolean firstRow) throws HiveException {
+    boolean childFirstRow = firstRow;
+    if (aliasNum < numValues) {
+      Iterator<ArrayList<Object>> aliasRes = storage.get(order[aliasNum])
+          .iterator();
+      iterators.push(aliasRes);
+      while (aliasRes.hasNext()) {
+        ArrayList<Object> newObj = aliasRes.next();
+        intObj.pushObj(newObj);
+        Vector<boolean[]> newNulls = joinObjects(inputNulls, newObj, intObj,
+                                                 aliasNum, childFirstRow);
+        genObject(newNulls, aliasNum + 1, intObj, firstRow);
+        intObj.popObj();
+        firstRow = false;
+      }
+      iterators.pop();
+    } else {
+      if (inputNulls == null)
+        return;
+      Iterator<boolean[]> nullsIter = inputNulls.iterator();
+      while (nullsIter.hasNext()) {
+        boolean[] nullsVec = nullsIter.next();
+        createForwardJoinObject(intObj, nullsVec);
+      }
+    }
+  }
+
+  /**
+   * Forward a record of join results.
+   * 
+   * @throws HiveException
+   */
+  public void endGroup() throws HiveException {
+    LOG.trace("Join Op: endGroup called: numValues=" + numValues);
+    checkAndGenObject();
+  }
+
+  protected void checkAndGenObject() throws HiveException {
+    // does any result need to be emitted
+    for (int i = 0; i < numValues; i++) {
+      Byte alias = order[i];
+      if (storage.get(alias).iterator().hasNext() == false) {
+        if (noOuterJoin) {
+          LOG.trace("No data for alias=" + i);
+          return;
+        } else {
+          storage.put(alias, dummyObjVectors[i]);
+        }
+      }
+    }
+
+    LOG.trace("calling genObject");
+    genObject(null, 0, new IntermediateObject(new ArrayList[numValues], 0), true);
+    LOG.trace("called genObject");
+  }
+
+  /**
+   * All done
+   * 
+   */
+  public void close(boolean abort) throws HiveException {
+    LOG.trace("Join Op close");
+    super.close(abort);
+  }
+
+  @Override
+  public String getName() {
+    return "JOIN";
+  }
+
+  /**
+   * @return the posToAliasMap
+   */
+  public Map<Integer, Set<String>> getPosToAliasMap() {
+    return posToAliasMap;
+  }
+
+  /**
+   * @param posToAliasMap the posToAliasMap to set
+   */
+  public void setPosToAliasMap(Map<Integer, Set<String>> posToAliasMap) {
+    this.posToAliasMap = posToAliasMap;
+  }
+  
+}

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Thu Jun  4 01:21:30 2009
@@ -649,14 +649,7 @@
       pathToAliases.put(newPath.toString(), newList);
       
       Map<String,partitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo();
-      partitionDesc pDesc = new partitionDesc();
-      
-      tableDesc tDesc = new tableDesc(LazySimpleSerDe.class,
-                                      SequenceFileInputFormat.class,
-                                      SequenceFileOutputFormat.class,
-                                      new Properties());
-      
-      pDesc.setTableDesc(tDesc);
+      partitionDesc pDesc = work.getAliasToPartnInfo().get(oneAlias);
       pathToPartitionInfo.put(newPath.toString(), pDesc);
       
       RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newPath, Text.class, false, new Properties(), null);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Thu Jun  4 01:21:30 2009
@@ -27,24 +27,61 @@
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.hive.ql.plan.mapredWork;
+import org.apache.hadoop.hive.ql.plan.mapredLocalWork;
+import org.apache.hadoop.hive.ql.plan.fetchWork;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 
 public class ExecMapper extends MapReduceBase implements Mapper {
 
   private MapOperator mo;
+  private Map<String, FetchOperator> fetchOperators;
   private OutputCollector oc;
   private JobConf jc;
   private boolean abort = false;
   private Reporter rp;
   public static final Log l4j = LogFactory.getLog("ExecMapper");
   private static boolean done;
+
+  private void init() {
+    mo = null;
+    fetchOperators = null;
+    oc = null;
+    jc = null;
+    abort = false;
+    rp = null;
+  }
   
   public void configure(JobConf job) {
-    jc = job;
-    mapredWork mrwork = Utilities.getMapRedWork(job);
-    mo = new MapOperator ();
-    mo.setConf(mrwork);
-    // we don't initialize the operator until we have set the output collector
+    try {
+      init();
+      jc = job;
+      mapredWork mrwork = Utilities.getMapRedWork(job);
+      mo = new MapOperator();
+      mo.setConf(mrwork);
+      mapredLocalWork mlo = mrwork.getMapLocalWork();
+      if (mlo != null) {
+        fetchOperators = new HashMap<String, FetchOperator>();
+        Map<String, fetchWork> aliasToFetchWork = mlo.getAliasToFetchWork();
+        Iterator<Map.Entry<String, fetchWork>> fetchWorkSet = aliasToFetchWork
+            .entrySet().iterator();
+        while (fetchWorkSet.hasNext()) {
+          Map.Entry<String, fetchWork> entry = fetchWorkSet.next();
+          String alias = entry.getKey();
+          fetchWork fWork = entry.getValue();
+          fetchOperators.put(alias, new FetchOperator(fWork, job));
+          l4j.info("fetchoperator for " + alias + " initialized");
+        }
+      }
+
+      // we don't initialize the operator until we have set the output collector
+    } catch (Exception e) {
+      // Bail out ungracefully - we should never hit
+      // this here - but would have hit it in SemanticAnalyzer
+      throw new RuntimeException(e);
+    }
   }
 
   public void map(Object key, Object value,
@@ -55,6 +92,37 @@
         oc = output;
         mo.setOutputCollector(oc);
         mo.initialize(jc, reporter, null);
+        if (fetchOperators != null) {
+          mapredWork mrwork = Utilities.getMapRedWork(jc);
+          mapredLocalWork localWork = mrwork.getMapLocalWork();
+          Iterator<Map.Entry<String, FetchOperator>> fetchOps = fetchOperators.entrySet().iterator();
+          while (fetchOps.hasNext()) {
+            Map.Entry<String, FetchOperator> entry = fetchOps.next();
+            String alias = entry.getKey();
+            FetchOperator fetchOp = entry.getValue();
+            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias); 
+            // All the operators need to be initialized before process
+            forwardOp.initialize(jc, reporter, new ObjectInspector[]{fetchOp.getOutputObjectInspector()});
+          }
+
+          fetchOps = fetchOperators.entrySet().iterator();
+          while (fetchOps.hasNext()) {
+            Map.Entry<String, FetchOperator> entry = fetchOps.next();
+            String alias = entry.getKey();
+            FetchOperator fetchOp = entry.getValue();
+            Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias); 
+
+            while (true) {
+              InspectableObject row = fetchOp.getNextRow();
+              if (row == null) {
+                break;
+              }
+
+              forwardOp.process(row.o, row.oi, 0);
+            }
+          }
+        }
+
         rp = reporter;
       } catch (HiveException e) {
         abort = true;
@@ -94,6 +162,18 @@
     // ideally hadoop should let us know whether map execution failed or not
     try {
       mo.close(abort);
+      if (fetchOperators != null) {
+        mapredWork mrwork = Utilities.getMapRedWork(jc);
+        mapredLocalWork localWork = mrwork.getMapLocalWork();
+        Iterator<Map.Entry<String, FetchOperator>> fetchOps = fetchOperators.entrySet().iterator();
+        while (fetchOps.hasNext()) {
+          Map.Entry<String, FetchOperator> entry = fetchOps.next();
+          String alias = entry.getKey();
+          Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias); 
+          forwardOp.close(abort);
+        }
+      }
+
       reportStats rps = new reportStats (rp);
       mo.preorderMap(rps);
       return;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java Thu Jun  4 01:21:30 2009
@@ -35,16 +35,14 @@
   private static final long serialVersionUID = 1L;
   transient protected ExprNodeEvaluator eval;
 
-  public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
-    super.initialize(hconf, reporter, inputObjInspector);
+  public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
     eval = ExprNodeEvaluatorFactory.get(conf.getCol());
+    outputRowInspector = eval.initialize(inputObjInspector[0]);
+    initializeChildren(hconf, reporter, new ObjectInspector[]{outputRowInspector});
   }
 
   ObjectInspector outputRowInspector;
   public void process(Object row, ObjectInspector rowInspector, int tag) throws HiveException {
-    if (outputRowInspector == null) {
-      outputRowInspector = eval.initialize(rowInspector);
-    }
     forward(eval.evaluate(row), outputRowInspector);
   }
 

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=781633&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Thu Jun  4 01:21:30 2009
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.fetchWork;
+import org.apache.hadoop.hive.ql.plan.partitionDesc;
+import org.apache.hadoop.hive.ql.plan.tableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+/**
+ * FetchTask implementation
+ **/
+public class FetchOperator {
+
+  transient protected Log LOG;
+  transient protected LogHelper console;
+
+  public FetchOperator(fetchWork work, JobConf job) {
+    LOG = LogFactory.getLog(this.getClass().getName());
+    console = new LogHelper(LOG);
+
+    this.work = work;
+    this.job = job;
+    
+    currRecReader = null;
+    currPath = null;
+    currTbl = null;
+    currPart = null;
+    iterPath = null;
+    iterPartDesc = null;
+    tblDataDone = false;
+    rowWithPart = new Object[2];
+  }
+
+  private fetchWork work;
+  private int splitNum;
+  private RecordReader<WritableComparable, Writable> currRecReader;
+  private InputSplit[] inputSplits;
+  private InputFormat inputFormat;
+  private JobConf job;
+  private WritableComparable key;
+  private Writable value;
+  private Deserializer serde;
+  private Iterator<Path> iterPath;
+  private Iterator<partitionDesc> iterPartDesc;
+  private Path currPath;
+  private partitionDesc currPart;
+  private tableDesc currTbl;
+  private boolean tblDataDone;
+  private StructObjectInspector rowObjectInspector;
+  private Object[] rowWithPart;
+
+  /**
+   * A cache of InputFormat instances.
+   */
+  private static Map<Class, InputFormat<WritableComparable, Writable>> inputFormats = new HashMap<Class, InputFormat<WritableComparable, Writable>>();
+
+  static InputFormat<WritableComparable, Writable> getInputFormatFromCache(
+      Class inputFormatClass, Configuration conf) throws IOException {
+    if (!inputFormats.containsKey(inputFormatClass)) {
+      try {
+        InputFormat<WritableComparable, Writable> newInstance = (InputFormat<WritableComparable, Writable>) ReflectionUtils
+            .newInstance(inputFormatClass, conf);
+        inputFormats.put(inputFormatClass, newInstance);
+      } catch (Exception e) {
+        throw new IOException("Cannot create an instance of InputFormat class "
+          + inputFormatClass.getName() + " as specified in mapredWork!");
+      }
+    }
+    return inputFormats.get(inputFormatClass);
+  }
+
+  private void setPrtnDesc() throws Exception {
+    List<String> partNames = new ArrayList<String>();
+    List<String> partValues = new ArrayList<String>();
+
+    String pcols = currPart
+        .getTableDesc()
+        .getProperties()
+        .getProperty(
+            org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS);
+    LinkedHashMap<String, String> partSpec = currPart.getPartSpec();
+
+    List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
+    String[] partKeys = pcols.trim().split("/");
+    for (String key : partKeys) {
+      partNames.add(key);
+      partValues.add(partSpec.get(key));
+      partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+    }
+    StructObjectInspector partObjectInspector = ObjectInspectorFactory
+        .getStandardStructObjectInspector(partNames, partObjectInspectors);
+    rowObjectInspector = (StructObjectInspector) serde.getObjectInspector();
+
+    rowWithPart[1] = partValues;
+    rowObjectInspector = ObjectInspectorFactory
+        .getUnionStructObjectInspector(Arrays
+            .asList(new StructObjectInspector[] { rowObjectInspector,
+                partObjectInspector }));
+  }
+
+  private void getNextPath() throws Exception {
+    // first time
+    if (iterPath == null) {
+      if (work.getTblDir() != null) {
+        if (!tblDataDone) {
+          currPath = work.getTblDirPath();
+          currTbl = work.getTblDesc();
+          FileSystem fs = currPath.getFileSystem(job);
+          if (fs.exists(currPath)) {
+            FileStatus[] fStats = fs.listStatus(currPath);
+            for (FileStatus fStat : fStats) {
+              if (fStat.getLen() > 0) {
+                tblDataDone = true;
+                break;
+              }
+            }
+          }
+
+          if (!tblDataDone)
+            currPath = null;
+          return;
+        } else {
+          currTbl = null;
+          currPath = null;
+        }
+        return;
+      } else {
+        iterPath = fetchWork.convertStringToPathArray(work.getPartDir()).iterator();
+        iterPartDesc = work.getPartDesc().iterator();
+      }
+    }
+
+    while (iterPath.hasNext()) {
+      Path nxt = iterPath.next();
+      partitionDesc prt = iterPartDesc.next();
+      FileSystem fs = nxt.getFileSystem(job);
+      if (fs.exists(nxt)) {
+        FileStatus[] fStats = fs.listStatus(nxt);
+        for (FileStatus fStat : fStats) {
+          if (fStat.getLen() > 0) {
+            currPath = nxt;
+            currPart = prt;
+            return;
+          }
+        }
+      }
+    }
+  }
+
+  private RecordReader<WritableComparable, Writable> getRecordReader()
+      throws Exception {
+    if (currPath == null) {
+      getNextPath();
+      if (currPath == null)
+        return null;
+
+      // not using FileInputFormat.setInputPaths() here because it forces a connection
+      // to the default file system - which may or may not be online during pure metadata
+      // operations
+      job.set("mapred.input.dir",
+              org.apache.hadoop.util.StringUtils.escapeString(currPath.toString()));
+      
+      tableDesc tmp = currTbl;
+      if (tmp == null)
+        tmp = currPart.getTableDesc();
+      inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job);
+      inputSplits = inputFormat.getSplits(job, 1);
+      splitNum = 0;
+      serde = tmp.getDeserializerClass().newInstance();
+      serde.initialize(job, tmp.getProperties());
+      LOG.debug("Creating fetchTask with deserializer typeinfo: "
+        + serde.getObjectInspector().getTypeName());
+      LOG.debug("deserializer properties: " + tmp.getProperties());
+      if (!tblDataDone)
+        setPrtnDesc();
+    }
+
+    if (splitNum >= inputSplits.length) {
+      if (currRecReader != null) {
+        currRecReader.close();
+        currRecReader = null;
+      }
+      currPath = null;
+      return getRecordReader();
+    }
+
+    currRecReader = inputFormat.getRecordReader(inputSplits[splitNum++], job,
+        Reporter.NULL);
+    key = currRecReader.createKey();
+    value = currRecReader.createValue();
+    return currRecReader;
+  }
+
+  /**
+   * Get the next row. The fetch context is modified appropriately.
+   * 
+   * @param ctx
+   *          fetch context
+   **/
+  public InspectableObject getNextRow() {
+    try {
+      if (currRecReader == null) {
+        currRecReader = getRecordReader();
+        if (currRecReader == null)
+          return null;
+      }
+
+      boolean ret = currRecReader.next(key, value);
+      if (ret) {
+        if (tblDataDone) {
+          Object obj = serde.deserialize(value);
+          return new InspectableObject(obj, serde.getObjectInspector());
+        } else {
+          rowWithPart[0] = serde.deserialize(value);
+          return new InspectableObject(rowWithPart, rowObjectInspector);
+        }
+      } else {
+        currRecReader.close();
+        currRecReader = null;
+        currRecReader = getRecordReader();
+        if (currRecReader == null)
+          return null;
+        else
+          return getNextRow();
+      }
+    } catch (Exception e) {
+      return null;
+    }
+  }
+
+  /**
+   * Clear the context, if anything needs to be done.
+   * 
+   * @param ctx
+   *          fetch context
+   **/
+  public void clearFetchContext() throws HiveException {
+    try {
+      if (currRecReader != null) {
+        currRecReader.close();
+        currRecReader = null;
+      }
+    } catch (Exception e) {
+      throw new HiveException("Failed with exception " + e.getMessage()
+          + org.apache.hadoop.util.StringUtils.stringifyException(e));
+    }
+  }
+
+  public ObjectInspector getOutputObjectInspector() throws HiveException {
+    try {
+      ObjectInspector outInspector;
+      if (work.getTblDir() != null) {
+        tableDesc tbl = work.getTblDesc();
+        Deserializer serde = tbl.getDeserializerClass().newInstance();
+        serde.initialize(job, tbl.getProperties());
+        return serde.getObjectInspector();
+      }
+      else {
+        List<partitionDesc> listParts = work.getPartDesc();
+        currPart = listParts.get(0);
+        serde = currPart.getTableDesc().getDeserializerClass().newInstance();
+        serde.initialize(job, currPart.getTableDesc().getProperties());
+        setPrtnDesc();
+        currPart = null;
+        return rowObjectInspector;
+      }
+    } catch (Exception e) {
+      throw new HiveException("Failed with exception " + e.getMessage()
+          + org.apache.hadoop.util.StringUtils.stringifyException(e));
+    }
+  }
+}

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Thu Jun  4 01:21:30 2009
@@ -57,6 +57,7 @@
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
 
 /**
  * FetchTask implementation
@@ -65,28 +66,24 @@
   private static final long serialVersionUID = 1L;
 
   private int maxRows = 100;
+  private FetchOperator ftOp;
+ 	private LazySimpleSerDe mSerde;
+ 	private int totalRows;
   
   public void initialize (HiveConf conf) {
     super.initialize(conf);
-    currRecReader = null;
     
-    try {
-      job = new JobConf(conf, ExecDriver.class);
-       
-      mSerde = new LazySimpleSerDe();
-      Properties mSerdeProp = new Properties();
-      mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
-      mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, ((fetchWork)work).getSerializationNullFormat());
-      mSerde.initialize(job, mSerdeProp);
+   	try {
+       // Create a file system handle  
+       JobConf job = new JobConf(conf, ExecDriver.class);
+
+       mSerde = new LazySimpleSerDe();
+       Properties mSerdeProp = new Properties();
+       mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
+       mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, ((fetchWork)work).getSerializationNullFormat());
+       mSerde.initialize(job, mSerdeProp);
        
-      currPath = null;
-      currTbl = null;
-      currPart = null;
-      iterPath = null;
-      iterPartDesc = null;
-      totalRows = 0;
-      tblDataDone = false;
-      rowWithPart = new Object[2];
+       ftOp = new FetchOperator(work, job);
     } catch (Exception e) {
       // Bail out ungracefully - we should never hit
       // this here - but would have hit it in SemanticAnalyzer
@@ -107,158 +104,6 @@
   }
 
   /**
-   * A cache of InputFormat instances.
-   */
-  private static Map<Class, InputFormat<WritableComparable, Writable>> inputFormats =
-    new HashMap<Class, InputFormat<WritableComparable, Writable>>();
-  
-  static InputFormat<WritableComparable, Writable> getInputFormatFromCache(Class inputFormatClass, Configuration conf) throws IOException {
-    if (!inputFormats.containsKey(inputFormatClass)) {
-      try {
-        InputFormat<WritableComparable, Writable> newInstance =
-          (InputFormat<WritableComparable, Writable>)ReflectionUtils.newInstance(inputFormatClass, conf);
-        inputFormats.put(inputFormatClass, newInstance);
-      } catch (Exception e) {
-        throw new IOException("Cannot create an instance of InputFormat class " + inputFormatClass.getName()
-                              + " as specified in mapredWork!");
-      }
-    }
-    return inputFormats.get(inputFormatClass);
-  }
-  
-  private int splitNum;
-  private RecordReader<WritableComparable, Writable> currRecReader;
-  private InputSplit[] inputSplits;
-  private InputFormat  inputFormat;
-  private JobConf      job;
-  private WritableComparable key; 
-  private Writable value;
-  private Deserializer  serde;
-  private LazySimpleSerDe mSerde;
-  private int totalRows;
-  private Iterator<Path> iterPath;
-  private Iterator<partitionDesc> iterPartDesc; 
-  private Path currPath;
-  private partitionDesc currPart;
-  private tableDesc     currTbl;
-  private boolean       tblDataDone;
-  private StructObjectInspector rowObjectInspector;
-  private Object[] rowWithPart;
-
-  private void setPrtnDesc() throws Exception {
-    List<String> partNames = new ArrayList<String>();
-    List<String> partValues = new ArrayList<String>();
-    
-    String pcols = currPart.getTableDesc().getProperties().getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS);
-    LinkedHashMap<String, String> partSpec = currPart.getPartSpec();
-    
-    List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
-    String[] partKeys = pcols.trim().split("/");
-    for(String key: partKeys) {
-      partNames.add(key);
-      partValues.add(partSpec.get(key));
-      partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
-    }
-    StructObjectInspector partObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
-    rowObjectInspector = (StructObjectInspector)serde.getObjectInspector();
-    
-    rowWithPart[1] = partValues;
-    rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(Arrays.asList(new StructObjectInspector[]{
-          rowObjectInspector, partObjectInspector}));
-  }
-
-  private void getNextPath() throws Exception {
-    // first time
-    if (iterPath == null) {
-      if (work.getTblDir() != null) {
-        if (!tblDataDone) {
-          currPath = work.getTblDir();
-          currTbl = work.getTblDesc();
-          FileSystem fs = currPath.getFileSystem(conf);
-          if (fs.exists(currPath)) 
-            {
-              FileStatus[] fStats = fs.listStatus(currPath);
-              for (FileStatus fStat:fStats) {
-                if (fStat.getLen() > 0) {
-                  tblDataDone = true;
-                  break;
-                }
-              }
-            }
-
-          if (!tblDataDone) currPath = null;
-          return;
-        } else {
-          currTbl = null;
-          currPath = null;
-        }
-        return;
-      } else {
-        iterPath = work.getPartDir().iterator();
-        iterPartDesc = work.getPartDesc().iterator();
-      }
-    }
-
-    while (iterPath.hasNext()) {
-      Path nxt = iterPath.next();
-      partitionDesc prt = iterPartDesc.next();
-      FileSystem fs = nxt.getFileSystem(conf);
-      if (fs.exists(nxt)) 
-        {
-          FileStatus[] fStats = fs.listStatus(nxt);
-          for (FileStatus fStat:fStats) {
-            if (fStat.getLen() > 0) {
-              currPath = nxt;
-              currPart = prt;
-              return;
-            }
-          }
-        }
-    }
-  }
-  
-  private RecordReader<WritableComparable, Writable> getRecordReader() throws Exception {
-    if (currPath == null) {
-      getNextPath();
-      if (currPath == null)
-        return null;
-
-      // not using FileInputFormat.setInputPaths() here because it forces a connection
-      // to the default file system - which may or may not be online during pure metadata
-      // operations
-      job.set("mapred.input.dir",
-              org.apache.hadoop.util.StringUtils.escapeString(currPath.toString()));
-
-      tableDesc tmp = currTbl;
-      if (tmp == null)
-        tmp = currPart.getTableDesc();
-      inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job);
-      inputSplits = inputFormat.getSplits(job, 1); 		
-      splitNum = 0;
-      serde = tmp.getDeserializerClass().newInstance();
-      serde.initialize(job, tmp.getProperties());
-      LOG.debug("Creating fetchTask with deserializer typeinfo: " + serde.getObjectInspector().getTypeName());
-      LOG.debug("deserializer properties: " + tmp.getProperties());
-      if (!tblDataDone)
-        setPrtnDesc();
-    }
- 		
-    if (splitNum >= inputSplits.length) {
-      if (currRecReader != null) {
-        currRecReader.close();
-        currRecReader = null;
-      }
-      currPath = null;
-      return getRecordReader();
-    }
- 		
-    currRecReader = inputFormat.getRecordReader(inputSplits[splitNum++], job, Reporter.NULL);
-    key = currRecReader.createKey();
-    value = currRecReader.createValue();
-    return currRecReader;
-  }
- 
-  /**
    * Return the maximum number of rows returned by fetch
    */
   public int getMaxRows() {
@@ -279,50 +124,22 @@
       if ((work.getLimit() >= 0) && ((work.getLimit() - totalRows) < rowsRet))
         rowsRet = work.getLimit() - totalRows;
       if (rowsRet <= 0) {
-        if (currRecReader != null) {
-          currRecReader.close();
-          currRecReader = null;
-        }
+        ftOp.clearFetchContext();
         return false;
       }
 
-      while (numRows < rowsRet) {
-        if (currRecReader == null) {
-          currRecReader = getRecordReader();
-          if (currRecReader == null) {
-            if (numRows == 0) 
-              return false;
-            totalRows += numRows;
-            return true;
-          }
-        }
-      	boolean ret = currRecReader.next(key, value);
-        if (ret) {
-          if (tblDataDone) {
-            Object obj = serde.deserialize(value);
-            res.add(((Text)mSerde.serialize(obj, serde.getObjectInspector())).toString());
-          } else {
-            rowWithPart[0] = serde.deserialize(value);
-            res.add(((Text)mSerde.serialize(rowWithPart, rowObjectInspector)).toString());
-          }
-          numRows++;
-        }
-        else {
-          currRecReader.close();
-          currRecReader = null;
-          currRecReader = getRecordReader();
-          if (currRecReader == null) {
-            if (numRows == 0) 
-              return false;
-            totalRows += numRows;
-            return true;
-          }
-          else {
-            key = currRecReader.createKey();
-            value = currRecReader.createValue();
-          }
-      	}
-      }
+    	while (numRows < rowsRet) {
+    	  InspectableObject io = ftOp.getNextRow();
+    	  if (io == null) {
+          if (numRows == 0) 
+            return false;
+          totalRows += numRows;
+          return true;
+    	  } 
+  	    
+    	  res.add(((Text)mSerde.serialize(io.o, io.oi)).toString());
+   	    numRows++;
+    	}
       totalRows += numRows;
       return true;
     }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Thu Jun  4 01:21:30 2009
@@ -75,6 +75,10 @@
   }
 
   public void close(boolean abort) throws HiveException {
+    if (state == state.CLOSE) 
+      return;
+
+    state = state.CLOSE;
     if(!abort) {
       if (outWriter != null) {
         try {
@@ -96,9 +100,7 @@
     }
   }
 
-  public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
-    super.initialize(hconf, reporter, inputObjInspector);
-    
+  public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
     try {
       serializer = (Serializer)conf.getTableInfo().getDeserializerClass().newInstance();
       serializer.initialize(null, conf.getTableInfo().getProperties());
@@ -157,6 +159,8 @@
       e.printStackTrace();
       throw new HiveException(e);
     }
+    
+    initializeChildren(hconf, reporter, inputObjInspector);
   }
 
   public static RecordWriter getRecordWriter(JobConf jc, HiveOutputFormat<?, ?> hiveOutputFormat,

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java Thu Jun  4 01:21:30 2009
@@ -47,13 +47,14 @@
     passed_count = new LongWritable();
   }
 
-  public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
-    super.initialize(hconf, reporter, inputObjInspector);
+  public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+ 
     try {
       this.conditionEvaluator = ExprNodeEvaluatorFactory.get(conf.getPredicate());
       statsMap.put(Counter.FILTERED, filtered_count);
       statsMap.put(Counter.PASSED, passed_count);
       conditionInspector = null;
+      initializeChildren(hconf, reporter, inputObjInspector);
     } catch (Throwable e) {
       e.printStackTrace();
       throw new RuntimeException(e);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java Thu Jun  4 01:21:30 2009
@@ -32,8 +32,8 @@
  **/
 public class ForwardOperator extends  Operator<forwardDesc>  implements Serializable {
   private static final long serialVersionUID = 1L;
-  public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
-    super.initialize(hconf, reporter, inputObjInspector);
+  public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+    initializeChildren(hconf, reporter, inputObjInspector);
     // nothing to do really ..
   }
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Thu Jun  4 01:21:30 2009
@@ -131,8 +131,8 @@
   transient int           numEntriesVarSize;
   transient int           numEntriesHashTable;
 
-  public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
-    super.initialize(hconf, reporter, inputObjInspector);
+  public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+
     totalMemory = Runtime.getRuntime().totalMemory();
     numRowsInput = 0;
     numRowsHashTbl = 0;
@@ -245,6 +245,14 @@
     for(int i=0; i<objectInspectors.size(); i++) {
       fieldNames.add(Integer.valueOf(i).toString());
     }
+
+    for (int i = 0; i < keyFields.length; i++) {
+      if (keyObjectInspectors[i] == null) {
+        keyObjectInspectors[i] = keyFields[i].initialize(inputObjInspector[0]);
+      }
+      objectInspectors.set(i, keyObjectInspectors[i]);
+    }
+    
     outputObjectInspector = 
       ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, objectInspectors);
     
@@ -253,6 +261,8 @@
     // is not known, estimate that based on the number of entries
     if (conf.getMode() == groupByDesc.Mode.HASH)
       computeMaxEntriesHashAggr(hconf);
+    
+    initializeChildren(hconf, reporter, new ObjectInspector[]{outputObjectInspector});
   }
 
   /**
@@ -460,6 +470,7 @@
   }
   
   public void process(Object row, ObjectInspector rowInspector, int tag) throws HiveException {
+    firstRow = false;
     // Total number of input rows is needed for hash aggregation only
     if (hashAggr) {
       numRowsInput++;
@@ -486,16 +497,6 @@
         }
         keyObjects[i] = keyFields[i].evaluate(row);
         newKeys.add(keyObjects[i]);
-        if (firstRow) {
-          objectInspectors.set(i, keyObjectInspectors[i]);
-        }
-      }
-
-      if (firstRow) {
-        firstRow = false;
-        // After first row, we can compute outputObjectInspector based on keyObjectInspectors.
-        outputObjectInspector = 
-          ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, objectInspectors);
       }
 
       if (hashAggr)