You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/06/04 03:21:35 UTC
svn commit: r781633 [1/13] - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ data/scripts/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/java/org/apache/hadoop/hive/ql/parse/ ...
Author: zshao
Date: Thu Jun 4 01:21:30 2009
New Revision: 781633
URL: http://svn.apache.org/viewvc?rev=781633&view=rev
Log:
HIVE-195. Map-side join. (Namit Jain via zshao)
Added:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinObject.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink4.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapJoinDesc.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredLocalWork.java
hadoop/hive/trunk/ql/src/test/queries/clientnegative/join2.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join25.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join26.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join27.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join28.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join29.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join30.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join31.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join32.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join33.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join34.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join35.q
hadoop/hive/trunk/ql/src/test/results/clientnegative/join2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join25.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join26.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join27.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join28.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join29.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join30.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join31.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join32.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join33.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join34.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join35.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/data/scripts/dumpdata_script.py
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/joinDesc.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/selectDesc.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerFactory.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby2_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby6_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby7_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby8_noskew.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby9.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join0.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join13.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join15.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join18.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join20.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join21.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join22.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/no_hooks.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup4.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_clusterby.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_gby_join.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_join2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/ppd_udf_case.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/regex_col.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_case_column_pruning.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union17.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union20.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union3.q.out
hadoop/hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/cast1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input8.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_part1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testxpath.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input_testxpath2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Jun 4 01:21:30 2009
@@ -39,6 +39,8 @@
HIVE-509. Add create table like.
(Zheng Shao via namit)
+ HIVE-195. Map-side join. (Namit Jain via zshao)
+
IMPROVEMENTS
HIVE-389. Option to build without ivy (jssarma)
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jun 4 01:21:30 2009
@@ -113,6 +113,9 @@
HIVEMAPSIDEAGGREGATE("hive.map.aggr", "true"),
HIVEGROUPBYSKEW("hive.groupby.skewindata", "false"),
HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000),
+ HIVEMAPJOINNUMROWSCACHE("hive.mapjoin.numrows", 1000),
+ HIVEMAPJOINROWSIZE("hive.mapjoin.size.key", 10000),
+ HIVEMAPJOINCACHEROWS("hive.mapjoin.cache.numrows", 10000),
HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", 100000),
HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", (float)0.5),
Modified: hadoop/hive/trunk/data/scripts/dumpdata_script.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/scripts/dumpdata_script.py?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/data/scripts/dumpdata_script.py (original)
+++ hadoop/hive/trunk/data/scripts/dumpdata_script.py Thu Jun 4 01:21:30 2009
@@ -1,4 +1,9 @@
+import sys
+
for i in xrange(50):
for j in xrange(5):
for k in xrange(20022):
print 20000 * i + k
+
+for line in sys.stdin:
+ pass
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CollectOperator.java Thu Jun 4 01:21:30 2009
@@ -39,10 +39,10 @@
transient protected ObjectInspector standardRowInspector;
transient int maxSize;
- public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
- super.initialize(hconf, reporter, inputObjInspector);
+ public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
rowList = new ArrayList<Object> ();
maxSize = conf.getBufferSize().intValue();
+ initializeChildren(hconf, reporter, inputObjInspector);
}
boolean firstRow = true;
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=781633&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Thu Jun 4 01:21:30 2009
@@ -0,0 +1,585 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+import java.util.Vector;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.joinCond;
+import org.apache.hadoop.hive.ql.plan.joinDesc;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * Join operator implementation.
+ */
+public class CommonJoinOperator<T extends joinDesc> extends Operator<T> implements Serializable {
+ private static final long serialVersionUID = 1L;
+ static final protected Log LOG = LogFactory.getLog(CommonJoinOperator.class.getName());
+
+ public static class IntermediateObject {
+ ArrayList<Object>[] objs;
+ int curSize;
+
+ public IntermediateObject(ArrayList<Object>[] objs, int curSize) {
+ this.objs = objs;
+ this.curSize = curSize;
+ }
+
+ public ArrayList<Object>[] getObjs() {
+ return objs;
+ }
+
+ public int getCurSize() {
+ return curSize;
+ }
+
+ public void pushObj(ArrayList<Object> obj) {
+ objs[curSize++] = obj;
+ }
+
+ public void popObj() {
+ curSize--;
+ }
+ }
+
+ transient protected int numValues; // number of aliases
+ transient protected Map<Byte, List<ExprNodeEvaluator>> joinValues;
+ transient protected Map<Byte, List<ObjectInspector>> joinValuesObjectInspectors;
+
+ transient static protected Byte[] order; // order in which the results should be output
+ transient protected joinCond[] condn;
+ transient protected boolean noOuterJoin;
+ transient private Object[] dummyObj; // for outer joins, contains the
+ // potential nulls for the concerned
+ // aliases
+ transient private Vector<ArrayList<Object>>[] dummyObjVectors;
+ transient private Stack<Iterator<ArrayList<Object>>> iterators;
+ transient protected int totalSz; // total size of the composite object
+ transient ObjectInspector joinOutputObjectInspector;
+
+ // keys are the column names. basically this maps the position of the column in
+ // the output of the CommonJoinOperator to the input columnInfo.
+ transient private Map<Integer, Set<String>> posToAliasMap;
+
+ HashMap<Byte, Vector<ArrayList<Object>>> storage;
+ int joinEmitInterval = -1;
+ int nextSz = 0;
+ transient Byte lastAlias = null;
+
+ protected int populateJoinKeyValue(Map<Byte, List<ExprNodeEvaluator>> outMap,
+ Map<Byte, List<exprNodeDesc>> inputMap) {
+
+ int total = 0;
+
+ Iterator<Map.Entry<Byte, List<exprNodeDesc>>> entryIter = inputMap.entrySet().iterator();
+ while (entryIter.hasNext()) {
+ Map.Entry<Byte, List<exprNodeDesc>> e = (Map.Entry<Byte, List<exprNodeDesc>>) entryIter.next();
+ Byte key = (Byte) e.getKey();
+
+ List<exprNodeDesc> expr = (List<exprNodeDesc>) e.getValue();
+ int sz = expr.size();
+ total += sz;
+
+ List<ExprNodeEvaluator> valueFields = new ArrayList<ExprNodeEvaluator>();
+
+ for (int j = 0; j < sz; j++)
+ valueFields.add(ExprNodeEvaluatorFactory.get(expr.get(j)));
+
+ outMap.put(key, valueFields);
+ }
+
+ return total;
+ }
+
+ public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+ LOG.info("COMMONJOIN " + ((StructObjectInspector)inputObjInspector[0]).getTypeName());
+ totalSz = 0;
+ // Map that contains the rows for each alias
+ storage = new HashMap<Byte, Vector<ArrayList<Object>>>();
+
+ numValues = conf.getExprs().size();
+
+ joinValues = new HashMap<Byte, List<ExprNodeEvaluator>>();
+ joinValuesObjectInspectors = new HashMap<Byte, List<ObjectInspector>>();
+
+ if (order == null) {
+ order = new Byte[numValues];
+ for (int i = 0; i < numValues; i++)
+ order[i] = (byte) i;
+ }
+ condn = conf.getConds();
+ noOuterJoin = conf.getNoOuterJoin();
+
+ totalSz = populateJoinKeyValue(joinValues, conf.getExprs());
+
+ dummyObj = new Object[numValues];
+ dummyObjVectors = new Vector[numValues];
+
+ int pos = 0;
+ for (Byte alias : order) {
+ int sz = conf.getExprs().get(alias).size();
+ ArrayList<Object> nr = new ArrayList<Object>(sz);
+
+ for (int j = 0; j < sz; j++)
+ nr.add(null);
+
+ dummyObj[pos] = nr;
+ Vector<ArrayList<Object>> values = new Vector<ArrayList<Object>>();
+ values.add((ArrayList<Object>) dummyObj[pos]);
+ dummyObjVectors[pos] = values;
+ pos++;
+ }
+
+ iterators = new Stack<Iterator<ArrayList<Object>>>();
+
+ joinEmitInterval = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEJOINEMITINTERVAL);
+
+ forwardCache = new Object[totalSz];
+ }
+
+ public void startGroup() throws HiveException {
+ LOG.trace("Join: Starting new group");
+ storage.clear();
+ for (Byte alias : order)
+ storage.put(alias, new Vector<ArrayList<Object>>());
+ }
+
+ private int getNextSize(int sz) {
+ // A very simple counter to keep track of join entries for a key
+ if (sz >= 100000)
+ return sz + 100000;
+
+ return 2 * sz;
+ }
+
+ transient protected Byte alias;
+
+ protected ArrayList<Object> computeValues(Object row, ObjectInspector rowInspector,
+ List<ExprNodeEvaluator> valueFields, Map<Byte, List<ObjectInspector>> joinExprsObjectInspectors) throws HiveException {
+
+ // Get the valueFields Object Inspectors
+ List<ObjectInspector> valueFieldOI = joinExprsObjectInspectors.get(alias);
+ if (valueFieldOI == null) {
+ // Initialize the ExprEvaluator if necessary
+ valueFieldOI = new ArrayList<ObjectInspector>();
+ for (int i=0; i<valueFields.size(); i++) {
+ valueFieldOI.add(valueFields.get(i).initialize(rowInspector));
+ }
+ joinExprsObjectInspectors.put(alias, valueFieldOI);
+ }
+
+ // Compute the values
+ ArrayList<Object> nr = new ArrayList<Object>(valueFields.size());
+ for (int i=0; i<valueFields.size(); i++) {
+ nr.add(ObjectInspectorUtils.copyToStandardObject(
+ valueFields.get(i).evaluate(row),
+ valueFieldOI.get(i)));
+ }
+
+ return nr;
+ }
+
+ public void process(Object row, ObjectInspector rowInspector, int tag)
+ throws HiveException {
+ try {
+ // get alias
+ alias = (byte)tag;
+
+ if ((lastAlias == null) || (!lastAlias.equals(alias)))
+ nextSz = joinEmitInterval;
+
+ ArrayList<Object> nr = computeValues(row, rowInspector, joinValues.get(alias), joinValuesObjectInspectors);
+
+ // number of rows for the key in the given table
+ int sz = storage.get(alias).size();
+
+ // Are we consuming too much memory
+ if (alias == numValues - 1) {
+ if (sz == joinEmitInterval) {
+ // The input is sorted by alias, so if we are already in the last join operand,
+ // we can emit some results now.
+ // Note this has to be done before adding the current row to the storage,
+ // to preserve the correctness for outer joins.
+ checkAndGenObject();
+ storage.get(alias).clear();
+ }
+ } else {
+ if (sz == nextSz) {
+ // Output a warning if we reached at least 1000 rows for a join operand
+ // We won't output a warning for the last join operand since the size
+ // will never goes to joinEmitInterval.
+ StructObjectInspector soi = (StructObjectInspector)rowInspector;
+ StructField sf = soi.getStructFieldRef(Utilities.ReduceField.KEY.toString());
+ Object keyObject = soi.getStructFieldData(row, sf);
+ LOG.warn("table " + alias + " has " + sz + " rows for join key " + keyObject);
+ nextSz = getNextSize(nextSz);
+ }
+ }
+
+ // Add the value to the vector
+ storage.get(alias).add(nr);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new HiveException(e);
+ }
+ }
+
+ transient Object[] forwardCache;
+
+ private void createForwardJoinObject(IntermediateObject intObj,
+ boolean[] nullsArr) throws HiveException {
+ int p = 0;
+ for (int i = 0; i < numValues; i++) {
+ Byte alias = order[i];
+ int sz = joinValues.get(alias).size();
+ if (nullsArr[i]) {
+ for (int j = 0; j < sz; j++) {
+ forwardCache[p++] = null;
+ }
+ } else {
+ ArrayList<Object> obj = intObj.getObjs()[i];
+ for (int j = 0; j < sz; j++) {
+ forwardCache[p++] = obj.get(j);
+ }
+ }
+ }
+ forward(forwardCache, joinOutputObjectInspector);
+ }
+
+ private void copyOldArray(boolean[] src, boolean[] dest) {
+ for (int i = 0; i < src.length; i++)
+ dest[i] = src[i];
+ }
+
+ private Vector<boolean[]> joinObjectsInnerJoin(Vector<boolean[]> resNulls,
+ Vector<boolean[]> inputNulls, ArrayList<Object> newObj,
+ IntermediateObject intObj, int left, boolean newObjNull) {
+ if (newObjNull)
+ return resNulls;
+ Iterator<boolean[]> nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] oldNulls = nullsIter.next();
+ boolean oldObjNull = oldNulls[left];
+ if (!oldObjNull) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ copyOldArray(oldNulls, newNulls);
+ newNulls[oldNulls.length] = false;
+ resNulls.add(newNulls);
+ }
+ }
+ return resNulls;
+ }
+
+ private Vector<boolean[]> joinObjectsLeftOuterJoin(
+ Vector<boolean[]> resNulls, Vector<boolean[]> inputNulls,
+ ArrayList<Object> newObj, IntermediateObject intObj, int left,
+ boolean newObjNull) {
+ Iterator<boolean[]> nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] oldNulls = nullsIter.next();
+ boolean oldObjNull = oldNulls[left];
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ copyOldArray(oldNulls, newNulls);
+ if (oldObjNull)
+ newNulls[oldNulls.length] = true;
+ else
+ newNulls[oldNulls.length] = newObjNull;
+ resNulls.add(newNulls);
+ }
+ return resNulls;
+ }
+
+ private Vector<boolean[]> joinObjectsRightOuterJoin(
+ Vector<boolean[]> resNulls, Vector<boolean[]> inputNulls,
+ ArrayList<Object> newObj, IntermediateObject intObj, int left,
+ boolean newObjNull, boolean firstRow) {
+ if (newObjNull)
+ return resNulls;
+
+ if (inputNulls.isEmpty() && firstRow) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ for (int i = 0; i < intObj.getCurSize() - 1; i++)
+ newNulls[i] = true;
+ newNulls[intObj.getCurSize()-1] = newObjNull;
+ resNulls.add(newNulls);
+ return resNulls;
+ }
+
+ boolean allOldObjsNull = firstRow;
+
+ Iterator<boolean[]> nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] oldNulls = nullsIter.next();
+ if (!oldNulls[left]) {
+ allOldObjsNull = false;
+ break;
+ }
+ }
+
+ nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] oldNulls = nullsIter.next();
+ boolean oldObjNull = oldNulls[left];
+
+ if (!oldObjNull) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ copyOldArray(oldNulls, newNulls);
+ newNulls[oldNulls.length] = newObjNull;
+ resNulls.add(newNulls);
+ } else if (allOldObjsNull) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ for (int i = 0; i < intObj.getCurSize() - 1; i++)
+ newNulls[i] = true;
+ newNulls[oldNulls.length] = newObjNull;
+ resNulls.add(newNulls);
+ return resNulls;
+ }
+ }
+ return resNulls;
+ }
+
+ private Vector<boolean[]> joinObjectsFullOuterJoin(
+ Vector<boolean[]> resNulls, Vector<boolean[]> inputNulls,
+ ArrayList<Object> newObj, IntermediateObject intObj, int left,
+ boolean newObjNull, boolean firstRow) {
+ if (newObjNull) {
+ Iterator<boolean[]> nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] oldNulls = nullsIter.next();
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ copyOldArray(oldNulls, newNulls);
+ newNulls[oldNulls.length] = newObjNull;
+ resNulls.add(newNulls);
+ }
+ return resNulls;
+ }
+
+ if (inputNulls.isEmpty() && firstRow) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ for (int i = 0; i < intObj.getCurSize() - 1; i++)
+ newNulls[i] = true;
+ newNulls[intObj.getCurSize()-1] = newObjNull;
+ resNulls.add(newNulls);
+ return resNulls;
+ }
+
+ boolean allOldObjsNull = firstRow;
+
+ Iterator<boolean[]> nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] oldNulls = nullsIter.next();
+ if (!oldNulls[left]) {
+ allOldObjsNull = false;
+ break;
+ }
+ }
+ boolean rhsPreserved = false;
+
+ nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] oldNulls = nullsIter.next();
+ boolean oldObjNull = oldNulls[left];
+
+ if (!oldObjNull) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ copyOldArray(oldNulls, newNulls);
+ newNulls[oldNulls.length] = newObjNull;
+ resNulls.add(newNulls);
+ } else if (oldObjNull) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ copyOldArray(oldNulls, newNulls);
+ newNulls[oldNulls.length] = true;
+ resNulls.add(newNulls);
+
+ if (allOldObjsNull && !rhsPreserved) {
+ newNulls = new boolean[intObj.getCurSize()];
+ for (int i = 0; i < oldNulls.length; i++)
+ newNulls[i] = true;
+ newNulls[oldNulls.length] = false;
+ resNulls.add(newNulls);
+ rhsPreserved = true;
+ }
+ }
+ }
+ return resNulls;
+ }
+
+ /*
+ * The new input is added to the list of existing inputs. Each entry in the
+ * array of inputNulls denotes the entries in the intermediate object to be
+ * used. The intermediate object is augmented with the new object, and list of
+ * nulls is changed appropriately. The list will contain all non-nulls for a
+ * inner join. The outer joins are processed appropriately.
+ */
+ private Vector<boolean[]> joinObjects(Vector<boolean[]> inputNulls,
+ ArrayList<Object> newObj, IntermediateObject intObj,
+ int joinPos, boolean firstRow) {
+ Vector<boolean[]> resNulls = new Vector<boolean[]>();
+ boolean newObjNull = newObj == dummyObj[joinPos] ? true : false;
+ if (joinPos == 0) {
+ if (newObjNull)
+ return null;
+ boolean[] nulls = new boolean[1];
+ nulls[0] = newObjNull;
+ resNulls.add(nulls);
+ return resNulls;
+ }
+
+ int left = condn[joinPos - 1].getLeft();
+ int type = condn[joinPos - 1].getType();
+
+ // process all nulls for RIGHT and FULL OUTER JOINS
+ if (((type == joinDesc.RIGHT_OUTER_JOIN) || (type == joinDesc.FULL_OUTER_JOIN))
+ && !newObjNull && (inputNulls == null) && firstRow) {
+ boolean[] newNulls = new boolean[intObj.getCurSize()];
+ for (int i = 0; i < newNulls.length - 1; i++)
+ newNulls[i] = true;
+ newNulls[newNulls.length - 1] = false;
+ resNulls.add(newNulls);
+ return resNulls;
+ }
+
+ if (inputNulls == null)
+ return null;
+
+ if (type == joinDesc.INNER_JOIN)
+ return joinObjectsInnerJoin(resNulls, inputNulls, newObj, intObj, left,
+ newObjNull);
+ else if (type == joinDesc.LEFT_OUTER_JOIN)
+ return joinObjectsLeftOuterJoin(resNulls, inputNulls, newObj, intObj,
+ left, newObjNull);
+ else if (type == joinDesc.RIGHT_OUTER_JOIN)
+ return joinObjectsRightOuterJoin(resNulls, inputNulls, newObj, intObj,
+ left, newObjNull, firstRow);
+ assert (type == joinDesc.FULL_OUTER_JOIN);
+ return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left,
+ newObjNull, firstRow);
+ }
+
+ /*
+ * genObject is a recursive function. For the inputs, a array of bitvectors is
+ * maintained (inputNulls) where each entry denotes whether the element is to
+ * be used or not (whether it is null or not). The size of the bitvector is
+ * same as the number of inputs under consideration currently. When all inputs
+ * are accounted for, the output is forwared appropriately.
+ */
+ private void genObject(Vector<boolean[]> inputNulls, int aliasNum,
+ IntermediateObject intObj, boolean firstRow) throws HiveException {
+ boolean childFirstRow = firstRow;
+ if (aliasNum < numValues) {
+ Iterator<ArrayList<Object>> aliasRes = storage.get(order[aliasNum])
+ .iterator();
+ iterators.push(aliasRes);
+ while (aliasRes.hasNext()) {
+ ArrayList<Object> newObj = aliasRes.next();
+ intObj.pushObj(newObj);
+ Vector<boolean[]> newNulls = joinObjects(inputNulls, newObj, intObj,
+ aliasNum, childFirstRow);
+ genObject(newNulls, aliasNum + 1, intObj, firstRow);
+ intObj.popObj();
+ firstRow = false;
+ }
+ iterators.pop();
+ } else {
+ if (inputNulls == null)
+ return;
+ Iterator<boolean[]> nullsIter = inputNulls.iterator();
+ while (nullsIter.hasNext()) {
+ boolean[] nullsVec = nullsIter.next();
+ createForwardJoinObject(intObj, nullsVec);
+ }
+ }
+ }
+
+ /**
+ * Forward a record of join results.
+ *
+ * @throws HiveException
+ */
+ public void endGroup() throws HiveException {
+ LOG.trace("Join Op: endGroup called: numValues=" + numValues);
+ checkAndGenObject();
+ }
+
+ protected void checkAndGenObject() throws HiveException {
+ // does any result need to be emitted
+ for (int i = 0; i < numValues; i++) {
+ Byte alias = order[i];
+ if (storage.get(alias).iterator().hasNext() == false) {
+ if (noOuterJoin) {
+ LOG.trace("No data for alias=" + i);
+ return;
+ } else {
+ storage.put(alias, dummyObjVectors[i]);
+ }
+ }
+ }
+
+ LOG.trace("calling genObject");
+ genObject(null, 0, new IntermediateObject(new ArrayList[numValues], 0), true);
+ LOG.trace("called genObject");
+ }
+
+ /**
+ * All done
+ *
+ */
+ public void close(boolean abort) throws HiveException {
+ LOG.trace("Join Op close");
+ super.close(abort);
+ }
+
+ @Override
+ public String getName() {
+ return "JOIN";
+ }
+
+ /**
+ * @return the posToAliasMap
+ */
+ public Map<Integer, Set<String>> getPosToAliasMap() {
+ return posToAliasMap;
+ }
+
+ /**
+ * @param posToAliasMap the posToAliasMap to set
+ */
+ public void setPosToAliasMap(Map<Integer, Set<String>> posToAliasMap) {
+ this.posToAliasMap = posToAliasMap;
+ }
+
+}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Thu Jun 4 01:21:30 2009
@@ -649,14 +649,7 @@
pathToAliases.put(newPath.toString(), newList);
Map<String,partitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo();
- partitionDesc pDesc = new partitionDesc();
-
- tableDesc tDesc = new tableDesc(LazySimpleSerDe.class,
- SequenceFileInputFormat.class,
- SequenceFileOutputFormat.class,
- new Properties());
-
- pDesc.setTableDesc(tDesc);
+ partitionDesc pDesc = work.getAliasToPartnInfo().get(oneAlias);
pathToPartitionInfo.put(newPath.toString(), pDesc);
RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newPath, Text.class, false, new Properties(), null);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Thu Jun 4 01:21:30 2009
@@ -27,24 +27,61 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.plan.mapredWork;
+import org.apache.hadoop.hive.ql.plan.mapredLocalWork;
+import org.apache.hadoop.hive.ql.plan.fetchWork;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public class ExecMapper extends MapReduceBase implements Mapper {
private MapOperator mo;
+ private Map<String, FetchOperator> fetchOperators;
private OutputCollector oc;
private JobConf jc;
private boolean abort = false;
private Reporter rp;
public static final Log l4j = LogFactory.getLog("ExecMapper");
private static boolean done;
+
+ private void init() {
+ mo = null;
+ fetchOperators = null;
+ oc = null;
+ jc = null;
+ abort = false;
+ rp = null;
+ }
public void configure(JobConf job) {
- jc = job;
- mapredWork mrwork = Utilities.getMapRedWork(job);
- mo = new MapOperator ();
- mo.setConf(mrwork);
- // we don't initialize the operator until we have set the output collector
+ try {
+ init();
+ jc = job;
+ mapredWork mrwork = Utilities.getMapRedWork(job);
+ mo = new MapOperator();
+ mo.setConf(mrwork);
+ mapredLocalWork mlo = mrwork.getMapLocalWork();
+ if (mlo != null) {
+ fetchOperators = new HashMap<String, FetchOperator>();
+ Map<String, fetchWork> aliasToFetchWork = mlo.getAliasToFetchWork();
+ Iterator<Map.Entry<String, fetchWork>> fetchWorkSet = aliasToFetchWork
+ .entrySet().iterator();
+ while (fetchWorkSet.hasNext()) {
+ Map.Entry<String, fetchWork> entry = fetchWorkSet.next();
+ String alias = entry.getKey();
+ fetchWork fWork = entry.getValue();
+ fetchOperators.put(alias, new FetchOperator(fWork, job));
+ l4j.info("fetchoperator for " + alias + " initialized");
+ }
+ }
+
+ // we don't initialize the operator until we have set the output collector
+ } catch (Exception e) {
+ // Bail out ungracefully - we should never hit
+ // this here - but would have hit it in SemanticAnalyzer
+ throw new RuntimeException(e);
+ }
}
public void map(Object key, Object value,
@@ -55,6 +92,37 @@
oc = output;
mo.setOutputCollector(oc);
mo.initialize(jc, reporter, null);
+ if (fetchOperators != null) {
+ mapredWork mrwork = Utilities.getMapRedWork(jc);
+ mapredLocalWork localWork = mrwork.getMapLocalWork();
+ Iterator<Map.Entry<String, FetchOperator>> fetchOps = fetchOperators.entrySet().iterator();
+ while (fetchOps.hasNext()) {
+ Map.Entry<String, FetchOperator> entry = fetchOps.next();
+ String alias = entry.getKey();
+ FetchOperator fetchOp = entry.getValue();
+ Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);
+ // All the operators need to be initialized before process
+ forwardOp.initialize(jc, reporter, new ObjectInspector[]{fetchOp.getOutputObjectInspector()});
+ }
+
+ fetchOps = fetchOperators.entrySet().iterator();
+ while (fetchOps.hasNext()) {
+ Map.Entry<String, FetchOperator> entry = fetchOps.next();
+ String alias = entry.getKey();
+ FetchOperator fetchOp = entry.getValue();
+ Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);
+
+ while (true) {
+ InspectableObject row = fetchOp.getNextRow();
+ if (row == null) {
+ break;
+ }
+
+ forwardOp.process(row.o, row.oi, 0);
+ }
+ }
+ }
+
rp = reporter;
} catch (HiveException e) {
abort = true;
@@ -94,6 +162,18 @@
// ideally hadoop should let us know whether map execution failed or not
try {
mo.close(abort);
+ if (fetchOperators != null) {
+ mapredWork mrwork = Utilities.getMapRedWork(jc);
+ mapredLocalWork localWork = mrwork.getMapLocalWork();
+ Iterator<Map.Entry<String, FetchOperator>> fetchOps = fetchOperators.entrySet().iterator();
+ while (fetchOps.hasNext()) {
+ Map.Entry<String, FetchOperator> entry = fetchOps.next();
+ String alias = entry.getKey();
+ Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);
+ forwardOp.close(abort);
+ }
+ }
+
reportStats rps = new reportStats (rp);
mo.preorderMap(rps);
return;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java Thu Jun 4 01:21:30 2009
@@ -35,16 +35,14 @@
private static final long serialVersionUID = 1L;
transient protected ExprNodeEvaluator eval;
- public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
- super.initialize(hconf, reporter, inputObjInspector);
+ public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
eval = ExprNodeEvaluatorFactory.get(conf.getCol());
+ outputRowInspector = eval.initialize(inputObjInspector[0]);
+ initializeChildren(hconf, reporter, new ObjectInspector[]{outputRowInspector});
}
ObjectInspector outputRowInspector;
public void process(Object row, ObjectInspector rowInspector, int tag) throws HiveException {
- if (outputRowInspector == null) {
- outputRowInspector = eval.initialize(rowInspector);
- }
forward(eval.evaluate(row), outputRowInspector);
}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=781633&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Thu Jun 4 01:21:30 2009
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.fetchWork;
+import org.apache.hadoop.hive.ql.plan.partitionDesc;
+import org.apache.hadoop.hive.ql.plan.tableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+/**
+ * FetchTask implementation
+ **/
+public class FetchOperator {
+
+ transient protected Log LOG;
+ transient protected LogHelper console;
+
+ public FetchOperator(fetchWork work, JobConf job) {
+ LOG = LogFactory.getLog(this.getClass().getName());
+ console = new LogHelper(LOG);
+
+ this.work = work;
+ this.job = job;
+
+ currRecReader = null;
+ currPath = null;
+ currTbl = null;
+ currPart = null;
+ iterPath = null;
+ iterPartDesc = null;
+ tblDataDone = false;
+ rowWithPart = new Object[2];
+ }
+
+ private fetchWork work;
+ private int splitNum;
+ private RecordReader<WritableComparable, Writable> currRecReader;
+ private InputSplit[] inputSplits;
+ private InputFormat inputFormat;
+ private JobConf job;
+ private WritableComparable key;
+ private Writable value;
+ private Deserializer serde;
+ private Iterator<Path> iterPath;
+ private Iterator<partitionDesc> iterPartDesc;
+ private Path currPath;
+ private partitionDesc currPart;
+ private tableDesc currTbl;
+ private boolean tblDataDone;
+ private StructObjectInspector rowObjectInspector;
+ private Object[] rowWithPart;
+
+ /**
+ * A cache of InputFormat instances.
+ */
+ private static Map<Class, InputFormat<WritableComparable, Writable>> inputFormats = new HashMap<Class, InputFormat<WritableComparable, Writable>>();
+
+ static InputFormat<WritableComparable, Writable> getInputFormatFromCache(
+ Class inputFormatClass, Configuration conf) throws IOException {
+ if (!inputFormats.containsKey(inputFormatClass)) {
+ try {
+ InputFormat<WritableComparable, Writable> newInstance = (InputFormat<WritableComparable, Writable>) ReflectionUtils
+ .newInstance(inputFormatClass, conf);
+ inputFormats.put(inputFormatClass, newInstance);
+ } catch (Exception e) {
+ throw new IOException("Cannot create an instance of InputFormat class "
+ + inputFormatClass.getName() + " as specified in mapredWork!");
+ }
+ }
+ return inputFormats.get(inputFormatClass);
+ }
+
+ private void setPrtnDesc() throws Exception {
+ List<String> partNames = new ArrayList<String>();
+ List<String> partValues = new ArrayList<String>();
+
+ String pcols = currPart
+ .getTableDesc()
+ .getProperties()
+ .getProperty(
+ org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS);
+ LinkedHashMap<String, String> partSpec = currPart.getPartSpec();
+
+ List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
+ String[] partKeys = pcols.trim().split("/");
+ for (String key : partKeys) {
+ partNames.add(key);
+ partValues.add(partSpec.get(key));
+ partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ }
+ StructObjectInspector partObjectInspector = ObjectInspectorFactory
+ .getStandardStructObjectInspector(partNames, partObjectInspectors);
+ rowObjectInspector = (StructObjectInspector) serde.getObjectInspector();
+
+ rowWithPart[1] = partValues;
+ rowObjectInspector = ObjectInspectorFactory
+ .getUnionStructObjectInspector(Arrays
+ .asList(new StructObjectInspector[] { rowObjectInspector,
+ partObjectInspector }));
+ }
+
+ private void getNextPath() throws Exception {
+ // first time
+ if (iterPath == null) {
+ if (work.getTblDir() != null) {
+ if (!tblDataDone) {
+ currPath = work.getTblDirPath();
+ currTbl = work.getTblDesc();
+ FileSystem fs = currPath.getFileSystem(job);
+ if (fs.exists(currPath)) {
+ FileStatus[] fStats = fs.listStatus(currPath);
+ for (FileStatus fStat : fStats) {
+ if (fStat.getLen() > 0) {
+ tblDataDone = true;
+ break;
+ }
+ }
+ }
+
+ if (!tblDataDone)
+ currPath = null;
+ return;
+ } else {
+ currTbl = null;
+ currPath = null;
+ }
+ return;
+ } else {
+ iterPath = fetchWork.convertStringToPathArray(work.getPartDir()).iterator();
+ iterPartDesc = work.getPartDesc().iterator();
+ }
+ }
+
+ while (iterPath.hasNext()) {
+ Path nxt = iterPath.next();
+ partitionDesc prt = iterPartDesc.next();
+ FileSystem fs = nxt.getFileSystem(job);
+ if (fs.exists(nxt)) {
+ FileStatus[] fStats = fs.listStatus(nxt);
+ for (FileStatus fStat : fStats) {
+ if (fStat.getLen() > 0) {
+ currPath = nxt;
+ currPart = prt;
+ return;
+ }
+ }
+ }
+ }
+ }
+
+ private RecordReader<WritableComparable, Writable> getRecordReader()
+ throws Exception {
+ if (currPath == null) {
+ getNextPath();
+ if (currPath == null)
+ return null;
+
+ // not using FileInputFormat.setInputPaths() here because it forces a connection
+ // to the default file system - which may or may not be online during pure metadata
+ // operations
+ job.set("mapred.input.dir",
+ org.apache.hadoop.util.StringUtils.escapeString(currPath.toString()));
+
+ tableDesc tmp = currTbl;
+ if (tmp == null)
+ tmp = currPart.getTableDesc();
+ inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job);
+ inputSplits = inputFormat.getSplits(job, 1);
+ splitNum = 0;
+ serde = tmp.getDeserializerClass().newInstance();
+ serde.initialize(job, tmp.getProperties());
+ LOG.debug("Creating fetchTask with deserializer typeinfo: "
+ + serde.getObjectInspector().getTypeName());
+ LOG.debug("deserializer properties: " + tmp.getProperties());
+ if (!tblDataDone)
+ setPrtnDesc();
+ }
+
+ if (splitNum >= inputSplits.length) {
+ if (currRecReader != null) {
+ currRecReader.close();
+ currRecReader = null;
+ }
+ currPath = null;
+ return getRecordReader();
+ }
+
+ currRecReader = inputFormat.getRecordReader(inputSplits[splitNum++], job,
+ Reporter.NULL);
+ key = currRecReader.createKey();
+ value = currRecReader.createValue();
+ return currRecReader;
+ }
+
+ /**
+ * Get the next row. The fetch context is modified appropriately.
+ *
+ * @param ctx
+ * fetch context
+ **/
+ public InspectableObject getNextRow() {
+ try {
+ if (currRecReader == null) {
+ currRecReader = getRecordReader();
+ if (currRecReader == null)
+ return null;
+ }
+
+ boolean ret = currRecReader.next(key, value);
+ if (ret) {
+ if (tblDataDone) {
+ Object obj = serde.deserialize(value);
+ return new InspectableObject(obj, serde.getObjectInspector());
+ } else {
+ rowWithPart[0] = serde.deserialize(value);
+ return new InspectableObject(rowWithPart, rowObjectInspector);
+ }
+ } else {
+ currRecReader.close();
+ currRecReader = null;
+ currRecReader = getRecordReader();
+ if (currRecReader == null)
+ return null;
+ else
+ return getNextRow();
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Clear the context, if anything needs to be done.
+ *
+ * @param ctx
+ * fetch context
+ **/
+ public void clearFetchContext() throws HiveException {
+ try {
+ if (currRecReader != null) {
+ currRecReader.close();
+ currRecReader = null;
+ }
+ } catch (Exception e) {
+ throw new HiveException("Failed with exception " + e.getMessage()
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
+
+ public ObjectInspector getOutputObjectInspector() throws HiveException {
+ try {
+ ObjectInspector outInspector;
+ if (work.getTblDir() != null) {
+ tableDesc tbl = work.getTblDesc();
+ Deserializer serde = tbl.getDeserializerClass().newInstance();
+ serde.initialize(job, tbl.getProperties());
+ return serde.getObjectInspector();
+ }
+ else {
+ List<partitionDesc> listParts = work.getPartDesc();
+ currPart = listParts.get(0);
+ serde = currPart.getTableDesc().getDeserializerClass().newInstance();
+ serde.initialize(job, currPart.getTableDesc().getProperties());
+ setPrtnDesc();
+ currPart = null;
+ return rowObjectInspector;
+ }
+ } catch (Exception e) {
+ throw new HiveException("Failed with exception " + e.getMessage()
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
+}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Thu Jun 4 01:21:30 2009
@@ -57,6 +57,7 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
/**
* FetchTask implementation
@@ -65,28 +66,24 @@
private static final long serialVersionUID = 1L;
private int maxRows = 100;
+ private FetchOperator ftOp;
+ private LazySimpleSerDe mSerde;
+ private int totalRows;
public void initialize (HiveConf conf) {
super.initialize(conf);
- currRecReader = null;
- try {
- job = new JobConf(conf, ExecDriver.class);
-
- mSerde = new LazySimpleSerDe();
- Properties mSerdeProp = new Properties();
- mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
- mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, ((fetchWork)work).getSerializationNullFormat());
- mSerde.initialize(job, mSerdeProp);
+ try {
+ // Create a file system handle
+ JobConf job = new JobConf(conf, ExecDriver.class);
+
+ mSerde = new LazySimpleSerDe();
+ Properties mSerdeProp = new Properties();
+ mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
+ mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, ((fetchWork)work).getSerializationNullFormat());
+ mSerde.initialize(job, mSerdeProp);
- currPath = null;
- currTbl = null;
- currPart = null;
- iterPath = null;
- iterPartDesc = null;
- totalRows = 0;
- tblDataDone = false;
- rowWithPart = new Object[2];
+ ftOp = new FetchOperator(work, job);
} catch (Exception e) {
// Bail out ungracefully - we should never hit
// this here - but would have hit it in SemanticAnalyzer
@@ -107,158 +104,6 @@
}
/**
- * A cache of InputFormat instances.
- */
- private static Map<Class, InputFormat<WritableComparable, Writable>> inputFormats =
- new HashMap<Class, InputFormat<WritableComparable, Writable>>();
-
- static InputFormat<WritableComparable, Writable> getInputFormatFromCache(Class inputFormatClass, Configuration conf) throws IOException {
- if (!inputFormats.containsKey(inputFormatClass)) {
- try {
- InputFormat<WritableComparable, Writable> newInstance =
- (InputFormat<WritableComparable, Writable>)ReflectionUtils.newInstance(inputFormatClass, conf);
- inputFormats.put(inputFormatClass, newInstance);
- } catch (Exception e) {
- throw new IOException("Cannot create an instance of InputFormat class " + inputFormatClass.getName()
- + " as specified in mapredWork!");
- }
- }
- return inputFormats.get(inputFormatClass);
- }
-
- private int splitNum;
- private RecordReader<WritableComparable, Writable> currRecReader;
- private InputSplit[] inputSplits;
- private InputFormat inputFormat;
- private JobConf job;
- private WritableComparable key;
- private Writable value;
- private Deserializer serde;
- private LazySimpleSerDe mSerde;
- private int totalRows;
- private Iterator<Path> iterPath;
- private Iterator<partitionDesc> iterPartDesc;
- private Path currPath;
- private partitionDesc currPart;
- private tableDesc currTbl;
- private boolean tblDataDone;
- private StructObjectInspector rowObjectInspector;
- private Object[] rowWithPart;
-
- private void setPrtnDesc() throws Exception {
- List<String> partNames = new ArrayList<String>();
- List<String> partValues = new ArrayList<String>();
-
- String pcols = currPart.getTableDesc().getProperties().getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS);
- LinkedHashMap<String, String> partSpec = currPart.getPartSpec();
-
- List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
- String[] partKeys = pcols.trim().split("/");
- for(String key: partKeys) {
- partNames.add(key);
- partValues.add(partSpec.get(key));
- partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- }
- StructObjectInspector partObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
- rowObjectInspector = (StructObjectInspector)serde.getObjectInspector();
-
- rowWithPart[1] = partValues;
- rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(Arrays.asList(new StructObjectInspector[]{
- rowObjectInspector, partObjectInspector}));
- }
-
- private void getNextPath() throws Exception {
- // first time
- if (iterPath == null) {
- if (work.getTblDir() != null) {
- if (!tblDataDone) {
- currPath = work.getTblDir();
- currTbl = work.getTblDesc();
- FileSystem fs = currPath.getFileSystem(conf);
- if (fs.exists(currPath))
- {
- FileStatus[] fStats = fs.listStatus(currPath);
- for (FileStatus fStat:fStats) {
- if (fStat.getLen() > 0) {
- tblDataDone = true;
- break;
- }
- }
- }
-
- if (!tblDataDone) currPath = null;
- return;
- } else {
- currTbl = null;
- currPath = null;
- }
- return;
- } else {
- iterPath = work.getPartDir().iterator();
- iterPartDesc = work.getPartDesc().iterator();
- }
- }
-
- while (iterPath.hasNext()) {
- Path nxt = iterPath.next();
- partitionDesc prt = iterPartDesc.next();
- FileSystem fs = nxt.getFileSystem(conf);
- if (fs.exists(nxt))
- {
- FileStatus[] fStats = fs.listStatus(nxt);
- for (FileStatus fStat:fStats) {
- if (fStat.getLen() > 0) {
- currPath = nxt;
- currPart = prt;
- return;
- }
- }
- }
- }
- }
-
- private RecordReader<WritableComparable, Writable> getRecordReader() throws Exception {
- if (currPath == null) {
- getNextPath();
- if (currPath == null)
- return null;
-
- // not using FileInputFormat.setInputPaths() here because it forces a connection
- // to the default file system - which may or may not be online during pure metadata
- // operations
- job.set("mapred.input.dir",
- org.apache.hadoop.util.StringUtils.escapeString(currPath.toString()));
-
- tableDesc tmp = currTbl;
- if (tmp == null)
- tmp = currPart.getTableDesc();
- inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job);
- inputSplits = inputFormat.getSplits(job, 1);
- splitNum = 0;
- serde = tmp.getDeserializerClass().newInstance();
- serde.initialize(job, tmp.getProperties());
- LOG.debug("Creating fetchTask with deserializer typeinfo: " + serde.getObjectInspector().getTypeName());
- LOG.debug("deserializer properties: " + tmp.getProperties());
- if (!tblDataDone)
- setPrtnDesc();
- }
-
- if (splitNum >= inputSplits.length) {
- if (currRecReader != null) {
- currRecReader.close();
- currRecReader = null;
- }
- currPath = null;
- return getRecordReader();
- }
-
- currRecReader = inputFormat.getRecordReader(inputSplits[splitNum++], job, Reporter.NULL);
- key = currRecReader.createKey();
- value = currRecReader.createValue();
- return currRecReader;
- }
-
- /**
* Return the maximum number of rows returned by fetch
*/
public int getMaxRows() {
@@ -279,50 +124,22 @@
if ((work.getLimit() >= 0) && ((work.getLimit() - totalRows) < rowsRet))
rowsRet = work.getLimit() - totalRows;
if (rowsRet <= 0) {
- if (currRecReader != null) {
- currRecReader.close();
- currRecReader = null;
- }
+ ftOp.clearFetchContext();
return false;
}
- while (numRows < rowsRet) {
- if (currRecReader == null) {
- currRecReader = getRecordReader();
- if (currRecReader == null) {
- if (numRows == 0)
- return false;
- totalRows += numRows;
- return true;
- }
- }
- boolean ret = currRecReader.next(key, value);
- if (ret) {
- if (tblDataDone) {
- Object obj = serde.deserialize(value);
- res.add(((Text)mSerde.serialize(obj, serde.getObjectInspector())).toString());
- } else {
- rowWithPart[0] = serde.deserialize(value);
- res.add(((Text)mSerde.serialize(rowWithPart, rowObjectInspector)).toString());
- }
- numRows++;
- }
- else {
- currRecReader.close();
- currRecReader = null;
- currRecReader = getRecordReader();
- if (currRecReader == null) {
- if (numRows == 0)
- return false;
- totalRows += numRows;
- return true;
- }
- else {
- key = currRecReader.createKey();
- value = currRecReader.createValue();
- }
- }
- }
+ while (numRows < rowsRet) {
+ InspectableObject io = ftOp.getNextRow();
+ if (io == null) {
+ if (numRows == 0)
+ return false;
+ totalRows += numRows;
+ return true;
+ }
+
+ res.add(((Text)mSerde.serialize(io.o, io.oi)).toString());
+ numRows++;
+ }
totalRows += numRows;
return true;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Thu Jun 4 01:21:30 2009
@@ -75,6 +75,10 @@
}
public void close(boolean abort) throws HiveException {
+ if (state == state.CLOSE)
+ return;
+
+ state = state.CLOSE;
if(!abort) {
if (outWriter != null) {
try {
@@ -96,9 +100,7 @@
}
}
- public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
- super.initialize(hconf, reporter, inputObjInspector);
-
+ public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
try {
serializer = (Serializer)conf.getTableInfo().getDeserializerClass().newInstance();
serializer.initialize(null, conf.getTableInfo().getProperties());
@@ -157,6 +159,8 @@
e.printStackTrace();
throw new HiveException(e);
}
+
+ initializeChildren(hconf, reporter, inputObjInspector);
}
public static RecordWriter getRecordWriter(JobConf jc, HiveOutputFormat<?, ?> hiveOutputFormat,
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java Thu Jun 4 01:21:30 2009
@@ -47,13 +47,14 @@
passed_count = new LongWritable();
}
- public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
- super.initialize(hconf, reporter, inputObjInspector);
+ public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+
try {
this.conditionEvaluator = ExprNodeEvaluatorFactory.get(conf.getPredicate());
statsMap.put(Counter.FILTERED, filtered_count);
statsMap.put(Counter.PASSED, passed_count);
conditionInspector = null;
+ initializeChildren(hconf, reporter, inputObjInspector);
} catch (Throwable e) {
e.printStackTrace();
throw new RuntimeException(e);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ForwardOperator.java Thu Jun 4 01:21:30 2009
@@ -32,8 +32,8 @@
**/
public class ForwardOperator extends Operator<forwardDesc> implements Serializable {
private static final long serialVersionUID = 1L;
- public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
- super.initialize(hconf, reporter, inputObjInspector);
+ public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+ initializeChildren(hconf, reporter, inputObjInspector);
// nothing to do really ..
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=781633&r1=781632&r2=781633&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Thu Jun 4 01:21:30 2009
@@ -131,8 +131,8 @@
transient int numEntriesVarSize;
transient int numEntriesHashTable;
- public void initialize(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
- super.initialize(hconf, reporter, inputObjInspector);
+ public void initializeOp(Configuration hconf, Reporter reporter, ObjectInspector[] inputObjInspector) throws HiveException {
+
totalMemory = Runtime.getRuntime().totalMemory();
numRowsInput = 0;
numRowsHashTbl = 0;
@@ -245,6 +245,14 @@
for(int i=0; i<objectInspectors.size(); i++) {
fieldNames.add(Integer.valueOf(i).toString());
}
+
+ for (int i = 0; i < keyFields.length; i++) {
+ if (keyObjectInspectors[i] == null) {
+ keyObjectInspectors[i] = keyFields[i].initialize(inputObjInspector[0]);
+ }
+ objectInspectors.set(i, keyObjectInspectors[i]);
+ }
+
outputObjectInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, objectInspectors);
@@ -253,6 +261,8 @@
// is not known, estimate that based on the number of entries
if (conf.getMode() == groupByDesc.Mode.HASH)
computeMaxEntriesHashAggr(hconf);
+
+ initializeChildren(hconf, reporter, new ObjectInspector[]{outputObjectInspector});
}
/**
@@ -460,6 +470,7 @@
}
public void process(Object row, ObjectInspector rowInspector, int tag) throws HiveException {
+ firstRow = false;
// Total number of input rows is needed for hash aggregation only
if (hashAggr) {
numRowsInput++;
@@ -486,16 +497,6 @@
}
keyObjects[i] = keyFields[i].evaluate(row);
newKeys.add(keyObjects[i]);
- if (firstRow) {
- objectInspectors.set(i, keyObjectInspectors[i]);
- }
- }
-
- if (firstRow) {
- firstRow = false;
- // After first row, we can compute outputObjectInspector based on keyObjectInspectors.
- outputObjectInspector =
- ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, objectInspectors);
}
if (hashAggr)