You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/01/24 19:08:55 UTC
svn commit: r1438111 [1/4] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/common/
metastore/src/java/org/apache/hadoop/hive/metastore/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/java/org...
Author: hashutosh
Date: Thu Jan 24 18:08:52 2013
New Revision: 1438111
URL: http://svn.apache.org/viewvc?rev=1438111&view=rev
Log:
HIVE-3833 : object inspectors should be initialized based on partition metadata (Namit Jain via Ashutosh Chauhan)
Added:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java
hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat10.q
hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat11.q
hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat12.q
hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat13.q
hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat14.q
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat10.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat11.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat12.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat13.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat14.q.out
Removed:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/util/ObjectPair.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestPartition.java
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat8.q
hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat9.q
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin10.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin11.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin12.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin13.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin2.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin3.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin5.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin7.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin8.q.out
hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin9.q.out
hive/trunk/ql/src/test/results/clientpositive/columnstats_partlvl.q.out
hive/trunk/ql/src/test/results/clientpositive/combine2_hadoop20.q.out
hive/trunk/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_map_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out
hive/trunk/ql/src/test/results/clientpositive/groupby_sort_6.q.out
hive/trunk/ql/src/test/results/clientpositive/input23.q.out
hive/trunk/ql/src/test/results/clientpositive/input42.q.out
hive/trunk/ql/src/test/results/clientpositive/input_part1.q.out
hive/trunk/ql/src/test/results/clientpositive/input_part2.q.out
hive/trunk/ql/src/test/results/clientpositive/input_part7.q.out
hive/trunk/ql/src/test/results/clientpositive/input_part9.q.out
hive/trunk/ql/src/test/results/clientpositive/join26.q.out
hive/trunk/ql/src/test/results/clientpositive/join33.q.out
hive/trunk/ql/src/test/results/clientpositive/join9.q.out
hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/load_dyn_part8.q.out
hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/merge3.q.out
hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat8.q.out
hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat9.q.out
hive/trunk/ql/src/test/results/clientpositive/pcr.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_union_view.q.out
hive/trunk/ql/src/test/results/clientpositive/ppr_allchildsarenull.q.out
hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/sample1.q.out
hive/trunk/ql/src/test/results/clientpositive/sample10.q.out
hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_12.q.out
hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out
hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out
hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out
hive/trunk/ql/src/test/results/clientpositive/stats11.q.out
hive/trunk/ql/src/test/results/clientpositive/transform_ppr1.q.out
hive/trunk/ql/src/test/results/clientpositive/transform_ppr2.q.out
hive/trunk/ql/src/test/results/clientpositive/union22.q.out
hive/trunk/ql/src/test/results/clientpositive/union_ppr.q.out
hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
hive/trunk/ql/src/test/results/compiler/plan/cast1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input8.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_part1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_testxpath.q.xml
hive/trunk/ql/src/test/results/compiler/plan/input_testxpath2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml
hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableStructObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java
Added: hive/trunk/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java?rev=1438111&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java (added)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java Thu Jan 24 18:08:52 2013
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+public class ObjectPair<F, S> {
+ private F first;
+ private S second;
+
+ public ObjectPair() {}
+
+ public ObjectPair(F first, S second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ public F getFirst() {
+ return first;
+ }
+
+ public void setFirst(F first) {
+ this.first = first;
+ }
+
+ public S getSecond() {
+ return second;
+ }
+
+ public void setSecond(S second) {
+ this.second = second;
+ }
+}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Thu Jan 24 18:08:52 2013
@@ -157,7 +157,9 @@ public class MetaStoreUtils {
* hadoop config
* @param schema
* the properties to use to instantiate the deserializer
- * @return the Deserializer
+ * @return
+ * Returns instantiated deserializer by looking up class name of deserializer stored in passed
+ * in properties. Also, initializes the deserializer with schema stored in passed in properties.
* @exception MetaException
* if any problems instantiating the Deserializer
*
@@ -189,7 +191,10 @@ public class MetaStoreUtils {
* - hadoop config
* @param table
* the table
- * @return the Deserializer
+ * @return
+ * Returns instantiated deserializer by looking up class name of deserializer stored in
+ * storage descriptor of passed in table. Also, initializes the deserializer with schema
+ * of table.
* @exception MetaException
* if any problems instantiating the Deserializer
*
@@ -204,7 +209,7 @@ public class MetaStoreUtils {
}
try {
Deserializer deserializer = SerDeUtils.lookupDeserializer(lib);
- deserializer.initialize(conf, MetaStoreUtils.getSchema(table));
+ deserializer.initialize(conf, MetaStoreUtils.getTableMetadata(table));
return deserializer;
} catch (RuntimeException e) {
throw e;
@@ -226,7 +231,10 @@ public class MetaStoreUtils {
* @param part
* the partition
* @param table the table
- * @return the Deserializer
+ * @return
+ * Returns instantiated deserializer by looking up class name of deserializer stored in
+ * storage descriptor of passed in partition. Also, initializes the deserializer with
+ * schema of partition.
* @exception MetaException
* if any problems instantiating the Deserializer
*
@@ -237,7 +245,7 @@ public class MetaStoreUtils {
String lib = part.getSd().getSerdeInfo().getSerializationLib();
try {
Deserializer deserializer = SerDeUtils.lookupDeserializer(lib);
- deserializer.initialize(conf, MetaStoreUtils.getSchema(part, table));
+ deserializer.initialize(conf, MetaStoreUtils.getPartitionMetadata(part, table));
return deserializer;
} catch (RuntimeException e) {
throw e;
@@ -493,12 +501,21 @@ public class MetaStoreUtils {
return ddl.toString();
}
- public static Properties getSchema(
+ public static Properties getTableMetadata(
org.apache.hadoop.hive.metastore.api.Table table) {
return MetaStoreUtils.getSchema(table.getSd(), table.getSd(), table
.getParameters(), table.getDbName(), table.getTableName(), table.getPartitionKeys());
}
+ public static Properties getPartitionMetadata(
+ org.apache.hadoop.hive.metastore.api.Partition partition,
+ org.apache.hadoop.hive.metastore.api.Table table) {
+ return MetaStoreUtils
+ .getSchema(partition.getSd(), partition.getSd(), partition
+ .getParameters(), table.getDbName(), table.getTableName(),
+ table.getPartitionKeys());
+ }
+
public static Properties getSchema(
org.apache.hadoop.hive.metastore.api.Partition part,
org.apache.hadoop.hive.metastore.api.Table table) {
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Jan 24 18:08:52 2013
@@ -1286,7 +1286,7 @@ public class ObjectStore implements RawS
return null;
}
return new Partition(mpart.getValues(), dbName, tblName, mpart.getCreateTime(),
- mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), true),
+ mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), false),
mpart.getParameters());
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Thu Jan 24 18:08:52 2013
@@ -109,8 +109,6 @@ public class ExecMapper extends MapReduc
dummyOp.setExecContext(execContext);
dummyOp.initialize(jc,null);
}
-
-
} catch (Throwable e) {
abort = true;
if (e instanceof OutOfMemoryError) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Thu Jan 24 18:08:52 2013
@@ -50,6 +50,8 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.objectinspector.DelegatedObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -92,6 +94,9 @@ public class FetchOperator implements Se
private transient Writable value;
private transient Writable[] vcValues;
private transient Deserializer serde;
+ private transient Deserializer tblSerde;
+ private transient Converter partTblObjectInspectorConverter;
+
private transient Iterator<Path> iterPath;
private transient Iterator<PartitionDesc> iterPartDesc;
private transient Path currPath;
@@ -220,34 +225,35 @@ public class FetchOperator implements Se
return inputFormats.get(inputFormatClass);
}
- private StructObjectInspector setTableDesc(TableDesc table) throws Exception {
+ private StructObjectInspector getRowInspectorFromTable(TableDesc table) throws Exception {
Deserializer serde = table.getDeserializerClass().newInstance();
serde.initialize(job, table.getProperties());
- return createRowInspector(getCurrent(serde));
+ return createRowInspector(getStructOIFrom(serde.getObjectInspector()));
}
- private StructObjectInspector setPrtnDesc(PartitionDesc partition) throws Exception {
- Deserializer serde = partition.getDeserializerClass().newInstance();
- serde.initialize(job, partition.getProperties());
+ private StructObjectInspector getRowInspectorFromPartition(PartitionDesc partition,
+ ObjectInspector partitionOI) throws Exception {
+
String pcols = partition.getTableDesc().getProperties().getProperty(
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS);
String[] partKeys = pcols.trim().split("/");
row[1] = createPartValue(partKeys, partition.getPartSpec());
- return createRowInspector(getCurrent(serde), partKeys);
+
+ return createRowInspector(getStructOIFrom(partitionOI), partKeys);
}
- private StructObjectInspector setPrtnDesc(TableDesc table) throws Exception {
+ private StructObjectInspector getRowInspectorFromPartitionedTable(TableDesc table)
+ throws Exception {
Deserializer serde = table.getDeserializerClass().newInstance();
serde.initialize(job, table.getProperties());
String pcols = table.getProperties().getProperty(
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS);
String[] partKeys = pcols.trim().split("/");
row[1] = null;
- return createRowInspector(getCurrent(serde), partKeys);
+ return createRowInspector(getStructOIFrom(serde.getObjectInspector()), partKeys);
}
- private StructObjectInspector getCurrent(Deserializer serde) throws SerDeException {
- ObjectInspector current = serde.getObjectInspector();
+ private StructObjectInspector getStructOIFrom(ObjectInspector current) throws SerDeException {
if (objectInspector != null) {
current = DelegatedObjectInspectorFactory.reset(objectInspector, current);
} else {
@@ -360,16 +366,16 @@ public class FetchOperator implements Se
job.set("mapred.input.dir", org.apache.hadoop.util.StringUtils.escapeString(currPath
.toString()));
- PartitionDesc tmp;
+ PartitionDesc partDesc;
if (currTbl == null) {
- tmp = currPart;
+ partDesc = currPart;
} else {
- tmp = new PartitionDesc(currTbl, null);
+ partDesc = new PartitionDesc(currTbl, null);
}
- Class<? extends InputFormat> formatter = tmp.getInputFileFormatClass();
+ Class<? extends InputFormat> formatter = partDesc.getInputFileFormatClass();
inputFormat = getInputFormatFromCache(formatter, job);
- Utilities.copyTableJobPropertiesToConf(tmp.getTableDesc(), job);
+ Utilities.copyTableJobPropertiesToConf(partDesc.getTableDesc(), job);
InputSplit[] splits = inputFormat.getSplits(job, 1);
FetchInputFormatSplit[] inputSplits = new FetchInputFormatSplit[splits.length];
for (int i = 0; i < splits.length; i++) {
@@ -381,17 +387,32 @@ public class FetchOperator implements Se
this.inputSplits = inputSplits;
splitNum = 0;
- serde = tmp.getDeserializerClass().newInstance();
- serde.initialize(job, tmp.getProperties());
+ serde = partDesc.getDeserializerClass().newInstance();
+ serde.initialize(job, partDesc.getProperties());
+
+ if (currTbl != null) {
+ tblSerde = serde;
+ }
+ else {
+ tblSerde = currPart.getTableDesc().getDeserializerClass().newInstance();
+ tblSerde.initialize(job, currPart.getTableDesc().getProperties());
+ }
+
+ ObjectInspector outputOI = ObjectInspectorConverters.getConvertedOI(
+ serde.getObjectInspector(),
+ tblSerde.getObjectInspector());
+
+ partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(
+ serde.getObjectInspector(), outputOI);
if (LOG.isDebugEnabled()) {
LOG.debug("Creating fetchTask with deserializer typeinfo: "
+ serde.getObjectInspector().getTypeName());
- LOG.debug("deserializer properties: " + tmp.getProperties());
+ LOG.debug("deserializer properties: " + partDesc.getProperties());
}
if (currPart != null) {
- setPrtnDesc(currPart);
+ getRowInspectorFromPartition(currPart, outputOI);
}
}
@@ -503,14 +524,15 @@ public class FetchOperator implements Se
vcValues = MapOperator.populateVirtualColumnValues(context, vcCols, vcValues, serde);
row[isPartitioned ? 2 : 1] = vcValues;
}
- row[0] = serde.deserialize(value);
+ row[0] = partTblObjectInspectorConverter.convert(serde.deserialize(value));
+
if (hasVC || isPartitioned) {
inspectable.o = row;
inspectable.oi = rowObjectInspector;
return inspectable;
}
inspectable.o = row[0];
- inspectable.oi = serde.getObjectInspector();
+ inspectable.oi = tblSerde.getObjectInspector();
return inspectable;
} else {
currRecReader.close();
@@ -569,13 +591,33 @@ public class FetchOperator implements Se
public ObjectInspector getOutputObjectInspector() throws HiveException {
try {
if (work.isNotPartitioned()) {
- return setTableDesc(work.getTblDesc());
+ return getRowInspectorFromTable(work.getTblDesc());
}
List<PartitionDesc> listParts = work.getPartDesc();
+ // Chose the table descriptor if none of the partitions is present.
+ // For eg: consider the query:
+ // select /*+mapjoin(T1)*/ count(*) from T1 join T2 on T1.key=T2.key
+ // Both T1 and T2 and partitioned tables, but T1 does not have any partitions
+ // FetchOperator is invoked for T1, and listParts is empty. In that case,
+ // use T1's schema to get the ObjectInspector.
if (listParts == null || listParts.isEmpty()) {
- return setPrtnDesc(work.getTblDesc());
+ return getRowInspectorFromPartitionedTable(work.getTblDesc());
}
- return setPrtnDesc(listParts.get(0));
+
+ // Choose any partition. It's OI needs to be converted to the table OI
+ // Whenever a new partition is being read, a new converter is being created
+ PartitionDesc partition = listParts.get(0);
+ Deserializer tblSerde = partition.getTableDesc().getDeserializerClass().newInstance();
+ tblSerde.initialize(job, partition.getTableDesc().getProperties());
+
+ Deserializer partSerde = partition.getDeserializerClass().newInstance();
+ partSerde.initialize(job, partition.getProperties());
+
+ ObjectInspector partitionOI = ObjectInspectorConverters.getConvertedOI(
+ partSerde.getObjectInspector(),
+ tblSerde.getObjectInspector());
+
+ return getRowInspectorFromPartition(partition, partitionOI);
} catch (Exception e) {
throw new HiveException("Failed with exception " + e.getMessage()
+ org.apache.hadoop.util.StringUtils.stringifyException(e));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Thu Jan 24 18:08:52 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.Deserializer;
@@ -45,6 +46,8 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -78,7 +81,9 @@ public class MapOperator extends Operato
private transient Writable[] vcValues;
private transient List<VirtualColumn> vcs;
private transient Object[] rowWithPartAndVC;
- private transient StructObjectInspector rowObjectInspector;
+ private transient StructObjectInspector tblRowObjectInspector;
+ // convert from partition to table schema
+ private transient Converter partTblObjectInspectorConverter;
private transient boolean isPartitioned;
private transient boolean hasVC;
private Map<MapInputPath, MapOpCtx> opCtxMap;
@@ -112,9 +117,6 @@ public class MapOperator extends Operato
public boolean equals(Object o) {
if (o instanceof MapInputPath) {
MapInputPath mObj = (MapInputPath) o;
- if (mObj == null) {
- return false;
- }
return path.equals(mObj.path) && alias.equals(mObj.alias)
&& op.equals(mObj.op);
}
@@ -141,15 +143,16 @@ public class MapOperator extends Operato
}
private static class MapOpCtx {
- boolean isPartitioned;
- StructObjectInspector rawRowObjectInspector; // without partition
- StructObjectInspector partObjectInspector; // partition
- StructObjectInspector rowObjectInspector;
- Object[] rowWithPart;
- Object[] rowWithPartAndVC;
- Deserializer deserializer;
- public String tableName;
- public String partName;
+ private final boolean isPartitioned;
+ private final StructObjectInspector tblRawRowObjectInspector; // without partition
+ private final StructObjectInspector partObjectInspector; // partition
+ private StructObjectInspector rowObjectInspector;
+ private final Converter partTblObjectInspectorConverter;
+ private final Object[] rowWithPart;
+ private Object[] rowWithPartAndVC;
+ private final Deserializer deserializer;
+ private String tableName;
+ private String partName;
/**
* @param isPartitioned
@@ -158,18 +161,20 @@ public class MapOperator extends Operato
*/
public MapOpCtx(boolean isPartitioned,
StructObjectInspector rowObjectInspector,
- StructObjectInspector rawRowObjectInspector,
+ StructObjectInspector tblRawRowObjectInspector,
StructObjectInspector partObjectInspector,
Object[] rowWithPart,
Object[] rowWithPartAndVC,
- Deserializer deserializer) {
+ Deserializer deserializer,
+ Converter partTblObjectInspectorConverter) {
this.isPartitioned = isPartitioned;
this.rowObjectInspector = rowObjectInspector;
- this.rawRowObjectInspector = rawRowObjectInspector;
+ this.tblRawRowObjectInspector = tblRawRowObjectInspector;
this.partObjectInspector = partObjectInspector;
this.rowWithPart = rowWithPart;
this.rowWithPartAndVC = rowWithPartAndVC;
this.deserializer = deserializer;
+ this.partTblObjectInspectorConverter = partTblObjectInspectorConverter;
}
/**
@@ -186,6 +191,10 @@ public class MapOperator extends Operato
return rowObjectInspector;
}
+ public StructObjectInspector getTblRawRowObjectInspector() {
+ return tblRawRowObjectInspector;
+ }
+
/**
* @return the rowWithPart
*/
@@ -206,6 +215,10 @@ public class MapOperator extends Operato
public Deserializer getDeserializer() {
return deserializer;
}
+
+ public Converter getPartTblObjectInspectorConverter() {
+ return partTblObjectInspectorConverter;
+ }
}
/**
@@ -225,38 +238,46 @@ public class MapOperator extends Operato
}
private MapOpCtx initObjectInspector(MapredWork conf,
- Configuration hconf, String onefile) throws HiveException,
+ Configuration hconf, String onefile, Map<TableDesc, StructObjectInspector> convertedOI)
+ throws HiveException,
ClassNotFoundException, InstantiationException, IllegalAccessException,
SerDeException {
- PartitionDesc td = conf.getPathToPartitionInfo().get(onefile);
- LinkedHashMap<String, String> partSpec = td.getPartSpec();
- Properties tblProps = td.getProperties();
-
- Class sdclass = td.getDeserializerClass();
- if (sdclass == null) {
- String className = td.getSerdeClassName();
- if ((className == "") || (className == null)) {
+ PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
+ LinkedHashMap<String, String> partSpec = pd.getPartSpec();
+ // Use tblProps in case of unpartitioned tables
+ Properties partProps =
+ (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
+ pd.getTableDesc().getProperties() : pd.getProperties();
+
+ Class serdeclass = pd.getDeserializerClass();
+ if (serdeclass == null) {
+ String className = pd.getSerdeClassName();
+ if ((className == null) || (className.isEmpty())) {
throw new HiveException(
"SerDe class or the SerDe class name is not set for table: "
- + td.getProperties().getProperty("name"));
+ + pd.getProperties().getProperty("name"));
}
- sdclass = hconf.getClassByName(className);
+ serdeclass = hconf.getClassByName(className);
}
- String tableName = String.valueOf(tblProps.getProperty("name"));
+ String tableName = String.valueOf(partProps.getProperty("name"));
String partName = String.valueOf(partSpec);
- // HiveConf.setVar(hconf, HiveConf.ConfVars.HIVETABLENAME, tableName);
- // HiveConf.setVar(hconf, HiveConf.ConfVars.HIVEPARTITIONNAME, partName);
- Deserializer deserializer = (Deserializer) sdclass.newInstance();
- deserializer.initialize(hconf, tblProps);
- StructObjectInspector rawRowObjectInspector = (StructObjectInspector) deserializer
+ Deserializer partDeserializer = (Deserializer) serdeclass.newInstance();
+ partDeserializer.initialize(hconf, partProps);
+ StructObjectInspector partRawRowObjectInspector = (StructObjectInspector) partDeserializer
.getObjectInspector();
+ StructObjectInspector tblRawRowObjectInspector = convertedOI.get(pd.getTableDesc());
+
+ partTblObjectInspectorConverter =
+ ObjectInspectorConverters.getConverter(partRawRowObjectInspector,
+ tblRawRowObjectInspector);
+
MapOpCtx opCtx = null;
// Next check if this table has partitions and if so
// get the list of partition names as well as allocate
// the serdes for the partition columns
- String pcols = tblProps
+ String pcols = partProps
.getProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS);
// Log LOG = LogFactory.getLog(MapOperator.class.getName());
if (pcols != null && pcols.length() > 0) {
@@ -285,16 +306,16 @@ public class MapOperator extends Operato
rowWithPart[1] = partValues;
StructObjectInspector rowObjectInspector = ObjectInspectorFactory
.getUnionStructObjectInspector(Arrays
- .asList(new StructObjectInspector[] {rawRowObjectInspector, partObjectInspector}));
+ .asList(new StructObjectInspector[] {tblRawRowObjectInspector, partObjectInspector}));
// LOG.info("dump " + tableName + " " + partName + " " +
// rowObjectInspector.getTypeName());
- opCtx = new MapOpCtx(true, rowObjectInspector, rawRowObjectInspector, partObjectInspector,
- rowWithPart, null, deserializer);
+ opCtx = new MapOpCtx(true, rowObjectInspector, tblRawRowObjectInspector, partObjectInspector,
+ rowWithPart, null, partDeserializer, partTblObjectInspectorConverter);
} else {
// LOG.info("dump2 " + tableName + " " + partName + " " +
// rowObjectInspector.getTypeName());
- opCtx = new MapOpCtx(false, rawRowObjectInspector, rawRowObjectInspector, null, null,
- null, deserializer);
+ opCtx = new MapOpCtx(false, tblRawRowObjectInspector, tblRawRowObjectInspector, null, null,
+ null, partDeserializer, partTblObjectInspectorConverter);
}
opCtx.tableName = tableName;
opCtx.partName = partName;
@@ -312,15 +333,20 @@ public class MapOperator extends Operato
isPartitioned = opCtxMap.get(inp).isPartitioned();
rowWithPart = opCtxMap.get(inp).getRowWithPart();
rowWithPartAndVC = opCtxMap.get(inp).getRowWithPartAndVC();
- rowObjectInspector = opCtxMap.get(inp).getRowObjectInspector();
+ tblRowObjectInspector = opCtxMap.get(inp).getRowObjectInspector();
+ partTblObjectInspectorConverter = opCtxMap.get(inp).getPartTblObjectInspectorConverter();
if (listInputPaths.contains(inp)) {
return;
}
listInputPaths.add(inp);
+ // The op may not be a TableScan for mapjoins
+ // Consider the query: select /*+MAPJOIN(a)*/ count(*) FROM T1 a JOIN T2 b ON a.key = b.key;
+ // In that case, it will be a Select, but the rowOI need not be ammended
if (op instanceof TableScanOperator) {
- StructObjectInspector rawRowObjectInspector = opCtxMap.get(inp).rawRowObjectInspector;
+ StructObjectInspector tblRawRowObjectInspector =
+ opCtxMap.get(inp).getTblRawRowObjectInspector();
StructObjectInspector partObjectInspector = opCtxMap.get(inp).partObjectInspector;
TableScanOperator tsOp = (TableScanOperator) op;
TableScanDesc tsDesc = tsOp.getConf();
@@ -348,22 +374,100 @@ public class MapOperator extends Operato
this.rowWithPartAndVC = new Object[2];
}
if (partObjectInspector == null) {
- this.rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(Arrays
+ this.tblRowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(Arrays
.asList(new StructObjectInspector[] {
- rowObjectInspector, vcStructObjectInspector}));
+ tblRowObjectInspector, vcStructObjectInspector}));
} else {
- this.rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(Arrays
+ this.tblRowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(Arrays
.asList(new StructObjectInspector[] {
- rawRowObjectInspector, partObjectInspector,
+ tblRawRowObjectInspector, partObjectInspector,
vcStructObjectInspector}));
}
- opCtxMap.get(inp).rowObjectInspector = this.rowObjectInspector;
+ opCtxMap.get(inp).rowObjectInspector = this.tblRowObjectInspector;
opCtxMap.get(inp).rowWithPartAndVC = this.rowWithPartAndVC;
}
}
}
}
+ // Return the mapping for table descriptor to the expected table OI
+ /**
+ * Traverse all the partitions for a table, and get the OI for the table.
+ * Note that a conversion is required if any of the partition OI is different
+ * from the table OI. For eg. if the query references table T (partitions P1, P2),
+ * and P1's schema is same as T, whereas P2's scheme is different from T, conversion
+ * might be needed for both P1 and P2, since SettableOI might be needed for T
+ */
+ private Map<TableDesc, StructObjectInspector> getConvertedOI(Configuration hconf)
+ throws HiveException {
+ Map<TableDesc, StructObjectInspector> tableDescOI =
+ new HashMap<TableDesc, StructObjectInspector>();
+ Set<TableDesc> identityConverterTableDesc = new HashSet<TableDesc>();
+ try
+ {
+ for (String onefile : conf.getPathToAliases().keySet()) {
+ PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
+ TableDesc tableDesc = pd.getTableDesc();
+ Properties tblProps = tableDesc.getProperties();
+ // If the partition does not exist, use table properties
+ Properties partProps =
+ (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
+ tblProps : pd.getProperties();
+
+ Class sdclass = pd.getDeserializerClass();
+ if (sdclass == null) {
+ String className = pd.getSerdeClassName();
+ if ((className == null) || (className.isEmpty())) {
+ throw new HiveException(
+ "SerDe class or the SerDe class name is not set for table: "
+ + pd.getProperties().getProperty("name"));
+ }
+ sdclass = hconf.getClassByName(className);
+ }
+
+ Deserializer partDeserializer = (Deserializer) sdclass.newInstance();
+ partDeserializer.initialize(hconf, partProps);
+ StructObjectInspector partRawRowObjectInspector = (StructObjectInspector) partDeserializer
+ .getObjectInspector();
+
+ StructObjectInspector tblRawRowObjectInspector = tableDescOI.get(tableDesc);
+ if ((tblRawRowObjectInspector == null) ||
+ (identityConverterTableDesc.contains(tableDesc))) {
+ sdclass = tableDesc.getDeserializerClass();
+ if (sdclass == null) {
+ String className = tableDesc.getSerdeClassName();
+ if ((className == null) || (className.isEmpty())) {
+ throw new HiveException(
+ "SerDe class or the SerDe class name is not set for table: "
+ + tableDesc.getProperties().getProperty("name"));
+ }
+ sdclass = hconf.getClassByName(className);
+ }
+ Deserializer tblDeserializer = (Deserializer) sdclass.newInstance();
+ tblDeserializer.initialize(hconf, tblProps);
+ tblRawRowObjectInspector =
+ (StructObjectInspector) ObjectInspectorConverters.getConvertedOI(
+ partRawRowObjectInspector,
+ (StructObjectInspector) tblDeserializer.getObjectInspector());
+
+ if (identityConverterTableDesc.contains(tableDesc)) {
+ if (!partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
+ identityConverterTableDesc.remove(tableDesc);
+ }
+ }
+ else if (partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {
+ identityConverterTableDesc.add(tableDesc);
+ }
+
+ tableDescOI.put(tableDesc, tblRawRowObjectInspector);
+ }
+ }
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+ return tableDescOI;
+ }
+
public void setChildren(Configuration hconf) throws HiveException {
Path fpath = new Path((new Path(HiveConf.getVar(hconf,
@@ -375,10 +479,10 @@ public class MapOperator extends Operato
operatorToPaths = new HashMap<Operator<? extends OperatorDesc>, ArrayList<String>>();
statsMap.put(Counter.DESERIALIZE_ERRORS, deserialize_error_count);
-
+ Map<TableDesc, StructObjectInspector> convertedOI = getConvertedOI(hconf);
try {
for (String onefile : conf.getPathToAliases().keySet()) {
- MapOpCtx opCtx = initObjectInspector(conf, hconf, onefile);
+ MapOpCtx opCtx = initObjectInspector(conf, hconf, onefile, convertedOI);
Path onepath = new Path(new Path(onefile).toUri().getPath());
List<String> aliases = conf.getPathToAliases().get(onefile);
@@ -514,16 +618,18 @@ public class MapOperator extends Operato
Object row = null;
try {
if (this.hasVC) {
- this.rowWithPartAndVC[0] = deserializer.deserialize(value);
+ this.rowWithPartAndVC[0] =
+ partTblObjectInspectorConverter.convert(deserializer.deserialize(value));
int vcPos = isPartitioned ? 2 : 1;
if (context != null) {
populateVirtualColumnValues(context, vcs, vcValues, deserializer);
}
this.rowWithPartAndVC[vcPos] = this.vcValues;
} else if (!isPartitioned) {
- row = deserializer.deserialize((Writable) value);
+ row = partTblObjectInspectorConverter.convert(deserializer.deserialize((Writable) value));
} else {
- rowWithPart[0] = deserializer.deserialize((Writable) value);
+ rowWithPart[0] =
+ partTblObjectInspectorConverter.convert(deserializer.deserialize((Writable) value));
}
} catch (Exception e) {
// Serialize the row and output.
@@ -540,24 +646,26 @@ public class MapOperator extends Operato
throw new HiveException("Hive Runtime Error while processing writable " + rawRowString, e);
}
+ // The row has been converted to comply with table schema, irrespective of partition schema.
+ // So, use tblOI (and not partOI) for forwarding
try {
if (this.hasVC) {
- forward(this.rowWithPartAndVC, this.rowObjectInspector);
+ forward(this.rowWithPartAndVC, this.tblRowObjectInspector);
} else if (!isPartitioned) {
- forward(row, rowObjectInspector);
+ forward(row, tblRowObjectInspector);
} else {
- forward(rowWithPart, rowObjectInspector);
+ forward(rowWithPart, tblRowObjectInspector);
}
} catch (Exception e) {
// Serialize the row and output the error message.
String rowString;
try {
if (this.hasVC) {
- rowString = SerDeUtils.getJSONString(rowWithPartAndVC, rowObjectInspector);
+ rowString = SerDeUtils.getJSONString(rowWithPartAndVC, tblRowObjectInspector);
} else if (!isPartitioned) {
- rowString = SerDeUtils.getJSONString(row, rowObjectInspector);
+ rowString = SerDeUtils.getJSONString(row, tblRowObjectInspector);
} else {
- rowString = SerDeUtils.getJSONString(rowWithPart, rowObjectInspector);
+ rowString = SerDeUtils.getJSONString(rowWithPart, tblRowObjectInspector);
}
} catch (Exception e2) {
rowString = "[Error getting row data with exception " +
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Thu Jan 24 18:08:52 2013
@@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.persistence.RowContainer;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -40,7 +41,6 @@ import org.apache.hadoop.hive.ql.plan.Ma
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
-import org.apache.hadoop.hive.ql.util.ObjectPair;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu Jan 24 18:08:52 2013
@@ -120,8 +120,8 @@ import org.apache.hadoop.hive.ql.plan.Ma
import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
-import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.stats.StatsFactory;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
@@ -137,8 +137,8 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapred.FileOutputFormat;
@@ -559,30 +559,6 @@ public final class Utilities {
}
}
- /**
- * Tuple.
- *
- * @param <T>
- * @param <V>
- */
- public static class Tuple<T, V> {
- private final T one;
- private final V two;
-
- public Tuple(T one, V two) {
- this.one = one;
- this.two = two;
- }
-
- public T getOne() {
- return this.one;
- }
-
- public V getTwo() {
- return this.two;
- }
- }
-
public static TableDesc defaultTd;
static {
// by default we expect ^A separated strings
@@ -693,7 +669,7 @@ public final class Utilities {
public static TableDesc getTableDesc(Table tbl) {
return (new TableDesc(tbl.getDeserializer().getClass(), tbl.getInputFormatClass(), tbl
- .getOutputFormatClass(), tbl.getSchema()));
+ .getOutputFormatClass(), tbl.getMetadata()));
}
// column names and column types are all delimited by comma
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Thu Jan 24 18:08:52 2013
@@ -214,7 +214,7 @@ public class Partition implements Serial
getInputFormatClass();
// This will set up field: outputFormatClass
getOutputFormatClass();
-
+ getDeserializer();
}
public String getName() {
@@ -276,6 +276,10 @@ public class Partition implements Serial
return MetaStoreUtils.getSchema(tPartition, table.getTTable());
}
+ public Properties getMetadataFromPartitionSchema() {
+ return MetaStoreUtils.getPartitionMetadata(tPartition, table.getTTable());
+ }
+
public Properties getSchemaFromTableSchema(Properties tblSchema) {
return MetaStoreUtils.getPartSchemaFromTableSchema(tPartition.getSd(), table.getTTable().getSd(),
tPartition.getParameters(), table.getDbName(), table.getTableName(), table.getPartitionKeys(),
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Thu Jan 24 18:08:52 2013
@@ -228,8 +228,8 @@ public class Table implements Serializab
tTable.getSd().setOutputFormat(outputFormatClass.getName());
}
- final public Properties getSchema() {
- return MetaStoreUtils.getSchema(tTable);
+ final public Properties getMetadata() {
+ return MetaStoreUtils.getTableMetadata(tTable);
}
final public Path getPath() {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Thu Jan 24 18:08:52 2013
@@ -825,7 +825,12 @@ public final class GenMapRedUtils {
partDir.add(p);
try {
- partDesc.add(Utilities.getPartitionDescFromTableDesc(tblDesc, part));
+ if (part.getTable().isPartitioned()) {
+ partDesc.add(Utilities.getPartitionDesc(part));
+ }
+ else {
+ partDesc.add(Utilities.getPartitionDescFromTableDesc(tblDesc, part));
+ }
} catch (HiveException e) {
LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
throw new SemanticException(e.getMessage(), e);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Jan 24 18:08:52 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.TableType;
@@ -165,7 +166,6 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
-import org.apache.hadoop.hive.ql.util.ObjectPair;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Thu Jan 24 18:08:52 2013
@@ -87,7 +87,7 @@ public class PartitionDesc implements Se
public PartitionDesc(final org.apache.hadoop.hive.ql.metadata.Partition part)
throws HiveException {
tableDesc = Utilities.getTableDesc(part.getTable());
- properties = part.getSchema();
+ properties = part.getMetadataFromPartitionSchema();
partSpec = part.getSpec();
deserializerClass = part.getDeserializer(properties).getClass();
inputFileFormatClass = part.getInputFormatClass();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java Thu Jan 24 18:08:52 2013
@@ -20,9 +20,9 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.Enumeration;
-import java.util.Properties;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Properties;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@ -149,7 +149,7 @@ public class TableDesc implements Serial
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)
!= null);
}
-
+
@Override
public Object clone() {
TableDesc ret = new TableDesc();
@@ -170,4 +170,42 @@ public class TableDesc implements Serial
}
return ret;
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((deserializerClass == null) ? 0 : deserializerClass.hashCode());
+ result = prime * result +
+ ((inputFileFormatClass == null) ? 0 : inputFileFormatClass.hashCode());
+ result = prime * result +
+ ((outputFileFormatClass == null) ? 0 : outputFileFormatClass.hashCode());
+ result = prime * result + ((properties == null) ? 0 : properties.hashCode());
+ result = prime * result + ((serdeClassName == null) ? 0 : serdeClassName.hashCode());
+ result = prime * result + ((jobProperties == null) ? 0 : jobProperties.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TableDesc)) {
+ return false;
+ }
+
+ TableDesc target = (TableDesc) o;
+ boolean ret = true;
+ ret = ret && (deserializerClass == null ? target.deserializerClass == null :
+ deserializerClass.equals(target.deserializerClass));
+ ret = ret && (inputFileFormatClass == null ? target.inputFileFormatClass == null :
+ inputFileFormatClass.equals(target.inputFileFormatClass));
+ ret = ret && (outputFileFormatClass == null ? target.outputFileFormatClass == null :
+ outputFileFormatClass.equals(target.outputFileFormatClass));
+ ret = ret && (properties == null ? target.properties == null :
+ properties.equals(target.properties));
+ ret = ret && (serdeClassName == null ? target.serdeClassName == null :
+ serdeClassName.equals(target.serdeClassName));
+ ret = ret && (jobProperties == null ? target.jobProperties == null :
+ jobProperties.equals(target.jobProperties));
+ return ret;
+ }
}
Added: hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat10.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat10.q?rev=1438111&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat10.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat10.q Thu Jan 24 18:08:52 2013
@@ -0,0 +1,13 @@
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+-- This tests that the schema can be changed for binary serde data
+create table prt(key string, value string) partitioned by (dt string);
+insert overwrite table prt partition(dt='1') select * from src where key = 238;
+
+select * from prt where dt is not null;
+select key+key, value from prt where dt is not null;
+
+alter table prt add columns (value2 string);
+
+select key+key, value from prt where dt is not null;
+select * from prt where dt is not null;
Added: hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat11.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat11.q?rev=1438111&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat11.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat11.q Thu Jan 24 18:08:52 2013
@@ -0,0 +1,19 @@
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+-- This tests that the schema can be changed for binary serde data
+create table partition_test_partitioned(key string, value string) partitioned by (dt string) stored as rcfile;
+alter table partition_test_partitioned set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
+insert overwrite table partition_test_partitioned partition(dt='1') select * from src where key = 238;
+
+select * from partition_test_partitioned where dt is not null;
+select key+key, value from partition_test_partitioned where dt is not null;
+
+alter table partition_test_partitioned change key key int;
+
+select key+key, value from partition_test_partitioned where dt is not null;
+select * from partition_test_partitioned where dt is not null;
+
+alter table partition_test_partitioned add columns (value2 string);
+
+select key+key, value from partition_test_partitioned where dt is not null;
+select * from partition_test_partitioned where dt is not null;
Added: hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat12.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat12.q?rev=1438111&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat12.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat12.q Thu Jan 24 18:08:52 2013
@@ -0,0 +1,26 @@
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+-- This tests that the schema can be changed for binary serde data
+create table partition_test_partitioned(key string, value string) partitioned by (dt string) stored as rcfile;
+alter table partition_test_partitioned set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
+insert overwrite table partition_test_partitioned partition(dt='1') select * from src where key = 238;
+
+select * from partition_test_partitioned where dt is not null;
+select key+key, value from partition_test_partitioned where dt is not null;
+
+alter table partition_test_partitioned change key key int;
+
+select key+key, value from partition_test_partitioned where dt is not null;
+select * from partition_test_partitioned where dt is not null;
+
+insert overwrite table partition_test_partitioned partition(dt='2') select * from src where key = 97;
+
+alter table partition_test_partitioned add columns (value2 string);
+
+select key+key, value from partition_test_partitioned where dt is not null;
+select * from partition_test_partitioned where dt is not null;
+
+insert overwrite table partition_test_partitioned partition(dt='3') select key, value, value from src where key = 200;
+
+select key+key, value, value2 from partition_test_partitioned where dt is not null;
+select * from partition_test_partitioned where dt is not null;
Added: hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat13.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat13.q?rev=1438111&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat13.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat13.q Thu Jan 24 18:08:52 2013
@@ -0,0 +1,17 @@
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+-- This tests that the schema can be changed for partitioned tables for binary serde data for joins
+create table T1(key string, value string) partitioned by (dt string) stored as rcfile;
+alter table T1 set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
+insert overwrite table T1 partition (dt='1') select * from src where key = 238 or key = 97;
+
+alter table T1 change key key int;
+insert overwrite table T1 partition (dt='2') select * from src where key = 238 or key = 97;
+
+alter table T1 change key key string;
+
+create table T2(key string, value string) partitioned by (dt string) stored as rcfile;
+insert overwrite table T2 partition (dt='1') select * from src where key = 238 or key = 97;
+
+select /* + MAPJOIN(a) */ count(*) FROM T1 a JOIN T2 b ON a.key = b.key;
+select count(*) FROM T1 a JOIN T2 b ON a.key = b.key;
\ No newline at end of file
Added: hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat14.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat14.q?rev=1438111&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat14.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat14.q Thu Jan 24 18:08:52 2013
@@ -0,0 +1,57 @@
+set hive.enforce.bucketing = true;
+set hive.enforce.sorting = true;
+set hive.exec.reducers.max = 1;
+
+CREATE TABLE tbl1(key int, value string) PARTITIONED by (ds string)
+CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS rcfile;
+CREATE TABLE tbl2(key int, value string) PARTITIONED by (ds string)
+CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS rcfile;
+
+alter table tbl1 set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
+alter table tbl2 set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
+
+insert overwrite table tbl1 partition (ds='1') select * from src where key < 10;
+insert overwrite table tbl2 partition (ds='1') select * from src where key < 10;
+
+alter table tbl1 change key key int;
+insert overwrite table tbl1 partition (ds='2') select * from src where key < 10;
+
+alter table tbl1 change key key string;
+
+-- The subquery itself is being map-joined. Multiple partitions of tbl1 with different schemas are being read for tbl2
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key;
+
+set hive.optimize.bucketmapjoin = true;
+set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+
+-- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a bucketized mapside join. Multiple partitions of tbl1 with different schemas are being read for each
+-- bucket of tbl2
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key;
+
+set hive.optimize.bucketmapjoin.sortedmerge = true;
+
+-- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a sort-merge join. Multiple partitions of tbl1 with different schemas are being read for a
+-- given file of tbl2
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key;
+
+-- Since the join key is modified by the sub-query, neither sort-merge join not bucketized map-side
+-- join should be performed. Multiple partitions of tbl1 with different schemas are being read for tbl2
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key+1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
+ join
+ (select a.key+1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
+ on subq1.key = subq2.key;
Modified: hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat8.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat8.q?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat8.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat8.q Thu Jan 24 18:08:52 2013
@@ -10,3 +10,4 @@ alter table partition_test_partitioned s
insert overwrite table partition_test_partitioned partition(dt='3') select * from src;
select * from partition_test_partitioned where dt is not null order by key, value, dt limit 20;
+select key+key as key, value, dt from partition_test_partitioned where dt is not null order by key, value, dt limit 20;
Modified: hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat9.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat9.q?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat9.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat9.q Thu Jan 24 18:08:52 2013
@@ -8,3 +8,5 @@ alter table partition_test_partitioned s
insert overwrite table partition_test_partitioned partition(dt='2') select * from src;
select * from partition_test_partitioned where dt is not null order by key, value, dt limit 20;
+select key+key as key, value, dt from partition_test_partitioned where dt is not null order by key, value, dt limit 20;
+
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out Thu Jan 24 18:08:52 2013
@@ -162,7 +162,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -170,7 +169,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -212,7 +210,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -220,7 +217,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -408,7 +404,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -416,7 +411,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -458,7 +452,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -466,7 +459,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out Thu Jan 24 18:08:52 2013
@@ -150,7 +150,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -158,7 +157,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -200,7 +198,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -208,7 +205,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -396,7 +392,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -404,7 +399,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -446,7 +440,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -454,7 +447,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out Thu Jan 24 18:08:52 2013
@@ -150,7 +150,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -158,7 +157,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 1
numRows 0
partition_columns ds
rawDataSize 0
@@ -345,7 +343,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -353,7 +350,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 1
numRows 0
partition_columns ds
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out Thu Jan 24 18:08:52 2013
@@ -162,7 +162,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -170,7 +169,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 1
numRows 0
partition_columns ds
rawDataSize 0
@@ -357,7 +355,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -365,7 +362,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 1
numRows 0
partition_columns ds
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out Thu Jan 24 18:08:52 2013
@@ -149,7 +149,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -157,7 +156,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -199,7 +197,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -207,7 +204,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -393,7 +389,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -401,7 +396,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -443,7 +437,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -451,7 +444,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out Thu Jan 24 18:08:52 2013
@@ -175,7 +175,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -183,7 +182,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -225,7 +223,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -233,7 +230,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -423,7 +419,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -431,7 +426,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -473,7 +467,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
@@ -481,7 +474,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 2
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out Thu Jan 24 18:08:52 2013
@@ -175,7 +175,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -183,7 +182,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -225,7 +223,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -233,7 +230,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -423,7 +419,6 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -431,7 +426,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
@@ -473,7 +467,6 @@ STAGE PLANS:
partition values:
ds 2008-04-09
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 4
bucket_field_name key
columns key,value
@@ -481,7 +474,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.bucket_big
numFiles 4
- numPartitions 2
numRows 0
partition_columns ds
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out Thu Jan 24 18:08:52 2013
@@ -982,7 +982,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part
numFiles 4
- numPartitions 1
numRows 0
partition_columns ds
rawDataSize 0
@@ -1226,7 +1225,6 @@ STAGE PLANS:
hdfs directory: true
#### A masked pattern was here ####
-
PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result
select /*+mapjoin(a)*/ a.key, a.value, b.value
from srcbucket_mapjoin a join srcbucket_mapjoin_part b
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin10.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin10.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin10.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin10.q.out Thu Jan 24 18:08:52 2013
@@ -199,7 +199,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 2
- numPartitions 2
numRows 0
partition_columns part
rawDataSize 0
@@ -247,7 +246,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 3
- numPartitions 2
numRows 0
partition_columns part
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin11.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin11.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin11.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin11.q.out Thu Jan 24 18:08:52 2013
@@ -212,7 +212,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 2
- numPartitions 2
numRows 0
partition_columns part
rawDataSize 0
@@ -260,7 +259,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 4
- numPartitions 2
numRows 0
partition_columns part
rawDataSize 0
@@ -497,7 +495,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 2
- numPartitions 2
numRows 0
partition_columns part
rawDataSize 0
@@ -545,7 +542,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 4
- numPartitions 2
numRows 0
partition_columns part
rawDataSize 0
Modified: hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin12.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin12.q.out?rev=1438111&r1=1438110&r2=1438111&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin12.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin12.q.out Thu Jan 24 18:08:52 2013
@@ -180,7 +180,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 2
- numPartitions 1
numRows 0
partition_columns part
rawDataSize 0
@@ -409,7 +408,6 @@ STAGE PLANS:
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
numFiles 2
- numPartitions 1
numRows 0
partition_columns part
rawDataSize 0