You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jv...@apache.org on 2011/08/02 06:24:53 UTC
svn commit: r1153022 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/index/
java/org/apache/hadoop/hive/ql/index/bitmap/
java/org/apache/hadoop/hive/ql/index/compact/
java/org/apache/hadoop/hive/ql/udf/generic/ test/queries/clientnegative/
test...
Author: jvs
Date: Tue Aug 2 04:24:52 2011
New Revision: 1153022
URL: http://svn.apache.org/viewvc?rev=1153022&view=rev
Log:
HIVE-2326. Turn off bitmap indexing when map-side aggregation is turned off
(Syed Albiz via jvs)
Added:
hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q
hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java Tue Aug 2 04:24:52 2011
@@ -101,7 +101,7 @@ public abstract class TableBasedIndexHan
abstract protected Task<?> getIndexBuilderMapRedTask(Set<ReadEntity> inputs, Set<WriteEntity> outputs,
List<FieldSchema> indexField, boolean partitioned,
PartitionDesc indexTblPartDesc, String indexTableName,
- PartitionDesc baseTablePartDesc, String baseTableName, String dbName);
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName) throws HiveException;
protected List<String> getPartKVPairStringArray(
LinkedHashMap<String, String> partSpec) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java Tue Aug 2 04:24:52 2011
@@ -220,7 +220,7 @@ public class BitmapIndexHandler extends
protected Task<?> getIndexBuilderMapRedTask(Set<ReadEntity> inputs, Set<WriteEntity> outputs,
List<FieldSchema> indexField, boolean partitioned,
PartitionDesc indexTblPartDesc, String indexTableName,
- PartitionDesc baseTablePartDesc, String baseTableName, String dbName) {
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName) throws HiveException {
HiveConf conf = new HiveConf(getConf(), BitmapIndexHandler.class);
HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEROWOFFSET, true);
@@ -280,9 +280,9 @@ public class BitmapIndexHandler extends
}
// Require clusterby ROWOFFSET if map-size aggregation is off.
+ // TODO: Make this work without map side aggregation
if (!conf.get("hive.map.aggr", null).equals("true")) {
- command.append(" CLUSTER BY ");
- command.append(VirtualColumn.ROWOFFSET.getName());
+ throw new HiveException("Cannot construct index without map-side aggregation");
}
Driver driver = new Driver(conf);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java Tue Aug 2 04:24:52 2011
@@ -81,7 +81,7 @@ public class CompactIndexHandler extends
protected Task<?> getIndexBuilderMapRedTask(Set<ReadEntity> inputs, Set<WriteEntity> outputs,
List<FieldSchema> indexField, boolean partitioned,
PartitionDesc indexTblPartDesc, String indexTableName,
- PartitionDesc baseTablePartDesc, String baseTableName, String dbName) {
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName) throws HiveException {
String indexCols = HiveUtils.getUnparsedColumnNamesFromFieldSchema(indexField);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java?rev=1153022&r1=1153021&r2=1153022&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java Tue Aug 2 04:24:52 2011
@@ -92,14 +92,19 @@ public class GenericUDAFEWAHBitmap exten
inputOI = (PrimitiveObjectInspector) parameters[0];
return ObjectInspectorFactory
.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
- } else {
- //no map aggregation
+ } else if (m == Mode.PARTIAL2 || m == Mode.FINAL) {
internalMergeOI = (StandardListObjectInspector) parameters[0];
bitmapLongOI = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector;
loi = (StandardListObjectInspector) ObjectInspectorFactory
.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
return loi;
+ } else { // Mode.COMPLETE, ie. no map-side aggregation, requires ordering
+ bitmapLongOI = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector;
+ loi = (StandardListObjectInspector) ObjectInspectorFactory
+ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+ return loi;
}
}
Added: hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q?rev=1153022&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/index_bitmap_no_map_aggr.q Tue Aug 2 04:24:52 2011
@@ -0,0 +1,7 @@
+EXPLAIN
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD;
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+SET hive.map.aggr=false;
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD;
+ALTER INDEX src1_index ON src REBUILD;
Added: hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out?rev=1153022&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/index_bitmap_no_map_aggr.q.out Tue Aug 2 04:24:52 2011
@@ -0,0 +1,21 @@
+PREHOOK: query: EXPLAIN
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+POSTHOOK: query: EXPLAIN
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+ABSTRACT SYNTAX TREE:
+ (TOK_CREATEINDEX src1_index 'BITMAP' (TOK_TABNAME src) (TOK_TABCOLNAME key) TOK_DEFERRED_REBUILDINDEX)
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+
+
+PREHOOK: query: CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+POSTHOOK: query: CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+FAILED: Error in semantic analysis: org.apache.hadoop.hive.ql.parse.SemanticException: org.apache.hadoop.hive.ql.metadata.HiveException: Cannot construct index without map-side aggregation