You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/12/02 00:54:45 UTC
svn commit: r886014 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
Author: namit
Date: Tue Dec 1 23:54:44 2009
New Revision: 886014
URL: http://svn.apache.org/viewvc?rev=886014&view=rev
Log:
HIVE-961 bug in group by key wrapper
(He Yongqiang via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=886014&r1=886013&r2=886014&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Dec 1 23:54:44 2009
@@ -290,6 +290,9 @@
HIVE-957 Separate table and partition metadata
(He Yongqiang via namit)
+ HIVE-961 bug in group by key wrapper
+ (He Yongqiang via namit)
+
Release 0.4.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=886014&r1=886013&r2=886014&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Tue Dec 1 23:54:44 2009
@@ -523,14 +523,22 @@
class KeyWrapper{
int hashcode;
- ArrayList<Object> copiedKeys;
+ ArrayList<Object> keys;
+ // decide whether this is already in hashmap (keys in hashmap are deepcopied
+ // version, and we need to use 'currentKeyObjectInspector').
+ boolean copy = false;
KeyWrapper() {}
public KeyWrapper(int hashcode, ArrayList<Object> copiedKeys) {
+ this(hashcode, copiedKeys, false);
+ }
+
+ public KeyWrapper(int hashcode, ArrayList<Object> copiedKeys, boolean inHashMap) {
super();
this.hashcode = hashcode;
- this.copiedKeys = copiedKeys;
+ this.keys = copiedKeys;
+ this.copy = inHashMap;
}
public int hashCode(){
@@ -538,29 +546,32 @@
}
public boolean equals(Object obj) {
- ArrayList<Object> toBeCopied = ((KeyWrapper) obj).copiedKeys;
- return ObjectInspectorUtils.compare(toBeCopied, currentKeyObjectInspector, copiedKeys, newKeyObjectInspector) == 0;
+ ArrayList<Object> copied_in_hashmap = ((KeyWrapper) obj).keys;
+ if(!copy)
+ return ObjectInspectorUtils.compare(copied_in_hashmap, currentKeyObjectInspector, keys, newKeyObjectInspector) == 0;
+ else
+ return ObjectInspectorUtils.compare(copied_in_hashmap, currentKeyObjectInspector, keys, currentKeyObjectInspector) == 0;
}
}
+ KeyWrapper keyProber = new KeyWrapper();
private void processHashAggr(Object row, ObjectInspector rowInspector, ArrayList<Object> newKeys) throws HiveException {
// Prepare aggs for updating
AggregationBuffer[] aggs = null;
boolean newEntryForHashAggr = false;
- KeyWrapper keyProber = new KeyWrapper();
keyProber.hashcode = newKeys.hashCode();
//use this to probe the hashmap
- keyProber.copiedKeys = newKeys;
+ keyProber.keys = newKeys;
// hash-based aggregations
aggs = hashAggregations.get(keyProber);
ArrayList<Object> newDefaultKeys = null;
if(aggs == null) {
newDefaultKeys = deepCopyElements(keyObjects, keyObjectInspectors, ObjectInspectorCopyOption.WRITABLE);
- keyProber.copiedKeys = newDefaultKeys;
+ KeyWrapper newKeyProber = new KeyWrapper(keyProber.hashcode, newDefaultKeys, true);
aggs = newAggregations();
- hashAggregations.put(keyProber, aggs);
+ hashAggregations.put(newKeyProber, aggs);
newEntryForHashAggr = true;
numRowsHashTbl++; // new entry in the hash table
}
@@ -684,7 +695,7 @@
iter = hashAggregations.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry<KeyWrapper, AggregationBuffer[]> m = iter.next();
- forward(m.getKey().copiedKeys, m.getValue());
+ forward(m.getKey().keys, m.getValue());
}
hashAggregations.clear();
hashAggregations = null;
@@ -699,7 +710,7 @@
int numDel = 0;
while (iter.hasNext()) {
Map.Entry<KeyWrapper, AggregationBuffer[]> m = iter.next();
- forward(m.getKey().copiedKeys, m.getValue());
+ forward(m.getKey().keys, m.getValue());
iter.remove();
numDel++;
if (numDel * 10 >= oldSize) {
@@ -763,7 +774,7 @@
Iterator iter = hashAggregations.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry<KeyWrapper, AggregationBuffer[]> m = (Map.Entry)iter.next();
- forward(m.getKey().copiedKeys, m.getValue());
+ forward(m.getKey().keys, m.getValue());
iter.remove();
}
hashAggregations.clear();