You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/01/16 00:57:14 UTC
svn commit: r1652314 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpo...
Author: sershe
Date: Thu Jan 15 23:57:14 2015
New Revision: 1652314
URL: http://svn.apache.org/r1652314
Log:
HIVE-9331 : get rid of pre-optimized-hashtable memory optimizations (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java
Removed:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyBytes.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q
hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jan 15 23:57:14 2015
@@ -699,13 +699,6 @@ public class HiveConf extends Configurat
HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true,
"Whether Hive should use memory-optimized hash table for MapJoin. Only works on Tez,\n" +
"because memory-optimized hashtable cannot be serialized."),
- HIVEMAPJOINUSEOPTIMIZEDKEYS("hive.mapjoin.optimized.keys", true,
- "Whether MapJoin hashtable should use optimized (size-wise), keys, allowing the table to take less\n" +
- "memory. Depending on key, the memory savings for entire table can be 5-15% or so."),
- HIVEMAPJOINLAZYHASHTABLE("hive.mapjoin.lazy.hashtable", true,
- "Whether MapJoin hashtable should deserialize values on demand. Depending on how many values in\n" +
- "the table the join will actually touch, it can save a lot of memory by not creating objects for\n" +
- "rows that are not needed. If all rows are needed obviously there's no gain."),
HIVEHASHTABLEWBSIZE("hive.mapjoin.optimized.hashtable.wbsize", 10 * 1024 * 1024,
"Optimized hashtable (see hive.mapjoin.optimized.hashtable) uses a chain of buffers to\n" +
"store data. This is one buffer size. HT may be slightly faster if this is larger, but for small\n" +
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java?rev=1652314&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java Thu Jan 15 23:57:14 2015
@@ -0,0 +1,401 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.persistence;
+
+import java.io.ObjectOutputStream;
+import java.util.AbstractCollection;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+
+@SuppressWarnings("deprecation")
+public class FlatRowContainer extends AbstractCollection<Object>
+ implements MapJoinRowContainer, AbstractRowContainer.RowIterator<List<Object>>, List<Object> {
+ private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0];
+ private static final int UNKNOWN = Integer.MAX_VALUE;
+
+ private static Log LOG = LogFactory.getLog(FlatRowContainer.class);
+
+ /**
+ * In lazy mode, 0s element contains context for deserialization and all the other
+ * elements contains byte arrays to be deserialized. After deserialization, the array
+ * contains row count * row size elements - a matrix of rows stored.
+ */
+ private Object[] array;
+ /**
+ * This is kind of tricky. UNKNOWN number means unknown. Other positive numbers represent
+ * row length (see array javadoc). Non-positive numbers mean row length is zero (thus,
+ * array is empty); they represent (negated) number of rows (for joins w/o projections).
+ */
+ private int rowLength = UNKNOWN;
+ private byte aliasFilter = (byte) 0xff;
+ private boolean isAliasFilterSet = true; // by default assume no filter tag so we are good
+
+ public FlatRowContainer() {
+ this.array = EMPTY_OBJECT_ARRAY;
+ }
+
+ /** Called when loading the hashtable. */
+ public void add(MapJoinObjectSerDeContext context,
+ BytesWritable value) throws HiveException {
+ SerDe serde = context.getSerDe();
+ isAliasFilterSet = !context.hasFilterTag(); // has tag => need to set later
+ if (rowLength == UNKNOWN) {
+ try {
+ rowLength = ObjectInspectorUtils.getStructSize(serde.getObjectInspector());
+ } catch (SerDeException ex) {
+ throw new HiveException("Get structure size error", ex);
+ }
+ if (rowLength == 0) {
+ array = EMPTY_OBJECT_ARRAY;
+ }
+ }
+ if (rowLength > 0) {
+ int rowCount = (array.length / rowLength);
+ listRealloc(array.length + rowLength);
+ read(serde, value, rowCount);
+ } else {
+ --rowLength; // see rowLength javadoc
+ }
+ }
+
+ // Implementation of AbstractRowContainer and assorted methods
+
+ @Override
+ public void addRow(List<Object> t) throws HiveException {
+ LOG.debug("Add is called with " + t.size() + " objects");
+ // This is not called when building HashTable; we don't expect it to be called ever.
+ int offset = prepareForAdd(t.size());
+ if (offset < 0) return;
+ for (int i = 0; i < t.size(); ++i) {
+ this.array[offset + i] = t.get(i);
+ }
+ }
+
+ @Override
+ public void addRow(Object[] value) throws HiveException {
+ LOG.debug("Add is called with " + value.length + " objects");
+ // This is not called when building HashTable; we don't expect it to be called ever.
+ int offset = prepareForAdd(value.length);
+ if (offset < 0) return;
+ System.arraycopy(value, 0, this.array, offset, value.length);
+ }
+
+ private int prepareForAdd(int len) throws HiveException {
+ if (rowLength < 0) {
+ if (len != 0) {
+ throw new HiveException("Different size rows: 0 and " + len);
+ }
+ --rowLength; // see rowLength javadoc
+ return -1;
+ }
+ if (rowLength != len) {
+ throw new HiveException("Different size rows: " + rowLength + " and " + len);
+ }
+ int oldLen = this.array.length;
+ listRealloc(oldLen + len);
+ return oldLen;
+ }
+
+ @Override
+ public void write(MapJoinObjectSerDeContext valueContext, ObjectOutputStream out) {
+ throw new UnsupportedOperationException(this.getClass().getName() + " cannot be serialized");
+ }
+
+ @Override
+ public AbstractRowContainer.RowIterator<List<Object>> rowIter() throws HiveException {
+ if (array.length == rowLength) {
+ // optimize for common case - just one row for a key, container acts as iterator
+ return this;
+ }
+ return rowLength > 0 ? new RowIterator() : new EmptyRowIterator(-rowLength);
+ }
+
+ @Override
+ public List<Object> first() throws HiveException {
+ if (array.length != rowLength) {
+ throw new AssertionError("Incorrect iterator usage, not single-row");
+ }
+ return this; // optimize for common case - just one row for a key, container acts as row
+ }
+
+ @Override
+ public List<Object> next() {
+ return null; // single-row case, there's no next
+ }
+
+ /** Iterator for row length 0. */
+ private static class EmptyRowIterator implements AbstractRowContainer.RowIterator<List<Object>> {
+ private static final List<Object> EMPTY_ROW = new ArrayList<Object>();
+ private int rowCount;
+ public EmptyRowIterator(int rowCount) {
+ this.rowCount = rowCount;
+ }
+
+ @Override
+ public List<Object> first() throws HiveException {
+ return next();
+ }
+
+ @Override
+ public List<Object> next() throws HiveException {
+ return (--rowCount < 0) ? null : EMPTY_ROW;
+ }
+ }
+
+ /** Row iterator for non-zero-length rows. */
+ private class RowIterator implements AbstractRowContainer.RowIterator<List<Object>> {
+ private int index = 0;
+
+ @Override
+ public List<Object> first() throws HiveException {
+ index = 0;
+ if (array.length > 0) {
+ return new ReadOnlySubList(0, rowLength);
+ }
+ return null;
+ }
+
+ @Override
+ public List<Object> next() {
+ index += rowLength;
+ if (index < array.length) {
+ return new ReadOnlySubList(index, rowLength);
+ }
+ return null;
+ }
+ }
+
+ private void read(SerDe serde, Writable writable, int rowOffset) throws HiveException {
+ try {
+ ObjectInspectorUtils.copyStructToArray(
+ serde.deserialize(writable), serde.getObjectInspector(),
+ ObjectInspectorCopyOption.WRITABLE, this.array, rowOffset * rowLength);
+ } catch (SerDeException ex) {
+ throw new HiveException("Lazy deserialize error", ex);
+ }
+ }
+
+ @Override
+ public int rowCount() throws HiveException {
+ return rowLength > 0 ? (array.length / rowLength) : -rowLength; // see rowLength javadoc
+ }
+
+ @Override
+ public void clearRows() {
+ array = EMPTY_OBJECT_ARRAY;
+ rowLength = 0;
+ }
+
+ @Override
+ public byte getAliasFilter() throws HiveException {
+ ensureAliasFilter();
+ return this.aliasFilter;
+ }
+
+ private void ensureAliasFilter() throws HiveException {
+ if (!isAliasFilterSet && rowLength > 0) {
+ for (int offset = rowLength - 1; offset < array.length; offset += rowLength) {
+ aliasFilter &= ((ShortWritable)array[offset]).get();
+ }
+ }
+ isAliasFilterSet = true;
+ }
+
+ @Override
+ public MapJoinRowContainer copy() throws HiveException {
+ FlatRowContainer result = new FlatRowContainer();
+ result.array = new Object[this.array.length];
+ System.arraycopy(this.array, 0, result.array, 0, this.array.length);
+ result.rowLength = rowLength;
+ result.aliasFilter = aliasFilter;
+ return result;
+ }
+
+ // Implementation of List<Object> and assorted methods
+
+ private void listRealloc(int length) {
+ Object[] array = new Object[length];
+ if (this.array.length > 0) {
+ System.arraycopy(this.array, 0, array, 0, this.array.length);
+ }
+ this.array = array;
+ }
+
+ @Override
+ public int size() {
+ checkSingleRow();
+ return array.length;
+ }
+
+ @Override
+ public Object get(int index) {
+ return array[index];
+ }
+
+ private class ReadOnlySubList extends AbstractList<Object> {
+ private int offset;
+ private int size;
+
+ ReadOnlySubList(int from, int size) {
+ this.offset = from;
+ this.size = size;
+ }
+
+ public Object get(int index) {
+ return array[index + offset];
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public Iterator<Object> iterator() {
+ return listIterator();
+ }
+
+ public ListIterator<Object> listIterator(int index) {
+ return listIteratorInternal(offset + index, offset, offset + size);
+ }
+
+ public List<Object> subList(int fromIndex, int toIndex) {
+ return new ReadOnlySubList(offset + fromIndex, toIndex - fromIndex);
+ }
+
+ public Object[] toArray() {
+ Object[] result = new Object[size];
+ System.arraycopy(array, offset, result, 0, size);
+ return result;
+ }
+ } // end ReadOnlySubList
+
+ @Override
+ public Object[] toArray() {
+ checkSingleRow();
+ return array;
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return listIterator();
+ }
+
+ @Override
+ public ListIterator<Object> listIterator() {
+ return listIterator(0);
+ }
+
+ @Override
+ public ListIterator<Object> listIterator(final int index) {
+ checkSingleRow();
+ return listIteratorInternal(index, 0, array.length);
+ }
+
+ private ListIterator<Object> listIteratorInternal(
+ final int index, final int iterMinPos, final int iterMaxPos) {
+ return new ListIterator<Object>() {
+ private int pos = index - 1;
+ public int nextIndex() {
+ return pos + 1;
+ }
+ public int previousIndex() {
+ return pos - 1;
+ }
+ public boolean hasNext() {
+ return nextIndex() < iterMaxPos;
+ }
+ public boolean hasPrevious() {
+ return previousIndex() >= iterMinPos;
+ }
+ public Object next() {
+ if (!hasNext()) throw new NoSuchElementException();
+ return get(++pos);
+ }
+ public Object previous() {
+ if (!hasPrevious()) throw new NoSuchElementException();
+ return get(--pos);
+ }
+
+ public void remove() { throw new UnsupportedOperationException(); }
+ public void set(Object e) { throw new UnsupportedOperationException(); }
+ public void add(Object e) { throw new UnsupportedOperationException(); }
+ }; // end ListIterator
+ }
+
+ @Override
+ public int indexOf(Object o) {
+ checkSingleRow();
+ for (int i = 0; i < array.length; ++i) {
+ if (o == null) {
+ if (array[i] == null) return i;
+ } else {
+ if (o.equals(array[i])) return i;
+ }
+ }
+ return -1;
+ }
+
+ private void checkSingleRow() throws AssertionError {
+ if (array.length != rowLength) {
+ throw new AssertionError("Incorrect list usage, not single-row");
+ }
+ }
+
+ @Override
+ public int lastIndexOf(Object o) {
+ checkSingleRow();
+ for (int i = array.length - 1; i >= 0; --i) {
+ if (o == null) {
+ if (array[i] == null) return i;
+ } else {
+ if (o.equals(array[i])) return i;
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public List<Object> subList(int fromIndex, int toIndex) {
+ checkSingleRow();
+ return new ReadOnlySubList(fromIndex, toIndex - fromIndex);
+ }
+
+ public boolean addAll(int index, Collection<? extends Object> c) {
+ throw new UnsupportedOperationException();
+ }
+ public Object set(int index, Object element) { throw new UnsupportedOperationException(); }
+ public void add(int index, Object element) { throw new UnsupportedOperationException(); }
+ public Object remove(int index) { throw new UnsupportedOperationException(); }
+}
+
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java Thu Jan 15 23:57:14 2015
@@ -58,8 +58,6 @@ public class HashMapWrapper extends Abst
private static final float LOADFACTOR = 0.75f;
private final HashMap<MapJoinKey, MapJoinRowContainer> mHash; // main memory HashMap
private MapJoinKey lastKey = null;
- private final boolean useLazyRows;
- private final boolean useOptimizedKeys;
private Output output = new Output(0); // Reusable output for serialization
public HashMapWrapper(Map<String, String> metaData) {
@@ -67,30 +65,24 @@ public class HashMapWrapper extends Abst
int threshold = Integer.parseInt(metaData.get(THESHOLD_NAME));
float loadFactor = Float.parseFloat(metaData.get(LOAD_NAME));
mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor);
- useLazyRows = useOptimizedKeys = false;
}
public HashMapWrapper() {
this(HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT.defaultFloatVal,
HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD.defaultIntVal,
- HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, false, false, -1);
+ HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, -1);
}
public HashMapWrapper(Configuration hconf, long keyCount) {
this(HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT),
HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
- HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR),
- HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINLAZYHASHTABLE),
- HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS), keyCount);
+ HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR), keyCount);
}
- private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor,
- boolean useLazyRows, boolean useOptimizedKeys, long keyCount) {
+ private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor, long keyCount) {
super(createConstructorMetaData(threshold, loadFactor));
threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount);
mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor);
- this.useLazyRows = useLazyRows;
- this.useOptimizedKeys = useOptimizedKeys;
}
public static int calculateTableSize(
@@ -131,21 +123,14 @@ public class HashMapWrapper extends Abst
public MapJoinKey putRow(MapJoinObjectSerDeContext keyContext, Writable currentKey,
MapJoinObjectSerDeContext valueContext, Writable currentValue)
throws SerDeException, HiveException {
- // We pass key in as reference, to find out quickly if optimized keys can be used.
- // However, we do not reuse the object since we are putting them into the hashmap.
- // Later, we don't create optimized keys in MapJoin if hash map doesn't have optimized keys.
- if (lastKey == null && !useOptimizedKeys) {
- lastKey = new MapJoinKeyObject();
- }
-
- lastKey = MapJoinKey.read(output, lastKey, keyContext, currentKey, false);
- LazyFlatRowContainer values = (LazyFlatRowContainer)get(lastKey);
+ MapJoinKey key = MapJoinKey.read(output, keyContext, currentKey);
+ FlatRowContainer values = (FlatRowContainer)get(key);
if (values == null) {
- values = new LazyFlatRowContainer();
- put(lastKey, values);
+ values = new FlatRowContainer();
+ put(key, values);
}
- values.add(valueContext, (BytesWritable)currentValue, useLazyRows);
- return lastKey;
+ values.add(valueContext, (BytesWritable)currentValue);
+ return key;
}
@Override
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java Thu Jan 15 23:57:14 2015
@@ -57,22 +57,11 @@ public abstract class MapJoinKey {
public abstract boolean hasAnyNulls(int fieldCount, boolean[] nullsafes);
@SuppressWarnings("deprecation")
- public static MapJoinKey read(Output output, MapJoinKey key,
- MapJoinObjectSerDeContext context, Writable writable, boolean mayReuseKey)
- throws SerDeException, HiveException {
+ public static MapJoinKey read(Output output, MapJoinObjectSerDeContext context,
+ Writable writable) throws SerDeException, HiveException {
SerDe serde = context.getSerDe();
Object obj = serde.deserialize(writable);
- boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
- if (useOptimized || key == null) {
- byte[] structBytes = serialize(output, obj, serde.getObjectInspector(), !useOptimized);
- if (structBytes != null) {
- return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
- } else if (useOptimized) {
- throw new SerDeException(
- "Failed to serialize " + obj + " even though optimized keys are used");
- }
- }
- MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
+ MapJoinKeyObject result = new MapJoinKeyObject();
result.read(serde.getObjectInspector(), obj);
return result;
}
@@ -98,35 +87,6 @@ public abstract class MapJoinKey {
SUPPORTED_PRIMITIVES.add(PrimitiveCategory.CHAR);
}
- private static byte[] serialize(Output byteStream,
- Object obj, ObjectInspector oi, boolean checkTypes) throws HiveException {
- if (null == obj || !(oi instanceof StructObjectInspector)) {
- return null; // not supported
- }
- StructObjectInspector soi = (StructObjectInspector)oi;
- List<? extends StructField> fields = soi.getAllStructFieldRefs();
- int size = fields.size();
- if (size > 8) {
- return null; // not supported
- } else if (size == 0) {
- return EMPTY_BYTE_ARRAY; // shortcut for null keys
- }
- Object[] fieldData = new Object[size];
- List<ObjectInspector> fieldOis = new ArrayList<ObjectInspector>(size);
- for (int i = 0; i < size; ++i) {
- StructField field = fields.get(i);
- ObjectInspector foi = field.getFieldObjectInspector();
- if (checkTypes && !isSupportedField(foi)) {
- return null;
- }
- fieldData[i] = soi.getStructFieldData(obj, field);
- fieldOis.add(foi);
- }
-
- byteStream = serializeRow(byteStream, fieldData, fieldOis, null);
- return Arrays.copyOf(byteStream.getData(), byteStream.getLength());
- }
-
public static boolean isSupportedField(ObjectInspector foi) {
if (foi.getCategory() != Category.PRIMITIVE) return false; // not supported
PrimitiveCategory pc = ((PrimitiveObjectInspector)foi).getPrimitiveCategory();
@@ -136,19 +96,6 @@ public abstract class MapJoinKey {
public static MapJoinKey readFromVector(Output output, MapJoinKey key, Object[] keyObject,
List<ObjectInspector> keyOIs, boolean mayReuseKey) throws HiveException {
- boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
- if (useOptimized || key == null) {
- if (keyObject.length <= 8) {
- output = serializeRow(output, keyObject, keyOIs, null);
- return MapJoinKeyBytes.fromBytes(key, mayReuseKey,
- Arrays.copyOf(output.getData(), output.getLength()));
- }
- if (useOptimized) {
- throw new HiveException(
- "Failed to serialize " + Arrays.toString(keyObject) +
- " even though optimized keys are used");
- }
- }
MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
result.setKeyObjects(keyObject);
return result;
@@ -178,32 +125,11 @@ public abstract class MapJoinKey {
public static MapJoinKey readFromRow(Output output, MapJoinKey key, Object[] keyObject,
List<ObjectInspector> keyFieldsOI, boolean mayReuseKey) throws HiveException {
- boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
- if (useOptimized || key == null) {
- if (keyObject.length <= 8) {
- byte[] structBytes;
- if (keyObject.length == 0) {
- structBytes = EMPTY_BYTE_ARRAY; // shortcut for null keys
- } else {
- output = serializeRow(output, keyObject, keyFieldsOI, null);
- structBytes = Arrays.copyOf(output.getData(), output.getLength());
- }
- return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
- }
- if (useOptimized) {
- throw new HiveException(
- "Failed to serialize " + Arrays.toString(keyObject) +
- " even though optimized keys are used");
- }
- }
MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
result.readFromRow(keyObject, keyFieldsOI);
return result;
}
- private static final Log LOG = LogFactory.getLog(MapJoinKey.class);
-
-
/**
* Serializes row to output.
* @param byteStream Output to reuse. Can be null, in that case a new one would be created.
@@ -228,8 +154,4 @@ public abstract class MapJoinKey {
}
return byteStream;
}
-
- private static boolean useOptimizedKeyBasedOnPrev(MapJoinKey key) {
- return (key != null) && (key instanceof MapJoinKeyBytes);
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java Thu Jan 15 23:57:14 2015
@@ -57,8 +57,6 @@ public class HashTableLoader implements
private ExecMapperContext context;
private Configuration hconf;
private MapJoinDesc desc;
- private MapJoinKey lastKey = null;
- private int rowCount = 0;
@Override
public void init(ExecMapperContext context, Configuration hconf, MapJoinOperator joinOp) {
@@ -111,8 +109,7 @@ public class HashTableLoader implements
: new HashMapWrapper(hconf, keyCount);
while (kvReader.next()) {
- rowCount++;
- lastKey = tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),
+ tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),
valCtx, (Writable)kvReader.getCurrentValue());
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q Thu Jan 15 23:57:14 2015
@@ -29,11 +29,6 @@ explain
select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec);
set hive.mapjoin.optimized.hashtable=false;
-set hive.mapjoin.optimized.keys=false;
-
-select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec);
-
-set hive.mapjoin.optimized.keys=true;
select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec);
Modified: hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out Thu Jan 15 23:57:14 2015
@@ -381,119 +381,3 @@ POSTHOOK: Input: default@t2
9 9
9 9
9 9
-PREHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t1
-PREHOOK: Input: default@t2
-#### A masked pattern was here ####
-POSTHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t1
-POSTHOOK: Input: default@t2
-#### A masked pattern was here ####
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-45 45
-45 45
-45 45
-45 45
-45 45
-6 6
-6 6
-6 6
-6 6
-6 6
-6 6
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-70 70
-70 70
-70 70
-70 70
-70 70
-70 70
-70 70
-79 79
-79 79
-79 79
-79 79
-79 79
-79 79
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out Thu Jan 15 23:57:14 2015
@@ -374,119 +374,3 @@ POSTHOOK: Input: default@t2
9 9
9 9
9 9
-PREHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t1
-PREHOOK: Input: default@t2
-#### A masked pattern was here ####
-POSTHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t1
-POSTHOOK: Input: default@t2
-#### A masked pattern was here ####
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-14 14
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-17 17
-45 45
-45 45
-45 45
-45 45
-45 45
-6 6
-6 6
-6 6
-6 6
-6 6
-6 6
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-62 62
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-64 64
-70 70
-70 70
-70 70
-70 70
-70 70
-70 70
-70 70
-79 79
-79 79
-79 79
-79 79
-79 79
-79 79
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-89 89
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9
-9 9