You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2017/03/21 16:27:28 UTC
hive git commit: HIVE-16166: HS2 may still waste up to 15% of memory
on duplicate strings (Misha Dmitriev, reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/master 2e0493bdd -> 3c1dfe379
HIVE-16166: HS2 may still waste up to 15% of memory on duplicate strings (Misha Dmitriev, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c1dfe37
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c1dfe37
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c1dfe37
Branch: refs/heads/master
Commit: 3c1dfe379ec1fcaa688b064561a4daca098ed2b8
Parents: 2e0493b
Author: Misha Dmitriev <mi...@cloudera.com>
Authored: Tue Mar 21 11:27:02 2017 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Tue Mar 21 11:27:02 2017 -0500
----------------------------------------------------------------------
.../hadoop/hive/common/StringInternUtils.java | 16 ++++++++++++----
.../org/apache/hadoop/hive/ql/exec/ColumnInfo.java | 7 ++++---
.../apache/hadoop/hive/ql/hooks/LineageInfo.java | 3 ++-
.../hadoop/hive/ql/plan/ExprNodeConstantDesc.java | 6 +++++-
.../org/apache/hadoop/hive/ql/plan/TableDesc.java | 4 +++-
.../apache/hadoop/hive/serde2/avro/AvroSerDe.java | 6 ++++--
.../objectinspector/ObjectInspectorFactory.java | 11 +++++++----
.../StandardStructObjectInspector.java | 2 +-
8 files changed, 38 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/common/src/java/org/apache/hadoop/hive/common/StringInternUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/StringInternUtils.java b/common/src/java/org/apache/hadoop/hive/common/StringInternUtils.java
index c729991..92d37e8 100644
--- a/common/src/java/org/apache/hadoop/hive/common/StringInternUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/StringInternUtils.java
@@ -104,13 +104,21 @@ public class StringInternUtils {
* This method interns all the strings in the given list in place. That is,
* it iterates over the list, replaces each element with the interned copy
* and eventually returns the same list.
+ *
+ * Note that the provided List implementation should return an iterator
+ * (via list.listIterator()) method, and that iterator should implement
+ * the set(Object) method. That's what all List implementations in the JDK
+ * provide. However, if some custom List implementation doesn't have this
+ * functionality, this method will return without interning its elements.
*/
public static List<String> internStringsInList(List<String> list) {
if (list != null) {
- ListIterator<String> it = list.listIterator();
- while (it.hasNext()) {
- it.set(it.next().intern());
- }
+ try {
+ ListIterator<String> it = list.listIterator();
+ while (it.hasNext()) {
+ it.set(it.next().intern());
+ }
+ } catch (UnsupportedOperationException e) { } // set() not implemented - ignore
}
return list;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
index e3da7f0..bb8dcbb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
+import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -96,7 +97,7 @@ public class ColumnInfo implements Serializable {
this.tabAlias = tabAlias;
this.isVirtualCol = isVirtualCol;
this.isHiddenVirtualCol = isHiddenVirtualCol;
- this.typeName = getType().getTypeName();
+ setTypeName(getType().getTypeName());
}
public ColumnInfo(ColumnInfo columnInfo) {
@@ -114,7 +115,7 @@ public class ColumnInfo implements Serializable {
}
public void setTypeName(String typeName) {
- this.typeName = typeName;
+ this.typeName = StringInternUtils.internIfNotNull(typeName);
}
public TypeInfo getType() {
@@ -160,7 +161,7 @@ public class ColumnInfo implements Serializable {
}
public void setAlias(String col_alias) {
- alias = col_alias;
+ alias = StringInternUtils.internIfNotNull(col_alias);
}
public String getAlias() {
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
index 2806c54..7305436 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.commons.collections.SetUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -403,7 +404,7 @@ public class LineageInfo implements Serializable {
* @param expr the expr to set
*/
public void setExpr(String expr) {
- this.expr = expr;
+ this.expr = StringInternUtils.internIfNotNull(expr);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
index a5221a2..9bb3df4 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.List;
import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
@@ -67,7 +68,7 @@ public class ExprNodeConstantDesc extends ExprNodeDesc implements Serializable {
public ExprNodeConstantDesc(TypeInfo typeInfo, Object value) {
super(typeInfo);
- this.value = value;
+ setValue(value);
}
public ExprNodeConstantDesc(Object value) {
@@ -77,6 +78,9 @@ public class ExprNodeConstantDesc extends ExprNodeDesc implements Serializable {
public void setValue(Object value) {
// Kryo setter
+ if (value instanceof String) {
+ value = StringInternUtils.internIfNotNull((String) value);
+ }
this.value = value;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
index 63dc94c..117aa14 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.plan;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -67,7 +68,7 @@ public class TableDesc implements Serializable, Cloneable {
this.inputFileFormatClass = inputFormatClass;
outputFileFormatClass = HiveFileFormatUtils
.getOutputFormatSubstitute(outputFormatClass);
- this.properties = properties;
+ setProperties(properties);
}
public Class<? extends Deserializer> getDeserializerClass() {
@@ -129,6 +130,7 @@ public class TableDesc implements Serializable, Cloneable {
}
public void setProperties(final Properties properties) {
+ StringInternUtils.internValuesInMap((Map) properties);
this.properties = properties;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
index e5f2c5e..5467d8a 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Properties;
import org.apache.avro.Schema;
+import org.apache.hadoop.hive.common.StringInternUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -106,7 +107,8 @@ public class AvroSerDe extends AbstractSerDe {
schema = determineSchemaOrReturnErrorSchema(configuration, properties);
} else {
// Get column names and sort order
- columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
+ columnNames = StringInternUtils.internStringsInList(
+ Arrays.asList(columnNameProperty.split(columnNameDelimiter)));
columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
schema = getSchemaFromCols(properties, columnNames, columnTypes, columnCommentProperty);
@@ -127,7 +129,7 @@ public class AvroSerDe extends AbstractSerDe {
badSchema = schema.equals(SchemaResolutionProblem.SIGNAL_BAD_SCHEMA);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(schema);
- this.columnNames = aoig.getColumnNames();
+ this.columnNames = StringInternUtils.internStringsInList(aoig.getColumnNames());
this.columnTypes = aoig.getColumnTypes();
this.oi = aoig.getObjectInspector();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
index 2b3fded..cb63d59 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.thrift.TUnion;
@@ -229,8 +230,8 @@ public final class ObjectInspectorFactory {
}
- static ConcurrentHashMap<ObjectInspector, StandardListObjectInspector> cachedStandardListObjectInspector =
- new ConcurrentHashMap<ObjectInspector, StandardListObjectInspector>();
+ static ConcurrentHashMap<ObjectInspector, StandardListObjectInspector>
+ cachedStandardListObjectInspector = new ConcurrentHashMap<ObjectInspector, StandardListObjectInspector>();
public static StandardListObjectInspector getStandardListObjectInspector(
ObjectInspector listElementObjectInspector) {
@@ -316,13 +317,15 @@ public final class ObjectInspectorFactory {
List<ObjectInspector> structFieldObjectInspectors,
List<String> structComments) {
ArrayList<List<?>> signature = new ArrayList<List<?>>(3);
+ StringInternUtils.internStringsInList(structFieldNames);
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
- if(structComments != null) {
+ if (structComments != null) {
+ StringInternUtils.internStringsInList(structComments);
signature.add(structComments);
}
StandardStructObjectInspector result = cachedStandardStructObjectInspector.get(signature);
- if(result == null) {
+ if (result == null) {
result = new StandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, structComments);
StandardStructObjectInspector prev =
cachedStandardStructObjectInspector.putIfAbsent(signature, result);
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1dfe37/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
index d2e8823..36a5007 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
@@ -55,7 +55,7 @@ public class StandardStructObjectInspector extends
public MyField(int fieldID, String fieldName,
ObjectInspector fieldObjectInspector) {
this.fieldID = fieldID;
- this.fieldName = fieldName.toLowerCase();
+ this.fieldName = fieldName.toLowerCase().intern();
this.fieldObjectInspector = fieldObjectInspector;
}