You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/09/06 18:37:17 UTC
svn commit: r1622885 - in /hive/trunk:
common/src/java/org/apache/hive/common/util/ metastore/
metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/main/ metastore/src/main/resources/
Author: brock
Date: Sat Sep 6 16:37:17 2014
New Revision: 1622885
URL: http://svn.apache.org/r1622885
Log:
HIVE-7975 - HS2 memory optimization: Internalizing instance fields of Thrift-generated metastore API classes (Wilbur Yang via Brock)
Added:
hive/trunk/metastore/src/main/
hive/trunk/metastore/src/main/resources/
hive/trunk/metastore/src/main/resources/thrift-replacements.txt
Modified:
hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
hive/trunk/metastore/pom.xml
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java
Modified: hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java?rev=1622885&r1=1622884&r2=1622885&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java Sat Sep 6 16:37:17 2014
@@ -33,9 +33,13 @@ import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
import java.util.Locale;
import java.util.StringTokenizer;
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
@@ -57,10 +61,62 @@ public class HiveStringUtils {
public static final int SHUTDOWN_HOOK_PRIORITY = 0;
private static final DecimalFormat decimalFormat;
+
+ /**
+ * Maintain a String pool to reduce memory.
+ */
+ private static final Interner<String> STRING_INTERNER;
+
static {
- NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH);
- decimalFormat = (DecimalFormat) numberFormat;
- decimalFormat.applyPattern("#.##");
+ NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ decimalFormat = (DecimalFormat) numberFormat;
+ decimalFormat.applyPattern("#.##");
+
+ STRING_INTERNER = Interners.newWeakInterner();
+ }
+
+ /**
+ * Return the internalized string, or null if the given string is null.
+ * @param str The string to intern
+ * @return The identical string cached in the string pool.
+ */
+ public static String intern(String str) {
+ if(str == null) {
+ return null;
+ }
+ return STRING_INTERNER.intern(str);
+ }
+
+ /**
+ * Return an interned list with identical contents as the given list.
+ * @param list The list whose strings will be interned
+ * @return An identical list with its strings interned.
+ */
+ public static List<String> intern(List<String> list) {
+ if(list == null) {
+ return null;
+ }
+ List<String> newList = new ArrayList<String>(list.size());
+ for(String str : list) {
+ newList.add(intern(str));
+ }
+ return newList;
+ }
+
+ /**
+ * Return an interned map with identical contents as the given map.
+ * @param map The map whose strings will be interned
+ * @return An identical map with its strings interned.
+ */
+ public static Map<String, String> intern(Map<String, String> map) {
+ if(map == null) {
+ return null;
+ }
+ Map<String, String> newMap = new HashMap<String, String>(map.size());
+ for(Map.Entry<String, String> entry : map.entrySet()) {
+ newMap.put(intern(entry.getKey()), intern(entry.getValue()));
+ }
+ return newMap;
}
/**
Modified: hive/trunk/metastore/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/pom.xml?rev=1622885&r1=1622884&r2=1622885&view=diff
==============================================================================
--- hive/trunk/metastore/pom.xml (original)
+++ hive/trunk/metastore/pom.xml Sat Sep 6 16:37:17 2014
@@ -165,6 +165,39 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>thriftif</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.google.code.maven-replacer-plugin</groupId>
+ <artifactId>replacer</artifactId>
+ <version>1.5.3</version>
+ <executions>
+ <execution>
+ <id>process-thrift-sources</id>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>replace</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <basedir>${basedir}/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/</basedir>
+ <includes>
+ <include>FieldSchema.java</include>
+ <include>Partition.java</include>
+ <include>SerDeInfo.java</include>
+ <include>StorageDescriptor.java</include>
+ </includes>
+ <tokenValueMap>${basedir}/src/main/resources/thrift-replacements.txt</tokenValueMap>
+ <regex>true</regex>
+ <quiet>false</quiet>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<build>
Modified: hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java?rev=1622885&r1=1622884&r2=1622885&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java Sat Sep 6 16:37:17 2014
@@ -135,9 +135,9 @@ public class FieldSchema implements org.
String comment)
{
this();
- this.name = name;
- this.type = type;
- this.comment = comment;
+ this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
+ this.type = org.apache.hive.common.util.HiveStringUtils.intern(type);
+ this.comment = org.apache.hive.common.util.HiveStringUtils.intern(comment);
}
/**
@@ -145,13 +145,13 @@ public class FieldSchema implements org.
*/
public FieldSchema(FieldSchema other) {
if (other.isSetName()) {
- this.name = other.name;
+ this.name = org.apache.hive.common.util.HiveStringUtils.intern(other.name);
}
if (other.isSetType()) {
- this.type = other.type;
+ this.type = org.apache.hive.common.util.HiveStringUtils.intern(other.type);
}
if (other.isSetComment()) {
- this.comment = other.comment;
+ this.comment = org.apache.hive.common.util.HiveStringUtils.intern(other.comment);
}
}
@@ -171,7 +171,7 @@ public class FieldSchema implements org.
}
public void setName(String name) {
- this.name = name;
+ this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
}
public void unsetName() {
@@ -194,7 +194,7 @@ public class FieldSchema implements org.
}
public void setType(String type) {
- this.type = type;
+ this.type = org.apache.hive.common.util.HiveStringUtils.intern(type);
}
public void unsetType() {
@@ -217,7 +217,7 @@ public class FieldSchema implements org.
}
public void setComment(String comment) {
- this.comment = comment;
+ this.comment = org.apache.hive.common.util.HiveStringUtils.intern(comment);
}
public void unsetComment() {
Modified: hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java?rev=1622885&r1=1622884&r2=1622885&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java Sat Sep 6 16:37:17 2014
@@ -182,14 +182,14 @@ public class Partition implements org.ap
{
this();
this.values = values;
- this.dbName = dbName;
- this.tableName = tableName;
+ this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(dbName);
+ this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(tableName);
this.createTime = createTime;
setCreateTimeIsSet(true);
this.lastAccessTime = lastAccessTime;
setLastAccessTimeIsSet(true);
this.sd = sd;
- this.parameters = parameters;
+ this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
}
/**
@@ -205,10 +205,10 @@ public class Partition implements org.ap
this.values = __this__values;
}
if (other.isSetDbName()) {
- this.dbName = other.dbName;
+ this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(other.dbName);
}
if (other.isSetTableName()) {
- this.tableName = other.tableName;
+ this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(other.tableName);
}
this.createTime = other.createTime;
this.lastAccessTime = other.lastAccessTime;
@@ -222,9 +222,9 @@ public class Partition implements org.ap
String other_element_key = other_element.getKey();
String other_element_value = other_element.getValue();
- String __this__parameters_copy_key = other_element_key;
+ String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key);
- String __this__parameters_copy_value = other_element_value;
+ String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value);
__this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value);
}
@@ -296,7 +296,7 @@ public class Partition implements org.ap
}
public void setDbName(String dbName) {
- this.dbName = dbName;
+ this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(dbName);
}
public void unsetDbName() {
@@ -319,7 +319,7 @@ public class Partition implements org.ap
}
public void setTableName(String tableName) {
- this.tableName = tableName;
+ this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(tableName);
}
public void unsetTableName() {
@@ -420,7 +420,7 @@ public class Partition implements org.ap
}
public void setParameters(Map<String,String> parameters) {
- this.parameters = parameters;
+ this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
}
public void unsetParameters() {
Modified: hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java?rev=1622885&r1=1622884&r2=1622885&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java Sat Sep 6 16:37:17 2014
@@ -137,9 +137,9 @@ public class SerDeInfo implements org.ap
Map<String,String> parameters)
{
this();
- this.name = name;
- this.serializationLib = serializationLib;
- this.parameters = parameters;
+ this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
+ this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(serializationLib);
+ this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
}
/**
@@ -147,10 +147,10 @@ public class SerDeInfo implements org.ap
*/
public SerDeInfo(SerDeInfo other) {
if (other.isSetName()) {
- this.name = other.name;
+ this.name = org.apache.hive.common.util.HiveStringUtils.intern(other.name);
}
if (other.isSetSerializationLib()) {
- this.serializationLib = other.serializationLib;
+ this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(other.serializationLib);
}
if (other.isSetParameters()) {
Map<String,String> __this__parameters = new HashMap<String,String>();
@@ -159,9 +159,9 @@ public class SerDeInfo implements org.ap
String other_element_key = other_element.getKey();
String other_element_value = other_element.getValue();
- String __this__parameters_copy_key = other_element_key;
+ String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key);
- String __this__parameters_copy_value = other_element_value;
+ String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value);
__this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value);
}
@@ -185,7 +185,7 @@ public class SerDeInfo implements org.ap
}
public void setName(String name) {
- this.name = name;
+ this.name = org.apache.hive.common.util.HiveStringUtils.intern(name);
}
public void unsetName() {
@@ -208,7 +208,7 @@ public class SerDeInfo implements org.ap
}
public void setSerializationLib(String serializationLib) {
- this.serializationLib = serializationLib;
+ this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(serializationLib);
}
public void unsetSerializationLib() {
@@ -242,7 +242,7 @@ public class SerDeInfo implements org.ap
}
public void setParameters(Map<String,String> parameters) {
- this.parameters = parameters;
+ this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
}
public void unsetParameters() {
Modified: hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java?rev=1622885&r1=1622884&r2=1622885&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java Sat Sep 6 16:37:17 2014
@@ -216,17 +216,17 @@ public class StorageDescriptor implement
{
this();
this.cols = cols;
- this.location = location;
- this.inputFormat = inputFormat;
- this.outputFormat = outputFormat;
+ this.location = org.apache.hive.common.util.HiveStringUtils.intern(location);
+ this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(inputFormat);
+ this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(outputFormat);
this.compressed = compressed;
setCompressedIsSet(true);
this.numBuckets = numBuckets;
setNumBucketsIsSet(true);
this.serdeInfo = serdeInfo;
- this.bucketCols = bucketCols;
+ this.bucketCols = org.apache.hive.common.util.HiveStringUtils.intern(bucketCols);
this.sortCols = sortCols;
- this.parameters = parameters;
+ this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
}
/**
@@ -242,13 +242,13 @@ public class StorageDescriptor implement
this.cols = __this__cols;
}
if (other.isSetLocation()) {
- this.location = other.location;
+ this.location = org.apache.hive.common.util.HiveStringUtils.intern(other.location);
}
if (other.isSetInputFormat()) {
- this.inputFormat = other.inputFormat;
+ this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(other.inputFormat);
}
if (other.isSetOutputFormat()) {
- this.outputFormat = other.outputFormat;
+ this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(other.outputFormat);
}
this.compressed = other.compressed;
this.numBuckets = other.numBuckets;
@@ -276,9 +276,9 @@ public class StorageDescriptor implement
String other_element_key = other_element.getKey();
String other_element_value = other_element.getValue();
- String __this__parameters_copy_key = other_element_key;
+ String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key);
- String __this__parameters_copy_value = other_element_value;
+ String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value);
__this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value);
}
@@ -356,7 +356,7 @@ public class StorageDescriptor implement
}
public void setLocation(String location) {
- this.location = location;
+ this.location = org.apache.hive.common.util.HiveStringUtils.intern(location);
}
public void unsetLocation() {
@@ -379,7 +379,7 @@ public class StorageDescriptor implement
}
public void setInputFormat(String inputFormat) {
- this.inputFormat = inputFormat;
+ this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(inputFormat);
}
public void unsetInputFormat() {
@@ -402,7 +402,7 @@ public class StorageDescriptor implement
}
public void setOutputFormat(String outputFormat) {
- this.outputFormat = outputFormat;
+ this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(outputFormat);
}
public void unsetOutputFormat() {
@@ -507,7 +507,7 @@ public class StorageDescriptor implement
}
public void setBucketCols(List<String> bucketCols) {
- this.bucketCols = bucketCols;
+ this.bucketCols = org.apache.hive.common.util.HiveStringUtils.intern(bucketCols);
}
public void unsetBucketCols() {
@@ -579,7 +579,7 @@ public class StorageDescriptor implement
}
public void setParameters(Map<String,String> parameters) {
- this.parameters = parameters;
+ this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters);
}
public void unsetParameters() {
Added: hive/trunk/metastore/src/main/resources/thrift-replacements.txt
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/main/resources/thrift-replacements.txt?rev=1622885&view=auto
==============================================================================
--- hive/trunk/metastore/src/main/resources/thrift-replacements.txt (added)
+++ hive/trunk/metastore/src/main/resources/thrift-replacements.txt Sat Sep 6 16:37:17 2014
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###################################################################################################
+# #
+# Used for the internalizing of String instance field assignments in the Thrift generated files #
+# FieldSchema.java, Partition.java, SerDeInfo.java, and StorageDescriptor.java. #
+# #
+# Look in hive/metastore/pom.xml for the thriftif profile. #
+# Usage: thriftif profile automatically refers to this file. #
+# #
+###################################################################################################
+
+# Fix constructors and setters of String instance fields
+
+this\.name\ \=\ name;=this.name\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(name);
+this\.serializationLib\ \=\ serializationLib;=this.serializationLib\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(serializationLib);
+this\.type\ \=\ type;=this.type\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(type);
+this\.comment\ \=\ comment;=this.comment\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(comment);
+this\.location\ \=\ location;=this.location\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(location);
+this\.inputFormat\ \=\ inputFormat;=this.inputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(inputFormat);
+this\.outputFormat\ \=\ outputFormat;=this.outputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(outputFormat);
+this\.dbName\ \=\ dbName;=this.dbName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(dbName);
+this\.tableName\ \=\ tableName;=this.tableName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(tableName);
+
+# Fix constructors and setters of List<String> instance fields
+
+this\.bucketCols\ \=\ bucketCols;=this.bucketCols\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(bucketCols);
+
+# Fix constructors and setters of Map<String, String> instance fields
+
+this\.parameters\ \=\ parameters;=this.parameters\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(parameters);
+
+# Fix copy constructors
+
+this\.name\ \=\ other\.name;=this.name\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.name);
+this\.serializationLib\ \=\ other\.serializationLib;=this.serializationLib\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.serializationLib);
+this\.type\ \=\ other\.type;=this.type\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.type);
+this\.comment\ \=\ other\.comment;=this.comment\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.comment);
+this\.location\ \=\ other\.location;=this.location\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.location);
+this\.inputFormat\ \=\ other\.inputFormat;=this.inputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.inputFormat);
+this\.outputFormat\ \=\ other\.outputFormat;=this.outputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.outputFormat);
+this\.dbName\ \=\ other\.dbName;=this.dbName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.dbName);
+this\.tableName\ \=\ other\.tableName;=this.tableName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.tableName);
+
+__this__parameters_copy_key\ \=\ other_element_key;=__this__parameters_copy_key\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other_element_key);
+__this__parameters_copy_value\ \=\ other_element_value;=__this__parameters_copy_value\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other_element_value);
+__this_values\.add(other_element);=__this_values.add(org.apache.hive.common.util.HiveStringUtils.intern(other_element));