You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by tr...@apache.org on 2012/09/27 20:55:57 UTC
svn commit: r1391213 - in /incubator/hcatalog/trunk: ./
hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/
src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/data/
src/java/org/apache/hcatalog/data/schema/
Author: travis
Date: Thu Sep 27 20:55:56 2012
New Revision: 1391213
URL: http://svn.apache.org/viewvc?rev=1391213&view=rev
Log:
HCATALOG-497 HCatContext should use the jobconf instead of its own conf
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Sep 27 20:55:56 2012
@@ -115,6 +115,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-497 HCatContext should use the jobconf instead of its own conf (traviscrawford)
+
HCAT-494 MultiOutputFormat in 0.23 fails to setAliasConf() correctly. (mithun via toffer)
HCAT-507 e2e harness failing on 0.23 (toffer)
Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatLoader.java Thu Sep 27 20:55:56 2012
@@ -82,6 +82,8 @@ public class HCatLoader extends HCatBase
@Override
public void setLocation(String location, Job job) throws IOException {
+ HCatContext.setupHCatContext(job.getConfiguration()).getConf().get()
+ .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);
UDFContext udfContext = UDFContext.getUDFContext();
Properties udfProps = udfContext.getUDFProperties(this.getClass(),
@@ -185,9 +187,8 @@ public class HCatLoader extends HCatBase
@Override
public ResourceSchema getSchema(String location, Job job) throws IOException {
- HCatContext.getInstance().mergeConf(job.getConfiguration());
- HCatContext.getInstance().getConf().setBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);
+ HCatContext.setupHCatContext(job.getConfiguration()).getConf().get()
+ .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);
Table table = phutil.getTable(location,
hcatServerUri != null ? hcatServerUri : PigHCatUtil.getHCatServerUri(job),
Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java Thu Sep 27 20:55:56 2012
@@ -77,9 +77,8 @@ public class HCatStorer extends HCatBase
@Override
public void setStoreLocation(String location, Job job) throws IOException {
- HCatContext.getInstance().mergeConf(job.getConfiguration());
- HCatContext.getInstance().getConf().setBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, false);
+ HCatContext.setupHCatContext(job.getConfiguration()).getConf().get()
+ .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, false);
Configuration config = job.getConfiguration();
config.set(INNER_SIGNATURE, INNER_SIGNATURE_PREFIX + "_" + sign);
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatContext.java Thu Sep 27 20:55:56 2012
@@ -18,37 +18,54 @@
package org.apache.hcatalog.common;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import java.util.Map;
/**
- * HCatContext provides global access to configuration data.
+ * HCatContext provides global access to configuration data. It uses a reference to the
+ * job configuration so that settings are automatically passed to the backend by the
+ * MR framework.
*/
public class HCatContext {
private static final HCatContext hCatContext = new HCatContext();
- private final Configuration conf;
+ private Configuration conf = null;
private HCatContext() {
- conf = new Configuration();
}
- public static HCatContext getInstance() {
+ /**
+ * Setup the HCatContext as a reference to the given configuration. Keys
+ * exclusive to an existing config are set in the new conf.
+ */
+ public static synchronized HCatContext setupHCatContext(Configuration newConf) {
+ Preconditions.checkNotNull(newConf, "HCatContext must not have a null conf.");
+
+ if (hCatContext.conf == null) {
+ hCatContext.conf = newConf;
+ return hCatContext;
+ }
+
+ if (hCatContext.conf != newConf) {
+ for (Map.Entry<String, String> entry : hCatContext.conf) {
+ if (newConf.get(entry.getKey()) == null) {
+ newConf.set(entry.getKey(), entry.getValue());
+ }
+ }
+ hCatContext.conf = newConf;
+ }
return hCatContext;
}
- public Configuration getConf() {
- return conf;
+ public static HCatContext getInstance() {
+ return hCatContext;
}
- /**
- * Merge the given configuration into the HCatContext conf, overwriting any existing keys.
- */
- public void mergeConf(Configuration conf) {
- for (Map.Entry<String, String> entry : conf) {
- this.conf.set(entry.getKey(), entry.getValue());
- }
+ public Optional<Configuration> getConf() {
+ return Optional.fromNullable(conf);
}
}
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java Thu Sep 27 20:55:56 2012
@@ -188,24 +188,7 @@ public class HCatRecordSerDe implements
Object res;
if (fieldObjectInspector.getCategory() == Category.PRIMITIVE) {
- if (field != null && field instanceof Boolean &&
- HCatContext.getInstance().getConf().getBoolean(
- HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
- HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) {
- res = ((Boolean) field) ? 1 : 0;
- } else if (field != null && field instanceof Short &&
- HCatContext.getInstance().getConf().getBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
- res = new Integer((Short) field);
- } else if (field != null && field instanceof Byte &&
- HCatContext.getInstance().getConf().getBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
- res = new Integer((Byte) field);
- } else {
- res = ((PrimitiveObjectInspector) fieldObjectInspector).getPrimitiveJavaObject(field);
- }
+ res = serializePrimitiveField(field, fieldObjectInspector);
} else if (fieldObjectInspector.getCategory() == Category.STRUCT) {
res = serializeStruct(field, (StructObjectInspector) fieldObjectInspector);
} else if (fieldObjectInspector.getCategory() == Category.LIST) {
@@ -279,6 +262,32 @@ public class HCatRecordSerDe implements
}
}
+ private static Object serializePrimitiveField(Object field,
+ ObjectInspector fieldObjectInspector) {
+
+ if (field != null && HCatContext.getInstance().getConf().isPresent()) {
+ Configuration conf = HCatContext.getInstance().getConf().get();
+
+ if (field instanceof Boolean &&
+ conf.getBoolean(
+ HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
+ HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) {
+ return ((Boolean) field) ? 1 : 0;
+ } else if (field instanceof Short &&
+ conf.getBoolean(
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
+ return new Integer((Short) field);
+ } else if (field instanceof Byte &&
+ conf.getBoolean(
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
+ return new Integer((Byte) field);
+ }
+ }
+
+ return ((PrimitiveObjectInspector) fieldObjectInspector).getPrimitiveJavaObject(field);
+ }
/**
* Return an object inspector that can read through the object
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java?rev=1391213&r1=1391212&r2=1391213&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java Thu Sep 27 20:55:56 2012
@@ -139,14 +139,17 @@ public class HCatSchemaUtils {
private static Type getPrimitiveHType(TypeInfo basePrimitiveTypeInfo) {
switch (((PrimitiveTypeInfo) basePrimitiveTypeInfo).getPrimitiveCategory()) {
case BOOLEAN:
- return HCatContext.getInstance().getConf().getBoolean(
- HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
- HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT) ?
+ return (HCatContext.getInstance().getConf().isPresent() &&
+ HCatContext.getInstance().getConf().get().getBoolean(
+ HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
+ HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) ?
Type.INT : Type.BOOLEAN;
case BYTE:
- return HCatContext.getInstance().getConf().getBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT) ? Type.INT : Type.TINYINT;
+ return (HCatContext.getInstance().getConf().isPresent() &&
+ HCatContext.getInstance().getConf().get().getBoolean(
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) ?
+ Type.INT : Type.TINYINT;
case DOUBLE:
return Type.DOUBLE;
case FLOAT:
@@ -156,9 +159,10 @@ public class HCatSchemaUtils {
case LONG:
return Type.BIGINT;
case SHORT:
- return HCatContext.getInstance().getConf().getBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT) ?
+ return (HCatContext.getInstance().getConf().isPresent() &&
+ HCatContext.getInstance().getConf().get().getBoolean(
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
+ HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) ?
Type.INT : Type.SMALLINT;
case STRING:
return Type.STRING;