You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by tr...@apache.org on 2012/10/05 17:05:38 UTC
svn commit: r1394646 - in /incubator/hcatalog/trunk: CHANGES.txt
hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
Author: travis
Date: Fri Oct 5 17:05:38 2012
New Revision: 1394646
URL: http://svn.apache.org/viewvc?rev=1394646&view=rev
Log:
HCATALOG-364 HCatalog pig adapter should handle boolean fields
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1394646&r1=1394645&r2=1394646&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Oct 5 17:05:38 2012
@@ -23,6 +23,8 @@ Trunk (unreleased changes)
INCOMPATIBLE CHANGES
NEW FEATURES
+ HCAT-364 HCatalog pig adapter should handle boolean fields (traviscrawford)
+
HCAT-515 Upgrade hadoop23 version to use 0.23.3 (cdrome via toffer)
HCAT-132 Add HCatalog to Maven Repository (traviscrawford)
Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java?rev=1394646&r1=1394645&r2=1394646&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java Fri Oct 5 17:05:38 2012
@@ -52,10 +52,16 @@ import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PigHCatUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(PigHCatUtil.class);
+
static final int PIG_EXCEPTION_CODE = 1115; // http://wiki.apache.org/pig/PigErrorHandlingFunctionalSpecification#Error_codes
private static final String DEFAULT_DB = MetaStoreUtils.DEFAULT_DATABASE_NAME;
@@ -64,6 +70,39 @@ public class PigHCatUtil {
private static final TupleFactory tupFac = TupleFactory.getInstance();
+ private static boolean pigHasBooleanSupport = false;
+
+ /**
+ * Determine if the current Pig version supports boolean columns. This works around a
+ * dependency conflict preventing HCatalog from requiring a version of Pig with boolean
+ * field support and should be removed once HCATALOG-466 has been resolved.
+ */
+ static {
+ // DETAILS:
+ //
+ // PIG-1429 added support for boolean fields, which shipped in 0.10.0;
+ // this version of Pig depends on antlr 3.4.
+ //
+ // HCatalog depends heavily on Hive, which at this time uses antlr 3.0.1.
+ //
+ // antlr 3.0.1 and 3.4 are incompatible, so Pig 0.10.0 and Hive cannot be depended on in the
+ // same project. Pig 0.8.0 did not use antlr for its parser and can coexist with Hive,
+ // so that Pig version is depended on by HCatalog at this time.
+ try {
+ Schema schema = Utils.getSchemaFromString("myBooleanField: boolean");
+ pigHasBooleanSupport = (schema.getField("myBooleanField").type == DataType.BOOLEAN);
+ } catch (Throwable e) {
+ // pass
+ }
+
+ if (!pigHasBooleanSupport) {
+ LOG.info("This version of Pig does not support boolean fields. To enable "
+ + "boolean-to-integer conversion, set the "
+ + HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER
+ + "=true configuration parameter.");
+ }
+ }
+
static public Pair<String, String> getDBTableNames(String location) throws IOException {
// the location string will be of the form:
// <database name>.<table name> - parse it and
@@ -258,8 +297,6 @@ public class PigHCatUtil {
}
static public byte getPigType(Type type) throws IOException {
- String errMsg;
-
if (type == Type.STRING) {
return DataType.CHARARRAY;
}
@@ -296,14 +333,12 @@ public class PigHCatUtil {
return DataType.BYTEARRAY;
}
- if (type == Type.BOOLEAN) {
- errMsg = "HCatalog column type 'BOOLEAN' is not supported in " +
- "Pig as a column type";
- throw new PigException(errMsg, PIG_EXCEPTION_CODE);
+ if (type == Type.BOOLEAN && pigHasBooleanSupport) {
+ return DataType.BOOLEAN;
}
- errMsg = "HCatalog column type '" + type.toString() + "' is not supported in Pig as a column type";
- throw new PigException(errMsg, PIG_EXCEPTION_CODE);
+ throw new PigException("HCatalog column type '" + type.toString()
+ + "' is not supported in Pig as a column type", PIG_EXCEPTION_CODE);
}
public static Tuple transformToTuple(HCatRecord hr, HCatSchema hs) throws Exception {
@@ -406,7 +441,11 @@ public class PigHCatUtil {
Type hType = hcatField.getType();
switch (hType) {
case BOOLEAN:
- throw new PigException("Incompatible type found in hcat table schema: " + hcatField, PigHCatUtil.PIG_EXCEPTION_CODE);
+ if (!pigHasBooleanSupport) {
+ throw new PigException("Incompatible type found in HCat table schema: "
+ + hcatField, PigHCatUtil.PIG_EXCEPTION_CODE);
+ }
+ break;
case ARRAY:
validateHCatSchemaFollowsPigRules(hcatField.getArrayElementSchema());
break;