You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2013/09/03 22:31:52 UTC
svn commit: r1519839 - in /hive/trunk/hcatalog/hcatalog-pig-adapter/src:
main/java/org/apache/hcatalog/pig/HCatBaseStorer.java
test/java/org/apache/hcatalog/pig/TestHCatStorer.java
Author: khorgath
Date: Tue Sep 3 20:31:52 2013
New Revision: 1519839
URL: http://svn.apache.org/r1519839
Log:
HIVE-5104 : HCatStorer fails to store boolean type (Karl D. Gierach via Sushanth Sowmyan)
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java?rev=1519839&r1=1519838&r2=1519839&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatBaseStorer.java Tue Sep 3 20:31:52 2013
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import com.google.common.collect.Lists;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
@@ -53,6 +52,10 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
/**
* Base class for HCatStorer and HCatEximStorer
@@ -61,6 +64,8 @@ import org.apache.pig.impl.util.Utils;
abstract class HCatBaseStorer extends StoreFunc implements StoreMetadata {
+ private static final Logger LOG = LoggerFactory.getLogger( HCatBaseStorer.class );
+
private static final List<Type> SUPPORTED_INTEGER_CONVERSIONS =
Lists.newArrayList(Type.TINYINT, Type.SMALLINT, Type.INT);
protected static final String COMPUTED_OUTPUT_SCHEMA = "hcat.output.schema";
@@ -175,6 +180,9 @@ abstract class HCatBaseStorer extends St
case DataType.BYTEARRAY:
return new HCatFieldSchema(fSchema.alias, Type.BINARY, null);
+ case DataType.BOOLEAN:
+ return new HCatFieldSchema(fSchema.alias, Type.BOOLEAN, null);
+
case DataType.BAG:
Schema bagSchema = fSchema.schema;
List<HCatFieldSchema> arrFields = new ArrayList<HCatFieldSchema>(1);
@@ -328,10 +336,30 @@ abstract class HCatBaseStorer extends St
}
return ((Integer) pigObj).byteValue();
case BOOLEAN:
- // would not pass schema validation anyway
- throw new BackendException("Incompatible type " + type + " found in hcat table schema: " + hcatFS, PigHCatUtil.PIG_EXCEPTION_CODE);
+ if (pigObj == null) {
+ LOG.debug( "HCatBaseStorer.getJavaObj(BOOLEAN): obj null, bailing early" );
+ return null;
+ }
+
+ if( pigObj instanceof String ) {
+ if( ((String)pigObj).trim().compareTo("0") == 0 ) {
+ return Boolean.FALSE;
+ }
+ if( ((String)pigObj).trim().compareTo("1") == 0 ) {
+ return Boolean.TRUE;
+ }
+
+ throw new BackendException(
+ "Unexpected type " + type + " for value " + pigObj
+ + (pigObj == null ? "" : " of class "
+ + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE);
+ }
+
+ return Boolean.parseBoolean( pigObj.toString() );
default:
- throw new BackendException("Unexpected type " + type + " for value " + pigObj + (pigObj == null ? "" : " of class " + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE);
+ throw new BackendException("Unexpected type " + type + " for value " + pigObj
+ + (pigObj == null ? "" : " of class "
+ + pigObj.getClass().getName()), PigHCatUtil.PIG_EXCEPTION_CODE);
}
} catch (BackendException e) {
// provide the path to the field in the error message
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java?rev=1519839&r1=1519838&r2=1519839&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java Tue Sep 3 20:31:52 2013
@@ -403,7 +403,7 @@ public class TestHCatStorer extends HCat
public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException {
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, f binary, g binary) stored as RCFILE";
+ String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as RCFILE";
int retCode = driver.run(createTable).getResponseCode();
if (retCode != 0) {
throw new IOException("Failed to create table.");
@@ -411,16 +411,16 @@ public class TestHCatStorer extends HCat
int i = 0;
String[] input = new String[3];
- input[i++] = "0\t\t\t\t\t\t"; //Empty values except first column
- input[i++] = "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\tbinary-data"; //First column empty
- input[i++] = i + "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\tbinary-data";
+ input[i++] = "0\t\t\t\t\t\t\t"; //Empty values except first column
+ input[i++] = "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\t" + "true" + "\tbinary-data"; //First column empty
+ input[i++] = i + "\t" + i * 2.1f + "\t" + i * 1.1d + "\t" + i * 2L + "\t" + "lets hcat" + "\t" + "false" + "\tbinary-data";
HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input);
PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
- server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, f:bytearray);");
+ server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray);");
//null gets stored into column g which is a binary field.
- server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int, b:float, c:double, d:long, e:chararray,f:bytearray');");
+ server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray');");
server.executeBatch();
@@ -429,9 +429,10 @@ public class TestHCatStorer extends HCat
driver.getResults(res);
Iterator<String> itr = res.iterator();
- Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", itr.next());
- Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\tbinary-data\tNULL", itr.next());
- Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tbinary-data\tNULL", itr.next());
+ String next = itr.next();
+ Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", next );
+ Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\ttrue\tbinary-data\tNULL", itr.next());
+ Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tfalse\tbinary-data\tNULL", itr.next());
Assert.assertFalse(itr.hasNext());
server.registerQuery("B = load 'junit_unparted' using " + HCatLoader.class.getName() + ";");
@@ -440,12 +441,12 @@ public class TestHCatStorer extends HCat
int num5nulls = 0;
while (iter.hasNext()) {
Tuple t = iter.next();
- if (t.get(5) == null) {
+ if (t.get(6) == null) {
num5nulls++;
} else {
- Assert.assertTrue(t.get(5) instanceof DataByteArray);
+ Assert.assertTrue(t.get(6) instanceof DataByteArray);
}
- Assert.assertNull(t.get(6));
+ Assert.assertNull(t.get(7));
count++;
}
Assert.assertEquals(3, count);