You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ha...@apache.org on 2011/11/04 01:22:36 UTC
svn commit: r1197380 - in /incubator/hcatalog/trunk: ./ hive/
src/java/org/apache/hcatalog/cli/ src/java/org/apache/hcatalog/data/schema/
src/java/org/apache/hcatalog/pig/ src/test/org/apache/hcatalog/cli/
src/test/org/apache/hcatalog/pig/
Author: hashutosh
Date: Fri Nov 4 01:22:36 2011
New Revision: 1197380
URL: http://svn.apache.org/viewvc?rev=1197380&view=rev
Log:
Add support for binary data type
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/build-common.xml
incubator/hcatalog/trunk/build.xml
incubator/hcatalog/trunk/hive/ (props changed)
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java
incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Nov 4 01:22:36 2011
@@ -23,6 +23,8 @@ Trunk (unreleased changes)
INCOMPATIBLE CHANGES
NEW FEATURES
+ HCAT-145. Add support for binary data type (hashutosh)
+
HCAT-151. Fixed native table names used for tables stored in non-default DBs in HBaseInputStorageDriver (avandana via toffer)
HCAT-143. Projection pushdown for HBaseInputStorageDriver (avandana via toffer)
Modified: incubator/hcatalog/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build-common.xml?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build-common.xml (original)
+++ incubator/hcatalog/trunk/build-common.xml Fri Nov 4 01:22:36 2011
@@ -15,7 +15,6 @@
<!-- common classpaths for various builds -->
<path id="common.classpath">
<fileset dir="${hive.root}" includes="testlibs/*.jar"/>
- <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
<fileset dir="${hive.root}/lib" includes="*.jar"/>
<fileset dir="${hive.root}/build/cli" includes="*.jar"/>
<fileset dir="${hive.root}/build/common" includes="*.jar"/>
@@ -29,6 +28,7 @@
<fileset dir="${hive.root}/build/ivy/lib/default" includes="jdo2-api-2.3-ec.jar"/>
<fileset dir="${hive.root}/build/ivy/lib/default" includes="datanucleus-enhancer-2.0.3.jar"/>
<fileset dir="${hive.root}/build/ivy/lib/default" includes="datanucleus-core-2.0.3.jar"/>
+ <fileset dir="${hive.root}/build/ivy/lib/default" includes="libfb303-*.jar"/>
<fileset dir="${common.lib.dir}" includes="hadoop_archive-0.3.1.jar"/>
<fileset dir="${hive.root}/lib" includes="asm-3.1.jar"/>
</path>
Modified: incubator/hcatalog/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Fri Nov 4 01:22:36 2011
@@ -451,6 +451,7 @@
<include name="commons-pool-*.jar"/>
<include name="commons-dbcp-*.jar"/>
<include name="guava-*.jar"/>
+ <include name="libthrift-*.jar"/>
</fileset>
<fileset dir="hive/external/build/shims">
@@ -488,7 +489,6 @@
<include name="commons-lang-*.jar"/>
<include name="asm-api-*.jar"/>
<include name="derby-*.jar"/>
- <include name="thrift-fb303-*.jar"/>
</fileset>
<fileset dir="hive/external/cli/lib">
Propchange: incubator/hcatalog/trunk/hive/
------------------------------------------------------------------------------
--- svn:externals (original)
+++ svn:externals Fri Nov 4 01:22:36 2011
@@ -1 +1 @@
--r 1148878 http://svn.apache.org/repos/asf/hive/trunk external
+http://svn.apache.org/repos/asf/hive/trunk external
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/cli/HCatCli.java Fri Nov 4 01:22:36 2011
@@ -38,10 +38,11 @@ import org.apache.commons.cli.Parser;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.SetProcessor;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
@@ -53,7 +54,11 @@ public class HCatCli {
@SuppressWarnings("static-access")
public static void main(String[] args) {
- SessionState.initHiveLog4j();
+ try {
+ LogUtils.initHiveLog4j();
+ } catch (LogInitializationException e) {
+
+ }
CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
ss.in = System.in;
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatFieldSchema.java Fri Nov 4 01:22:36 2011
@@ -35,6 +35,7 @@ public class HCatFieldSchema implements
ARRAY,
MAP,
STRUCT,
+ BINARY,
}
public enum Category {
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java Fri Nov 4 01:22:36 2011
@@ -151,6 +151,8 @@ public class HCatSchemaUtils {
return Type.SMALLINT;
case STRING:
return Type.STRING;
+ case BINARY:
+ return Type.BINARY;
default:
throw new TypeNotPresentException(((PrimitiveTypeInfo)basePrimitiveTypeInfo).getTypeName(), null);
}
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/HCatBaseStorer.java Fri Nov 4 01:22:36 2011
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.RecordWriter;
@@ -42,6 +43,7 @@ import org.apache.pig.StoreMetadata;
import org.apache.pig.backend.BackendException;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -187,6 +189,9 @@ public abstract class HCatBaseStorer ext
case DataType.DOUBLE:
return new HCatFieldSchema(fSchema.alias, Type.DOUBLE, null);
+ case DataType.BYTEARRAY:
+ return new HCatFieldSchema(fSchema.alias, Type.BINARY, null);
+
case DataType.BAG:
Schema bagSchema = fSchema.schema;
List<HCatFieldSchema> arrFields = new ArrayList<HCatFieldSchema>(1);
@@ -219,6 +224,7 @@ public abstract class HCatBaseStorer ext
case INT:
case FLOAT:
case DOUBLE:
+ case BINARY:
valFS = new HCatFieldSchema(fSchema.alias, mapValType, null);
break;
default:
@@ -269,6 +275,12 @@ public abstract class HCatBaseStorer ext
switch(type){
+ case BINARY:
+ ByteArrayRef ba = new ByteArrayRef();
+ byte[] bytes = (null == pigObj) ? new byte[0] : ((DataByteArray)pigObj).get();
+ ba.setData(bytes);
+ return ba;
+
case STRUCT:
// Unwrap the tuple.
return ((Tuple)pigObj).getAll();
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/pig/PigHCatUtil.java Fri Nov 4 01:22:36 2011
@@ -31,6 +31,8 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.data.HCatArrayBag;
@@ -44,6 +46,7 @@ import org.apache.pig.ResourceSchema;
import org.apache.pig.LoadPushDown.RequiredField;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultDataBag;
import org.apache.pig.data.Tuple;
@@ -275,6 +278,10 @@ public class PigHCatUtil {
return DataType.DOUBLE;
}
+ if (type == Type.BINARY){
+ return DataType.BYTEARRAY;
+ }
+
if (type == Type.BOOLEAN){
errMsg = "HCatalog column type 'BOOLEAN' is not supported in " +
"Pig as a column type";
@@ -293,18 +300,20 @@ public class PigHCatUtil {
}
@SuppressWarnings("unchecked")
-public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws Exception {
+ public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws Exception {
Type itemType = hfs.getType();
- if ( ! hfs.isComplex()){
- return o;
- } else if (itemType == Type.STRUCT) {
- return transformToTuple((List<Object>)o,hfs);
- } else if (itemType == Type.ARRAY) {
- return transformToBag((List<? extends Object>) o,hfs);
- } else if (itemType == Type.MAP) {
- return transformToPigMap((Map<String, Object>)o,hfs);
+ switch (itemType){
+ case BINARY:
+ return new DataByteArray(((ByteArrayRef)o).getData());
+ case STRUCT:
+ return transformToTuple((List<Object>)o,hfs);
+ case ARRAY:
+ return transformToBag((List<? extends Object>) o,hfs);
+ case MAP:
+ return transformToPigMap((Map<String, Object>)o,hfs);
+ default:
+ return o;
}
- return null; // never invoked.
}
public static Tuple transformToTuple(List<? extends Object> objList, HCatFieldSchema hfs) throws Exception {
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/cli/TestSemanticAnalysis.java Fri Nov 4 01:22:36 2011
@@ -218,8 +218,8 @@ public class TestSemanticAnalysis extend
Table tbl = msc.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName);
List<FieldSchema> cols = tbl.getSd().getCols();
assertEquals(3, cols.size());
- assertTrue(cols.get(0).equals(new FieldSchema("a", "int", "from deserializer")));
- assertTrue(cols.get(1).equals(new FieldSchema("c", "string", "from deserializer")));
+ assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null)));
+ assertTrue(cols.get(1).equals(new FieldSchema("c", "string", null)));
assertTrue(cols.get(2).equals(new FieldSchema("d", "tinyint", null)));
hcatDriver.run("drop table junit_sem_analysis");
}
Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java?rev=1197380&r1=1197379&r2=1197380&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatStorer.java Fri Nov 4 01:22:36 2011
@@ -37,6 +37,7 @@ import org.apache.hcatalog.rcfile.RCFile
import org.apache.pig.ExecType;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
+import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.LogUtils;
@@ -498,7 +499,7 @@ public class TestHCatStorer extends Test
public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException{
driver.run("drop table junit_unparted");
- String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string) stored as RCFILE " +
+ String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, f binary) stored as RCFILE " +
"tblproperties('"+HCatConstants.HCAT_ISD_CLASS+"'='"+RCFileInputDriver.class.getName()+"'," +
"'"+HCatConstants.HCAT_OSD_CLASS+"'='"+RCFileOutputDriver.class.getName()+"') ";
int retCode = driver.run(createTable).getResponseCode();
@@ -509,17 +510,17 @@ public class TestHCatStorer extends Test
int LOOP_SIZE = 3;
String[] input = new String[LOOP_SIZE*LOOP_SIZE];
for(int i = 0; i < LOOP_SIZE*LOOP_SIZE; i++) {
- input[i] = i + "\t" + i * 2.1f +"\t"+ i*1.1d + "\t" + i * 2L +"\t"+"lets hcat";
+ input[i] = i + "\t" + i * 2.1f +"\t"+ i*1.1d + "\t" + i * 2L +"\t"+"lets hcat"+"\tbinary-data";
}
MiniCluster.createInputFile(cluster, fileName, input);
PigServer server = new PigServer(ExecType.LOCAL, props);
UDFContext.getUDFContext().setClientSystemProps();
server.setBatchOn();
- server.registerQuery("A = load '"+fullFileName+"' as (a:int, b:float, c:double, d:long, e:chararray);");
- server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','a:int, b:float, c:double, d:long, e:chararray');");
+ server.registerQuery("A = load '"+fullFileName+"' as (a:int, b:float, c:double, d:long, e:chararray, f:bytearray);");
+ server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','a:int, b:float, c:double, d:long, e:chararray,f:bytearray');");
server.executeBatch();
- MiniCluster.deleteFile(cluster, fileName);
+
driver.run("select * from junit_unparted");
ArrayList<String> res = new ArrayList<String>();
@@ -531,8 +532,19 @@ public class TestHCatStorer extends Test
}
assertFalse(itr.hasNext());
+ server.registerQuery("B = load 'junit_unparted' using "+HCatLoader.class.getName()+";");
+ Iterator<Tuple> i = server.openIterator("B");
+ int count = 0;
+ while(i.hasNext()){
+ Object o = i.next().get(5);
+ assertTrue(o instanceof DataByteArray);
+ count++;
+ }
+ assertEquals(LOOP_SIZE * LOOP_SIZE, count);
+ MiniCluster.deleteFile(cluster, fileName);
driver.run("drop table junit_unparted");
}
+
@Override
protected void tearDown() throws Exception {
super.tearDown();