You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/02/14 11:53:39 UTC
svn commit: r1568236 [2/25] - in /hive/branches/tez: ./ bin/
cli/src/java/org/apache/hadoop/hive/cli/
cli/src/test/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/c...
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecord.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecord.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecord.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecord.java Fri Feb 14 10:53:31 2014
@@ -19,11 +19,16 @@
package org.apache.hive.hcatalog.data;
+import java.sql.Date;
+import java.sql.Timestamp;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hive.hcatalog.common.HCatException;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
@@ -149,5 +154,43 @@ public abstract class HCatRecord impleme
public void setMap(String fieldName, HCatSchema recordSchema, Map<?, ?> value) throws HCatException {
set(fieldName, recordSchema, value);
}
+ public void setChar(String fieldName, HCatSchema recordSchema, HiveChar value)
+ throws HCatException {
+ set(fieldName, recordSchema, value);
+ }
+ public HiveChar getChar(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (HiveChar) get(fieldName, recordSchema, HiveChar.class);
+ }
+ public void setVarchar(String fieldName, HCatSchema recordSchema, HiveVarchar value)
+ throws HCatException {
+ set(fieldName, recordSchema, value);
+ }
+ public HiveVarchar getVarchar(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (HiveVarchar) get(fieldName, recordSchema, HiveVarchar.class);
+ }
+ public void setDecimal(String fieldName, HCatSchema recordSchema, HiveDecimal value)
+ throws HCatException {
+ set(fieldName, recordSchema, value);
+ }
+ public HiveDecimal getDecimal(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (HiveDecimal) get(fieldName, recordSchema, HiveDecimal.class);
+ }
+ /**
+ * Note that the proper way to construct a java.sql.Date for use with this object is
+ * Date.valueOf("1999-12-31").
+ */
+ public void setDate(String fieldName, HCatSchema recordSchema, Date value) throws HCatException {
+ set(fieldName, recordSchema, value);
+ }
+ public Date getDate(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (Date) get(fieldName, recordSchema, Date.class);
+ }
+ public void setTimestamp(String fieldName, HCatSchema recordSchema, Timestamp value)
+ throws HCatException {
+ set(fieldName, recordSchema, value);
+ }
+ public Timestamp getTimestamp(String fieldName, HCatSchema recordSchema) throws HCatException {
+ return (Timestamp) get(fieldName, recordSchema, Timestamp.class);
+ }
}
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java Fri Feb 14 10:53:31 2014
@@ -20,6 +20,8 @@ package org.apache.hive.hcatalog.data;
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.sql.Date;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -31,6 +33,9 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDe;
@@ -46,13 +51,19 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -75,9 +86,6 @@ public class JsonSerDe implements SerDe
private static final Logger LOG = LoggerFactory.getLogger(JsonSerDe.class);
private List<String> columnNames;
- private List<TypeInfo> columnTypes;
-
- private StructTypeInfo rowTypeInfo;
private HCatSchema schema;
private JsonFactory jsonFactory = null;
@@ -87,6 +95,8 @@ public class JsonSerDe implements SerDe
@Override
public void initialize(Configuration conf, Properties tbl)
throws SerDeException {
+ List<TypeInfo> columnTypes;
+ StructTypeInfo rowTypeInfo;
LOG.debug("Initializing JsonSerDe");
@@ -195,7 +205,7 @@ public class JsonSerDe implements SerDe
// column mapping, and we're about to proceed.
}
HCatFieldSchema hcatFieldSchema = s.getFields().get(fpos);
- Object currField = extractCurrentField(p, null, hcatFieldSchema, false);
+ Object currField = extractCurrentField(p, hcatFieldSchema, false);
r.set(fpos, currField);
}
@@ -241,17 +251,12 @@ public class JsonSerDe implements SerDe
/**
* Utility method to extract current expected field from given JsonParser
*
- * To get the field, we need either a type or a hcatFieldSchema(necessary for complex types)
- * It is possible that one of them can be null, and so, if so, the other is instantiated
- * from the other
- *
* isTokenCurrent is a boolean variable also passed in, which determines
* if the JsonParser is already at the token we expect to read next, or
* needs advancing to the next before we read.
*/
- private Object extractCurrentField(JsonParser p, Type t,
- HCatFieldSchema hcatFieldSchema, boolean isTokenCurrent) throws IOException, JsonParseException,
- HCatException {
+ private Object extractCurrentField(JsonParser p, HCatFieldSchema hcatFieldSchema,
+ boolean isTokenCurrent) throws IOException {
Object val = null;
JsonToken valueToken;
if (isTokenCurrent) {
@@ -259,11 +264,7 @@ public class JsonSerDe implements SerDe
} else {
valueToken = p.nextToken();
}
-
- if (hcatFieldSchema != null) {
- t = hcatFieldSchema.getType();
- }
- switch (t) {
+ switch (hcatFieldSchema.getType()) {
case INT:
val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getIntValue();
break;
@@ -295,6 +296,23 @@ public class JsonSerDe implements SerDe
break;
case BINARY:
throw new IOException("JsonSerDe does not support BINARY type");
+ case DATE:
+ val = (valueToken == JsonToken.VALUE_NULL) ? null : Date.valueOf(p.getText());
+ break;
+ case TIMESTAMP:
+ val = (valueToken == JsonToken.VALUE_NULL) ? null : Timestamp.valueOf(p.getText());
+ break;
+ case DECIMAL:
+ val = (valueToken == JsonToken.VALUE_NULL) ? null : HiveDecimal.create(p.getText());
+ break;
+ case VARCHAR:
+ int vLen = ((BaseCharTypeInfo)hcatFieldSchema.getTypeInfo()).getLength();
+ val = (valueToken == JsonToken.VALUE_NULL) ? null : new HiveVarchar(p.getText(), vLen);
+ break;
+ case CHAR:
+ int cLen = ((BaseCharTypeInfo)hcatFieldSchema.getTypeInfo()).getLength();
+ val = (valueToken == JsonToken.VALUE_NULL) ? null : new HiveChar(p.getText(), cLen);
+ break;
case ARRAY:
if (valueToken == JsonToken.VALUE_NULL) {
val = null;
@@ -305,7 +323,7 @@ public class JsonSerDe implements SerDe
}
List<Object> arr = new ArrayList<Object>();
while ((valueToken = p.nextToken()) != JsonToken.END_ARRAY) {
- arr.add(extractCurrentField(p, null, hcatFieldSchema.getArrayElementSchema().get(0), true));
+ arr.add(extractCurrentField(p, hcatFieldSchema.getArrayElementSchema().get(0), true));
}
val = arr;
break;
@@ -318,15 +336,14 @@ public class JsonSerDe implements SerDe
throw new IOException("Start of Object expected");
}
Map<Object, Object> map = new LinkedHashMap<Object, Object>();
- Type keyType = hcatFieldSchema.getMapKeyType();
HCatFieldSchema valueSchema = hcatFieldSchema.getMapValueSchema().get(0);
while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
- Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(), keyType);
+ Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(), hcatFieldSchema.getMapKeyTypeInfo());
Object v;
if (valueSchema.getType() == HCatFieldSchema.Type.STRUCT) {
- v = extractCurrentField(p, null, valueSchema, false);
+ v = extractCurrentField(p, valueSchema, false);
} else {
- v = extractCurrentField(p, null, valueSchema, true);
+ v = extractCurrentField(p, valueSchema, true);
}
map.put(k, v);
@@ -350,12 +367,16 @@ public class JsonSerDe implements SerDe
}
val = struct;
break;
+ default:
+ LOG.error("Unknown type found: " + hcatFieldSchema.getType());
+ return null;
}
return val;
}
- private Object getObjectOfCorrespondingPrimitiveType(String s, Type t) throws IOException {
- switch (t) {
+ private Object getObjectOfCorrespondingPrimitiveType(String s, PrimitiveTypeInfo mapKeyType)
+ throws IOException {
+ switch (Type.getPrimitiveHType(mapKeyType)) {
case INT:
return Integer.valueOf(s);
case TINYINT:
@@ -374,8 +395,18 @@ public class JsonSerDe implements SerDe
return s;
case BINARY:
throw new IOException("JsonSerDe does not support BINARY type");
+ case DATE:
+ return Date.valueOf(s);
+ case TIMESTAMP:
+ return Timestamp.valueOf(s);
+ case DECIMAL:
+ return HiveDecimal.create(s);
+ case VARCHAR:
+ return new HiveVarchar(s, ((BaseCharTypeInfo)mapKeyType).getLength());
+ case CHAR:
+ return new HiveChar(s, ((BaseCharTypeInfo)mapKeyType).getLength());
}
- throw new IOException("Could not convert from string to map type " + t);
+ throw new IOException("Could not convert from string to map type " + mapKeyType.getTypeName());
}
/**
@@ -399,9 +430,7 @@ public class JsonSerDe implements SerDe
if (i > 0) {
sb.append(SerDeUtils.COMMA);
}
- sb.append(SerDeUtils.QUOTE);
- sb.append(columnNames.get(i));
- sb.append(SerDeUtils.QUOTE);
+ appendWithQuotes(sb, columnNames.get(i));
sb.append(SerDeUtils.COLON);
buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)),
structFields.get(i).getFieldObjectInspector());
@@ -415,7 +444,9 @@ public class JsonSerDe implements SerDe
}
return new Text(sb.toString());
}
-
+ private static StringBuilder appendWithQuotes(StringBuilder sb, String value) {
+ return sb == null ? null : sb.append(SerDeUtils.QUOTE).append(value).append(SerDeUtils.QUOTE);
+ }
// TODO : code section copied over from SerDeUtils because of non-standard json production there
// should use quotes for all field names. We should fix this there, and then remove this copy.
// See http://jackson.codehaus.org/1.7.3/javadoc/org/codehaus/jackson/JsonParser.Feature.html#ALLOW_UNQUOTED_FIELD_NAMES
@@ -461,25 +492,37 @@ public class JsonSerDe implements SerDe
break;
}
case STRING: {
- sb.append('"');
- sb.append(SerDeUtils.escapeString(((StringObjectInspector) poi)
- .getPrimitiveJavaObject(o)));
- sb.append('"');
- break;
- }
- case TIMESTAMP: {
- sb.append('"');
- sb.append(((TimestampObjectInspector) poi)
- .getPrimitiveWritableObject(o));
- sb.append('"');
+ String s =
+ SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
+ appendWithQuotes(sb, s);
break;
}
case BINARY: {
throw new IOException("JsonSerDe does not support BINARY type");
}
+ case DATE:
+ Date d = ((DateObjectInspector)poi).getPrimitiveJavaObject(o);
+ appendWithQuotes(sb, d.toString());
+ break;
+ case TIMESTAMP: {
+ Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o);
+ appendWithQuotes(sb, t.toString());
+ break;
+ }
+ case DECIMAL:
+ sb.append(((HiveDecimalObjectInspector)poi).getPrimitiveJavaObject(o));
+ break;
+ case VARCHAR:
+ appendWithQuotes(sb,
+ ((HiveVarcharObjectInspector)poi).getPrimitiveJavaObject(o).toString());
+ break;
+ case CHAR:
+ //this should use HiveChar.getPaddedValue() but it's protected; currently (v0.13)
+ // HiveChar.toString() returns getPaddedValue()
+ appendWithQuotes(sb, ((HiveCharObjectInspector)poi).getPrimitiveJavaObject(o).toString());
+ break;
default:
- throw new RuntimeException("Unknown primitive type: "
- + poi.getPrimitiveCategory());
+ throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
}
}
break;
@@ -524,13 +567,11 @@ public class JsonSerDe implements SerDe
StringBuilder keyBuilder = new StringBuilder();
buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
String keyString = keyBuilder.toString().trim();
- boolean doQuoting = (!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE);
- if (doQuoting) {
- sb.append(SerDeUtils.QUOTE);
+ if((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
+ appendWithQuotes(sb, keyString);
}
- sb.append(keyString);
- if (doQuoting) {
- sb.append(SerDeUtils.QUOTE);
+ else {
+ sb.append(keyString);
}
sb.append(SerDeUtils.COLON);
buildJSONString(sb, e.getValue(), mapValueObjectInspector);
@@ -550,9 +591,7 @@ public class JsonSerDe implements SerDe
if (i > 0) {
sb.append(SerDeUtils.COMMA);
}
- sb.append(SerDeUtils.QUOTE);
- sb.append(structFields.get(i).getFieldName());
- sb.append(SerDeUtils.QUOTE);
+ appendWithQuotes(sb, structFields.get(i).getFieldName());
sb.append(SerDeUtils.COLON);
buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)),
structFields.get(i).getFieldObjectInspector());
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/ReaderWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/ReaderWriter.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/ReaderWriter.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/ReaderWriter.java Fri Feb 14 10:53:31 2014
@@ -22,6 +22,7 @@ package org.apache.hive.hcatalog.data;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.sql.Date;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -29,10 +30,21 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.io.VIntWritable;
import org.apache.hadoop.io.VLongWritable;
-
+/*
+ * when adding support for new types, we should try to use classes of Hive value system to keep
+ * things more readable (though functionally it should not make a difference).
+ */
public abstract class ReaderWriter {
private static final String UTF8 = "UTF-8";
@@ -96,7 +108,26 @@ public abstract class ReaderWriter {
list.add(readDatum(in));
}
return list;
-
+ case DataType.CHAR:
+ HiveCharWritable hcw = new HiveCharWritable();
+ hcw.readFields(in);
+ return hcw.getHiveChar();
+ case DataType.VARCHAR:
+ HiveVarcharWritable hvw = new HiveVarcharWritable();
+ hvw.readFields(in);
+ return hvw.getHiveVarchar();
+ case DataType.DECIMAL:
+ HiveDecimalWritable hdw = new HiveDecimalWritable();
+ hdw.readFields(in);
+ return hdw.getHiveDecimal();
+ case DataType.DATE:
+ DateWritable dw = new DateWritable();
+ dw.readFields(in);
+ return dw.get();
+ case DataType.TIMESTAMP:
+ TimestampWritable tw = new TimestampWritable();
+ tw.readFields(in);
+ return tw.getTimestamp();
default:
throw new IOException("Unexpected data type " + type +
" found in stream.");
@@ -106,9 +137,9 @@ public abstract class ReaderWriter {
public static void writeDatum(DataOutput out, Object val) throws IOException {
// write the data type
byte type = DataType.findType(val);
+ out.write(type);
switch (type) {
case DataType.LIST:
- out.writeByte(DataType.LIST);
List<?> list = (List<?>) val;
int sz = list.size();
out.writeInt(sz);
@@ -118,7 +149,6 @@ public abstract class ReaderWriter {
return;
case DataType.MAP:
- out.writeByte(DataType.MAP);
Map<?, ?> m = (Map<?, ?>) val;
out.writeInt(m.size());
Iterator<?> i =
@@ -131,59 +161,64 @@ public abstract class ReaderWriter {
return;
case DataType.INTEGER:
- out.writeByte(DataType.INTEGER);
new VIntWritable((Integer) val).write(out);
return;
case DataType.LONG:
- out.writeByte(DataType.LONG);
new VLongWritable((Long) val).write(out);
return;
case DataType.FLOAT:
- out.writeByte(DataType.FLOAT);
out.writeFloat((Float) val);
return;
case DataType.DOUBLE:
- out.writeByte(DataType.DOUBLE);
out.writeDouble((Double) val);
return;
case DataType.BOOLEAN:
- out.writeByte(DataType.BOOLEAN);
out.writeBoolean((Boolean) val);
return;
case DataType.BYTE:
- out.writeByte(DataType.BYTE);
out.writeByte((Byte) val);
return;
case DataType.SHORT:
- out.writeByte(DataType.SHORT);
out.writeShort((Short) val);
return;
case DataType.STRING:
String s = (String) val;
byte[] utfBytes = s.getBytes(ReaderWriter.UTF8);
- out.writeByte(DataType.STRING);
out.writeInt(utfBytes.length);
out.write(utfBytes);
return;
case DataType.BINARY:
byte[] ba = (byte[]) val;
- out.writeByte(DataType.BINARY);
out.writeInt(ba.length);
out.write(ba);
return;
case DataType.NULL:
- out.writeByte(DataType.NULL);
+ //for NULL we just write out the type
+ return;
+ case DataType.CHAR:
+ new HiveCharWritable((HiveChar)val).write(out);
+ return;
+ case DataType.VARCHAR:
+ new HiveVarcharWritable((HiveVarchar)val).write(out);
+ return;
+ case DataType.DECIMAL:
+ new HiveDecimalWritable((HiveDecimal)val).write(out);
+ return;
+ case DataType.DATE:
+ new DateWritable((Date)val).write(out);
+ return;
+ case DataType.TIMESTAMP:
+ new TimestampWritable((java.sql.Timestamp)val).write(out);
return;
-
default:
throw new IOException("Unexpected data type " + type +
" found in stream.");
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java Fri Feb 14 10:53:31 2014
@@ -21,23 +21,86 @@ package org.apache.hive.hcatalog.data.sc
import java.io.Serializable;
import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hive.hcatalog.common.HCatException;
+import org.apache.hive.hcatalog.common.HCatUtil;
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class HCatFieldSchema implements Serializable {
-
+/*the implementation of HCatFieldSchema is a bit messy since with the addition of parametrized
+types (e.g. char(7)) we need to represent something richer than an enum but for backwards
+compatibility (and effort required to do full refactoring) this class has both 'type' and 'typeInfo';
+similarly for mapKeyType/mapKeyTypeInfo */
+
public enum Type {
- INT,
- TINYINT,
- SMALLINT,
- BIGINT,
- BOOLEAN,
- FLOAT,
- DOUBLE,
- STRING,
- ARRAY,
- MAP,
- STRUCT,
- BINARY,
+ /*this captures mapping of Hive type names to HCat type names; in the long run
+ * we should just use Hive types directly but that is a larger refactoring effort
+ * For HCat->Pig mapping see PigHCatUtil.getPigType(Type)
+ * For Pig->HCat mapping see HCatBaseStorer#validateSchema(...)*/
+ BOOLEAN(PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN),
+ TINYINT(PrimitiveObjectInspector.PrimitiveCategory.BYTE),
+ SMALLINT(PrimitiveObjectInspector.PrimitiveCategory.SHORT),
+ INT(PrimitiveObjectInspector.PrimitiveCategory.INT),
+ BIGINT(PrimitiveObjectInspector.PrimitiveCategory.LONG),
+ FLOAT(PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
+ DOUBLE(PrimitiveObjectInspector.PrimitiveCategory.DOUBLE),
+ DECIMAL(PrimitiveObjectInspector.PrimitiveCategory.DECIMAL),
+ STRING(PrimitiveObjectInspector.PrimitiveCategory.STRING),
+ CHAR(PrimitiveObjectInspector.PrimitiveCategory.CHAR),
+ VARCHAR(PrimitiveObjectInspector.PrimitiveCategory.VARCHAR),
+ BINARY(PrimitiveObjectInspector.PrimitiveCategory.BINARY),
+ DATE(PrimitiveObjectInspector.PrimitiveCategory.DATE),
+ TIMESTAMP(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP),
+
+ ARRAY(ObjectInspector.Category.LIST),
+ MAP(ObjectInspector.Category.MAP),
+ STRUCT(ObjectInspector.Category.STRUCT);
+
+
+ private final ObjectInspector.Category category;
+ private final PrimitiveObjectInspector.PrimitiveCategory primitiveCategory;
+ private Type(ObjectInspector.Category cat) {
+ category = cat;
+ primitiveCategory = null;
+ assert category != ObjectInspector.Category.PRIMITIVE :
+ "This c'tor should be used for complex category types";
+ }
+ private Type(PrimitiveObjectInspector.PrimitiveCategory primCat) {
+ category = ObjectInspector.Category.PRIMITIVE;
+ primitiveCategory = primCat;
+ }
+ public ObjectInspector.Category getCategory() {
+ return category;
+ }
+ /**
+ * May return {@code null}
+ */
+ public PrimitiveObjectInspector.PrimitiveCategory getPrimitiveCategory() {
+ return primitiveCategory;
+ }
+ public static Type getPrimitiveHType(PrimitiveTypeInfo basePrimitiveTypeInfo) {
+ for(Type t : values()) {
+ if(t.getPrimitiveCategory() == basePrimitiveTypeInfo.getPrimitiveCategory()) {
+ return t;
+ }
+ }
+ throw new TypeNotPresentException(basePrimitiveTypeInfo.getTypeName(), null);
+ }
+ //aid in testing
+ public static int numPrimitiveTypes() {
+ int numPrimitives = 0;
+ for(Type t : values()) {
+ if(t.category == ObjectInspector.Category.PRIMITIVE) {
+ numPrimitives++;
+ }
+ }
+ return numPrimitives;
+ }
}
public enum Category {
@@ -59,10 +122,8 @@ public class HCatFieldSchema implements
}
}
- ;
-
public boolean isComplex() {
- return (category == Category.PRIMITIVE) ? false : true;
+ return category != Category.PRIMITIVE;
}
/**
@@ -72,6 +133,9 @@ public class HCatFieldSchema implements
String fieldName = null;
String comment = null;
+ /**
+ * @deprecated use {@link #typeInfo}
+ */
Type type = null;
Category category = null;
@@ -82,9 +146,18 @@ public class HCatFieldSchema implements
HCatSchema subSchema = null;
// populated if column is Map type
+ @Deprecated
Type mapKeyType = null;
private String typeString = null;
+ /**
+ * This is needed for parametrized types such as decimal(8,9), char(7), varchar(6)
+ */
+ private PrimitiveTypeInfo typeInfo;
+ /**
+ * represents key type for a Map; currently Hive only supports primitive keys
+ */
+ private PrimitiveTypeInfo mapKeyTypeInfo;
@SuppressWarnings("unused")
private HCatFieldSchema() {
@@ -94,6 +167,7 @@ public class HCatFieldSchema implements
/**
* Returns type of the field
* @return type of the field
+ * @deprecated use {@link #getTypeInfo()}
*/
public Type getType() {
return type;
@@ -118,12 +192,18 @@ public class HCatFieldSchema implements
public String getComment() {
return comment;
}
-
+ /**
+ * May return {@code null}
+ */
+ public PrimitiveTypeInfo getTypeInfo() {
+ return typeInfo;
+ }
/**
* Constructor constructing a primitive datatype HCatFieldSchema
* @param fieldName Name of the primitive field
* @param type Type of the primitive field
* @throws HCatException if call made on non-primitive types
+ * @deprecated as of Hive 0.13; use {@link #HCatFieldSchema(String, org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo, String)}
*/
public HCatFieldSchema(String fieldName, Type type, String comment) throws HCatException {
assertTypeInCategory(type, Category.PRIMITIVE, fieldName);
@@ -132,6 +212,17 @@ public class HCatFieldSchema implements
this.category = Category.PRIMITIVE;
this.comment = comment;
}
+ public HCatFieldSchema(String fieldName, PrimitiveTypeInfo typeInfo, String comment)
+ throws HCatException {
+ this.fieldName = fieldName;
+ //HCatUtil.assertNotNull(fieldName, "fieldName cannot be null");//seems sometimes it can be
+ // null, for ARRAY types in particular (which may be a complex type)
+ this.category = Category.PRIMITIVE;
+ this.typeInfo = typeInfo;
+ HCatUtil.assertNotNull(typeInfo, "typeInfo cannot be null; fieldName=" + fieldName, null);
+ type = Type.getPrimitiveHType(typeInfo);
+ this.comment = comment;
+ }
/**
* Constructor for constructing a ARRAY type or STRUCT type HCatFieldSchema, passing type and subschema
@@ -164,9 +255,12 @@ public class HCatFieldSchema implements
* @param mapKeyType - key type of the Map
* @param mapValueSchema - subschema of the value of the Map
* @throws HCatException if call made on non-Map types
+ * @deprecated use {@link #createMapTypeFieldSchema(String, org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo, HCatSchema, String)}
*/
public HCatFieldSchema(String fieldName, Type type, Type mapKeyType, HCatSchema mapValueSchema, String comment) throws HCatException {
assertTypeInCategory(type, Category.MAP, fieldName);
+ //Hive only supports primitive map keys:
+ //https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
assertTypeInCategory(mapKeyType, Category.PRIMITIVE, fieldName);
this.fieldName = fieldName;
this.type = Type.MAP;
@@ -176,6 +270,16 @@ public class HCatFieldSchema implements
this.subSchema.get(0).setName(null);
this.comment = comment;
}
+ public static HCatFieldSchema createMapTypeFieldSchema(String fieldName, PrimitiveTypeInfo mapKeyType,
+ HCatSchema mapValueSchema,
+ String comment) throws HCatException {
+ HCatFieldSchema mapSchema = new HCatFieldSchema(fieldName, Type.MAP,
+ Type.getPrimitiveHType(mapKeyType),
+ mapValueSchema, comment);
+ mapSchema.mapKeyTypeInfo = mapKeyType;
+ return mapSchema;
+ }
+
public HCatSchema getStructSubSchema() throws HCatException {
assertTypeInCategory(this.type, Category.STRUCT, this.fieldName);
@@ -186,12 +290,17 @@ public class HCatFieldSchema implements
assertTypeInCategory(this.type, Category.ARRAY, this.fieldName);
return subSchema;
}
-
+ /**
+ * @deprecated use {@link #getMapKeyTypeInfo()}
+ */
public Type getMapKeyType() throws HCatException {
assertTypeInCategory(this.type, Category.MAP, this.fieldName);
return mapKeyType;
}
-
+ public PrimitiveTypeInfo getMapKeyTypeInfo() throws HCatException {
+ assertTypeInCategory(this.type, Category.MAP, this.fieldName);
+ return mapKeyTypeInfo;
+ }
public HCatSchema getMapValueSchema() throws HCatException {
assertTypeInCategory(this.type, Category.MAP, this.fieldName);
return subSchema;
@@ -227,8 +336,8 @@ public class HCatFieldSchema implements
}
StringBuilder sb = new StringBuilder();
- if (Category.PRIMITIVE == category) {
- sb.append(type);
+ if (!isComplex()) {
+ sb.append(typeInfo == null ? type : typeInfo.getTypeName());
} else if (Category.STRUCT == category) {
sb.append("struct<");
sb.append(subSchema.getSchemaAsTypeString());
@@ -239,7 +348,7 @@ public class HCatFieldSchema implements
sb.append(">");
} else if (Category.MAP == category) {
sb.append("map<");
- sb.append(mapKeyType);
+ sb.append(mapKeyTypeInfo == null ? mapKeyType : mapKeyTypeInfo.getTypeName());
sb.append(",");
sb.append(subSchema.getSchemaAsTypeString());
sb.append(">");
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java Fri Feb 14 10:53:31 2014
@@ -79,13 +79,14 @@ public class HCatSchemaUtils {
public static class MapBuilder extends HCatSchemaBuilder {
- Type keyType = null;
+ PrimitiveTypeInfo keyType = null;
HCatSchema valueSchema = null;
@Override
public HCatSchema build() throws HCatException {
List<HCatFieldSchema> fslist = new ArrayList<HCatFieldSchema>();
- fslist.add(new HCatFieldSchema(null, Type.MAP, keyType, valueSchema, null));
+ fslist.add(HCatFieldSchema.createMapTypeFieldSchema(null, keyType, valueSchema, null));
+
return new HCatSchema(fslist);
}
@@ -94,7 +95,7 @@ public class HCatSchemaUtils {
return this;
}
- public MapBuilder withKeyType(Type keyType) {
+ public MapBuilder withKeyType(PrimitiveTypeInfo keyType) {
this.keyType = keyType;
return this;
}
@@ -118,7 +119,7 @@ public class HCatSchemaUtils {
Category typeCategory = fieldTypeInfo.getCategory();
HCatFieldSchema hCatFieldSchema;
if (Category.PRIMITIVE == typeCategory) {
- hCatFieldSchema = new HCatFieldSchema(fieldName, getPrimitiveHType(fieldTypeInfo), null);
+ hCatFieldSchema = new HCatFieldSchema(fieldName, (PrimitiveTypeInfo)fieldTypeInfo, null);
} else if (Category.STRUCT == typeCategory) {
HCatSchema subSchema = constructHCatSchema((StructTypeInfo) fieldTypeInfo);
hCatFieldSchema = new HCatFieldSchema(fieldName, HCatFieldSchema.Type.STRUCT, subSchema, null);
@@ -126,40 +127,15 @@ public class HCatSchemaUtils {
HCatSchema subSchema = getHCatSchema(((ListTypeInfo) fieldTypeInfo).getListElementTypeInfo());
hCatFieldSchema = new HCatFieldSchema(fieldName, HCatFieldSchema.Type.ARRAY, subSchema, null);
} else if (Category.MAP == typeCategory) {
- HCatFieldSchema.Type mapKeyType = getPrimitiveHType(((MapTypeInfo) fieldTypeInfo).getMapKeyTypeInfo());
HCatSchema subSchema = getHCatSchema(((MapTypeInfo) fieldTypeInfo).getMapValueTypeInfo());
- hCatFieldSchema = new HCatFieldSchema(fieldName, HCatFieldSchema.Type.MAP, mapKeyType, subSchema, null);
+ hCatFieldSchema = HCatFieldSchema.createMapTypeFieldSchema(fieldName,
+ (PrimitiveTypeInfo)((MapTypeInfo)fieldTypeInfo).getMapKeyTypeInfo(), subSchema, null);
} else {
throw new TypeNotPresentException(fieldTypeInfo.getTypeName(), null);
}
return hCatFieldSchema;
}
- private static Type getPrimitiveHType(TypeInfo basePrimitiveTypeInfo) {
- switch (((PrimitiveTypeInfo) basePrimitiveTypeInfo).getPrimitiveCategory()) {
- case BOOLEAN:
- return Type.BOOLEAN;
- case BYTE:
- return Type.TINYINT;
- case DOUBLE:
- return Type.DOUBLE;
- case FLOAT:
- return Type.FLOAT;
- case INT:
- return Type.INT;
- case LONG:
- return Type.BIGINT;
- case SHORT:
- return Type.SMALLINT;
- case STRING:
- return Type.STRING;
- case BINARY:
- return Type.BINARY;
- default:
- throw new TypeNotPresentException(((PrimitiveTypeInfo) basePrimitiveTypeInfo).getTypeName(), null);
- }
- }
-
public static HCatSchema getHCatSchema(Schema schema) throws HCatException {
return getHCatSchema(schema.getFieldSchemas());
}
@@ -174,8 +150,8 @@ public class HCatSchemaUtils {
private static HCatSchema constructHCatSchema(StructTypeInfo stypeInfo) throws HCatException {
CollectionBuilder builder = getStructSchemaBuilder();
- for (String fieldName : ((StructTypeInfo) stypeInfo).getAllStructFieldNames()) {
- builder.addField(getHCatFieldSchema(fieldName, ((StructTypeInfo) stypeInfo).getStructFieldTypeInfo(fieldName)));
+ for (String fieldName : stypeInfo.getAllStructFieldNames()) {
+ builder.addField(getHCatFieldSchema(fieldName, stypeInfo.getStructFieldTypeInfo(fieldName)));
}
return builder.build();
}
@@ -184,7 +160,7 @@ public class HCatSchemaUtils {
Category typeCategory = typeInfo.getCategory();
HCatSchema hCatSchema;
if (Category.PRIMITIVE == typeCategory) {
- hCatSchema = getStructSchemaBuilder().addField(new HCatFieldSchema(null, getPrimitiveHType(typeInfo), null)).build();
+ hCatSchema = getStructSchemaBuilder().addField(new HCatFieldSchema(null, (PrimitiveTypeInfo)typeInfo, null)).build();
} else if (Category.STRUCT == typeCategory) {
HCatSchema subSchema = constructHCatSchema((StructTypeInfo) typeInfo);
hCatSchema = getStructSchemaBuilder().addField(new HCatFieldSchema(null, Type.STRUCT, subSchema, null)).build();
@@ -193,10 +169,10 @@ public class HCatSchemaUtils {
builder.addField(getHCatFieldSchema(null, ((ListTypeInfo) typeInfo).getListElementTypeInfo()));
hCatSchema = new HCatSchema(Arrays.asList(new HCatFieldSchema("", Type.ARRAY, builder.build(), "")));
} else if (Category.MAP == typeCategory) {
- HCatFieldSchema.Type mapKeyType = getPrimitiveHType(((MapTypeInfo) typeInfo).getMapKeyTypeInfo());
HCatSchema subSchema = getHCatSchema(((MapTypeInfo) typeInfo).getMapValueTypeInfo());
MapBuilder builder = getMapSchemaBuilder();
- hCatSchema = builder.withKeyType(mapKeyType).withValueSchema(subSchema).build();
+ hCatSchema = builder.withKeyType((PrimitiveTypeInfo)((MapTypeInfo) typeInfo).getMapKeyTypeInfo())
+ .withValueSchema(subSchema).build();
} else {
throw new TypeNotPresentException(typeInfo.getTypeName(), null);
}
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java Fri Feb 14 10:53:31 2014
@@ -69,10 +69,13 @@ class FileOutputCommitterContainer exten
private static final String TEMP_DIR_NAME = "_temporary";
private static final String LOGS_DIR_NAME = "_logs";
+ /** The directory under which data is initially written for a partitioned table */
+ static final String DYNTEMP_DIR_NAME = "_DYN";
private static final Logger LOG = LoggerFactory.getLogger(FileOutputCommitterContainer.class);
private final boolean dynamicPartitioningUsed;
private boolean partitionsDiscovered;
+ private final boolean customDynamicLocationUsed;
private Map<String, Map<String, String>> partitionsDiscoveredByPath;
private Map<String, JobContext> contextDiscoveredByPath;
@@ -97,6 +100,14 @@ class FileOutputCommitterContainer exten
this.partitionsDiscovered = !dynamicPartitioningUsed;
cachedStorageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo());
+ Table table = new Table(jobInfo.getTableInfo().getTable());
+ if (dynamicPartitioningUsed && Boolean.valueOf((String)table.getProperty("EXTERNAL"))
+ && jobInfo.getCustomDynamicPath() != null
+ && jobInfo.getCustomDynamicPath().length() > 0) {
+ customDynamicLocationUsed = true;
+ } else {
+ customDynamicLocationUsed = false;
+ }
}
@Override
@@ -164,8 +175,12 @@ class FileOutputCommitterContainer exten
Path src;
OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(jobContext);
if (dynamicPartitioningUsed) {
- src = new Path(getPartitionRootLocation(jobInfo.getLocation(), jobInfo.getTableInfo().getTable()
- .getPartitionKeysSize()));
+ if (!customDynamicLocationUsed) {
+ src = new Path(getPartitionRootLocation(jobInfo.getLocation(), jobInfo.getTableInfo().getTable()
+ .getPartitionKeysSize()));
+ } else {
+ src = new Path(getCustomPartitionRootLocation(jobInfo, jobContext.getConfiguration()));
+ }
} else {
src = new Path(jobInfo.getLocation());
}
@@ -235,7 +250,26 @@ class FileOutputCommitterContainer exten
throw new IOException("The method cleanupJob is deprecated and should not be called.");
}
+ private String getCustomPartitionRootLocation(OutputJobInfo jobInfo, Configuration conf) {
+ if (ptnRootLocation == null) {
+ // we only need to calculate it once, it'll be the same for other partitions in this job.
+ String parentPath = jobInfo.getTableInfo().getTableLocation();
+ if (jobInfo.getCustomDynamicRoot() != null
+ && jobInfo.getCustomDynamicRoot().length() > 0) {
+ parentPath = new Path(parentPath, jobInfo.getCustomDynamicRoot()).toString();
+ }
+ Path ptnRoot = new Path(parentPath, DYNTEMP_DIR_NAME +
+ conf.get(HCatConstants.HCAT_DYNAMIC_PTN_JOBID));
+ ptnRootLocation = ptnRoot.toString();
+ }
+ return ptnRootLocation;
+ }
+
private String getPartitionRootLocation(String ptnLocn, int numPtnKeys) {
+ if (customDynamicLocationUsed) {
+ return null;
+ }
+
if (ptnRootLocation == null) {
// we only need to calculate it once, it'll be the same for other partitions in this job.
Path ptnRoot = new Path(ptnLocn);
@@ -255,6 +289,7 @@ class FileOutputCommitterContainer exten
* @param jobInfo The OutputJobInfo.
* @param partLocnRoot The table-equivalent location root of the partition
* (temporary dir if dynamic partition, table dir if static)
+ * @param dynPartPath The path of dynamic partition which is created
* @param partKVs The keyvalue pairs that form the partition
* @param outputSchema The output schema for the partition
* @param params The parameters to store inside the partition
@@ -268,7 +303,7 @@ class FileOutputCommitterContainer exten
private Partition constructPartition(
JobContext context, OutputJobInfo jobInfo,
- String partLocnRoot, Map<String, String> partKVs,
+ String partLocnRoot, String dynPartPath, Map<String, String> partKVs,
HCatSchema outputSchema, Map<String, String> params,
Table table, FileSystem fs,
String grpName, FsPermission perms) throws IOException {
@@ -292,7 +327,10 @@ class FileOutputCommitterContainer exten
// Sets permissions and group name on partition dirs and files.
Path partPath;
- if (Boolean.valueOf((String)table.getProperty("EXTERNAL"))
+ if (customDynamicLocationUsed) {
+ partPath = new Path(dynPartPath);
+ } else if (!dynamicPartitioningUsed
+ && Boolean.valueOf((String)table.getProperty("EXTERNAL"))
&& jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) {
// honor external table that specifies the location
partPath = new Path(jobInfo.getLocation());
@@ -315,7 +353,7 @@ class FileOutputCommitterContainer exten
// Set the location in the StorageDescriptor
if (dynamicPartitioningUsed) {
- String dynamicPartitionDestination = getFinalDynamicPartitionDestination(table, partKVs);
+ String dynamicPartitionDestination = getFinalDynamicPartitionDestination(table, partKVs, jobInfo);
if (harProcessor.isEnabled()) {
harProcessor.exec(context, partition, partPath);
partition.getSd().setLocation(
@@ -344,14 +382,25 @@ class FileOutputCommitterContainer exten
}
}
- private String getFinalDynamicPartitionDestination(Table table, Map<String, String> partKVs) {
- // file:///tmp/hcat_junit_warehouse/employee/_DYN0.7770480401313761/emp_country=IN/emp_state=KA ->
- // file:///tmp/hcat_junit_warehouse/employee/emp_country=IN/emp_state=KA
+ private String getFinalDynamicPartitionDestination(Table table, Map<String, String> partKVs,
+ OutputJobInfo jobInfo) {
Path partPath = new Path(table.getTTable().getSd().getLocation());
- for (FieldSchema partKey : table.getPartitionKeys()) {
- partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs);
+ if (!customDynamicLocationUsed) {
+ // file:///tmp/hcat_junit_warehouse/employee/_DYN0.7770480401313761/emp_country=IN/emp_state=KA ->
+ // file:///tmp/hcat_junit_warehouse/employee/emp_country=IN/emp_state=KA
+ for (FieldSchema partKey : table.getPartitionKeys()) {
+ partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs);
+ }
+
+ return partPath.toString();
+ } else {
+ // if custom root specified, update the parent path
+ if (jobInfo.getCustomDynamicRoot() != null
+ && jobInfo.getCustomDynamicRoot().length() > 0) {
+ partPath = new Path(partPath, jobInfo.getCustomDynamicRoot());
+ }
+ return new Path(partPath, HCatFileUtil.resolveCustomPath(jobInfo, partKVs, false)).toString();
}
- return partPath.toString();
}
private Map<String, String> getStorerParameterMap(StorerInfo storer) {
@@ -480,8 +529,11 @@ class FileOutputCommitterContainer exten
if (LOG.isDebugEnabled()) {
LOG.debug("Moving directory: " + file + " to " + parentDir);
}
- if (!fs.rename(file, parentDir)) {
- final String msg = "Failed to move file: " + file + " to " + parentDir;
+
+ // If custom dynamic location provided, need to rename to final output path
+ Path dstPath = !customDynamicLocationUsed ? parentDir : finalOutputPath;
+ if (!fs.rename(file, dstPath)) {
+ final String msg = "Failed to move file: " + file + " to " + dstPath;
LOG.error(msg);
throw new HCatException(ErrorType.ERROR_MOVE_FAILED, msg);
}
@@ -576,7 +628,12 @@ class FileOutputCommitterContainer exten
for (FileStatus st : status) {
LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<String, String>();
- Warehouse.makeSpecFromName(fullPartSpec, st.getPath());
+ if (!customDynamicLocationUsed) {
+ Warehouse.makeSpecFromName(fullPartSpec, st.getPath());
+ } else {
+ HCatFileUtil.getPartKeyValuesForCustomLocation(fullPartSpec, jobInfo,
+ st.getPath().toString());
+ }
partitionsDiscoveredByPath.put(st.getPath().toString(), fullPartSpec);
JobConf jobConf = (JobConf)context.getConfiguration();
JobContext currContext = HCatMapRedUtil.createJobContext(
@@ -636,7 +693,7 @@ class FileOutputCommitterContainer exten
partitionsToAdd.add(
constructPartition(
context,jobInfo,
- tblPath.toString(), jobInfo.getPartitionValues()
+ tblPath.toString(), null, jobInfo.getPartitionValues()
,jobInfo.getOutputSchema(), getStorerParameterMap(storer)
,table, fs
,grpName,perms));
@@ -645,7 +702,8 @@ class FileOutputCommitterContainer exten
partitionsToAdd.add(
constructPartition(
context,jobInfo,
- getPartitionRootLocation(entry.getKey(),entry.getValue().size()), entry.getValue()
+ getPartitionRootLocation(entry.getKey(),entry.getValue().size())
+ ,entry.getKey(), entry.getValue()
,jobInfo.getOutputSchema(), getStorerParameterMap(storer)
,table, fs
,grpName,perms));
@@ -659,13 +717,16 @@ class FileOutputCommitterContainer exten
//Publish the new partition(s)
if (dynamicPartitioningUsed && harProcessor.isEnabled() && (!partitionsToAdd.isEmpty())){
-
- Path src = new Path(ptnRootLocation);
- // check here for each dir we're copying out, to see if it
- // already exists, error out if so
- moveTaskOutputs(fs, src, src, tblPath, true);
- moveTaskOutputs(fs, src, src, tblPath, false);
- fs.delete(src, true);
+ if (!customDynamicLocationUsed) {
+ Path src = new Path(ptnRootLocation);
+ // check here for each dir we're copying out, to see if it
+ // already exists, error out if so
+ moveTaskOutputs(fs, src, src, tblPath, true);
+ moveTaskOutputs(fs, src, src, tblPath, false);
+ fs.delete(src, true);
+ } else {
+ moveCustomLocationTaskOutputs(fs, table, hiveConf);
+ }
try {
updateTableSchema(client, table, jobInfo.getOutputSchema());
LOG.info("HAR is being used. The table {} has new partitions {}.", table.getTableName(), ptnInfos);
@@ -687,10 +748,14 @@ class FileOutputCommitterContainer exten
updateTableSchema(client, table, jobInfo.getOutputSchema());
LOG.info("HAR not is not being used. The table {} has new partitions {}.", table.getTableName(), ptnInfos);
if (dynamicPartitioningUsed && (partitionsToAdd.size()>0)){
- Path src = new Path(ptnRootLocation);
- moveTaskOutputs(fs, src, src, tblPath, true);
- moveTaskOutputs(fs, src, src, tblPath, false);
- fs.delete(src, true);
+ if (!customDynamicLocationUsed) {
+ Path src = new Path(ptnRootLocation);
+ moveTaskOutputs(fs, src, src, tblPath, true);
+ moveTaskOutputs(fs, src, src, tblPath, false);
+ fs.delete(src, true);
+ } else {
+ moveCustomLocationTaskOutputs(fs, table, hiveConf);
+ }
}
client.add_partitions(partitionsToAdd);
partitionsAdded = partitionsToAdd;
@@ -720,6 +785,24 @@ class FileOutputCommitterContainer exten
}
}
+ private void moveCustomLocationTaskOutputs(FileSystem fs, Table table, Configuration conf)
+ throws IOException {
+ // in case of custom dynamic partitions, we can't just move the sub-tree of partition root
+ // directory since the partitions location contain regex pattern. We need to first find the
+ // final destination of each partition and move its output.
+ for (Entry<String, Map<String, String>> entry : partitionsDiscoveredByPath.entrySet()) {
+ Path src = new Path(entry.getKey());
+ Path destPath = new Path(getFinalDynamicPartitionDestination(table, entry.getValue(), jobInfo));
+ moveTaskOutputs(fs, src, src, destPath, true);
+ moveTaskOutputs(fs, src, src, destPath, false);
+ }
+ // delete the parent temp directory of all custom dynamic partitions
+ Path parentPath = new Path(getCustomPartitionRootLocation(jobInfo, conf));
+ if (fs.exists(parentPath)) {
+ fs.delete(parentPath, true);
+ }
+ }
+
private void cancelDelegationTokens(JobContext context) throws IOException{
LOG.info("Cancelling deletgation token for the job.");
HiveMetaStoreClient client = null;
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FosterStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FosterStorageHandler.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FosterStorageHandler.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FosterStorageHandler.java Fri Feb 14 10:53:31 2014
@@ -49,9 +49,6 @@ import java.util.Map;
public class FosterStorageHandler extends DefaultStorageHandler {
public Configuration conf;
- /** The directory under which data is initially written for a partitioned table */
- protected static final String DYNTEMP_DIR_NAME = "_DYN";
-
/** The directory under which data is initially written for a non partitioned table */
protected static final String TEMP_DIR_NAME = "_TEMP";
@@ -118,17 +115,28 @@ public class FosterStorageHandler extend
// For dynamic partitioned writes without all keyvalues specified,
// we create a temp dir for the associated write job
if (dynHash != null) {
- parentPath = new Path(parentPath,
- DYNTEMP_DIR_NAME + dynHash).toString();
+ // if external table and custom root specified, update the parent path
+ if (Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL"))
+ && jobInfo.getCustomDynamicRoot() != null
+ && jobInfo.getCustomDynamicRoot().length() > 0) {
+ parentPath = new Path(parentPath, jobInfo.getCustomDynamicRoot()).toString();
+ }
+ parentPath = new Path(parentPath, FileOutputCommitterContainer.DYNTEMP_DIR_NAME + dynHash).toString();
}
String outputLocation;
- if ((dynHash == null)
+ if ((dynHash != null)
+ && Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL"))
+ && jobInfo.getCustomDynamicPath() != null
+ && jobInfo.getCustomDynamicPath().length() > 0) {
+ // dynamic partitioning with custom path; resolve the custom path
+ // using partition column values
+ outputLocation = HCatFileUtil.resolveCustomPath(jobInfo, null, true);
+ } else if ((dynHash == null)
&& Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL"))
&& jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) {
// honor custom location for external table apart from what metadata specifies
- // only if we're not using dynamic partitioning - see HIVE-5011
outputLocation = jobInfo.getLocation();
} else if (dynHash == null && jobInfo.getPartitionValues().size() == 0) {
// For non-partitioned tables, we send them to the temp dir
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java Fri Feb 14 10:53:31 2014
@@ -159,6 +159,11 @@ public class HCatOutputFormat extends HC
}
conf.set(HCatConstants.HCAT_DYNAMIC_PTN_JOBID, dynHash);
+ // if custom pattern is set in case of dynamic partitioning, configure custom path
+ String customPattern = conf.get(HCatConstants.HCAT_DYNAMIC_CUSTOM_PATTERN);
+ if (customPattern != null) {
+ HCatFileUtil.setCustomPath(customPattern, outputJobInfo);
+ }
}
outputJobInfo.setPartitionValues(valueMap);
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java Fri Feb 14 10:53:31 2014
@@ -50,6 +50,12 @@ public class OutputJobInfo implements Se
/** The location of the partition being written */
private String location;
+ /** The root location of custom dynamic partitions being written */
+ private String customDynamicRoot;
+
+ /** The relative path of custom dynamic partitions being written */
+ private String customDynamicPath;
+
/** The partition values to publish to, if used for output*/
private Map<String, String> partitionValues;
@@ -71,10 +77,6 @@ public class OutputJobInfo implements Se
* @param databaseName the db name
* @param tableName the table name
* @param partitionValues The partition values to publish to, can be null or empty Map to
- * work with hadoop security, the kerberos principal name of the server - else null
- * The principal name should be of the form:
- * <servicename>/_HOST@<realm> like "hcat/_HOST@myrealm.com"
- * The special string _HOST will be replaced automatically with the correct host name
* indicate write to a unpartitioned table. For partitioned tables, this map should
* contain keys for all partition columns with corresponding values.
*/
@@ -167,6 +169,28 @@ public class OutputJobInfo implements Se
}
/**
+ * @param customDynamicLocation the custom location for dynamic partitions
+ */
+ void setCustomDynamicLocation(String customDynamicRoot, String customDynamicPath) {
+ this.customDynamicRoot = customDynamicRoot;
+ this.customDynamicPath = customDynamicPath;
+ }
+
+ /**
+ * @return the root location for custom dynamic partitions
+ */
+ String getCustomDynamicRoot() {
+ return customDynamicRoot;
+ }
+
+ /**
+ * @return the relative path custom location for dynamic partitions
+ */
+ String getCustomDynamicPath() {
+ return customDynamicPath;
+ }
+
+ /**
* Sets the value of partitionValues
* @param partitionValues the partition values to set
*/
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java Fri Feb 14 10:53:31 2014
@@ -97,10 +97,4 @@ public class HcatTestUtils {
}
}
- /**
- * Used by various tests to make sure the path is safe for Windows
- */
- public static String makePathASafeFileName(String filePath) {
- return new File(filePath).getPath().replaceAll("\\\\", "/");
- }
}
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/HCatDataCheckUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/HCatDataCheckUtil.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/HCatDataCheckUtil.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/HCatDataCheckUtil.java Fri Feb 14 10:53:31 2014
@@ -19,6 +19,7 @@
package org.apache.hive.hcatalog.data;
import java.io.IOException;
+import java.sql.Date;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
@@ -87,14 +88,23 @@ public class HCatDataCheckUtil {
public static boolean recordsEqual(HCatRecord first, HCatRecord second) {
- return (compareRecords(first, second) == 0);
+ return recordsEqual(first, second, null);
+ }
+ public static boolean recordsEqual(HCatRecord first, HCatRecord second,
+ StringBuilder debugDetail) {
+ return (compareRecords(first, second, debugDetail) == 0);
}
public static int compareRecords(HCatRecord first, HCatRecord second) {
- return compareRecordContents(first.getAll(), second.getAll());
+ return compareRecords(first, second, null);
+ }
+ public static int compareRecords(HCatRecord first, HCatRecord second,
+ StringBuilder debugDetail) {
+ return compareRecordContents(first.getAll(), second.getAll(), debugDetail);
}
- public static int compareRecordContents(List<Object> first, List<Object> second) {
+ public static int compareRecordContents(List<Object> first, List<Object> second,
+ StringBuilder debugDetail) {
int mySz = first.size();
int urSz = second.size();
if (mySz != urSz) {
@@ -103,6 +113,22 @@ public class HCatDataCheckUtil {
for (int i = 0; i < first.size(); i++) {
int c = DataType.compare(first.get(i), second.get(i));
if (c != 0) {
+ if(debugDetail != null) {
+ String msg = "first.get(" + i + "}='" + first.get(i) + "' second.get(" +
+ i + ")='" + second.get(i) + "' compared as " + c + "\n" +
+ "Types 1st/2nd=" + DataType.findType(first.get(i)) + "/" +DataType.findType(
+ second.get(i)) + '\n' +
+ "first='" + first.get(i) + "' second='" + second.get(i) + "'";
+ if(first.get(i) instanceof Date) {
+ msg += "\n((Date)first.get(i)).getTime()=" + ((Date)first.get(i)).getTime();
+ }
+ if(second.get(i) instanceof Date) {
+ msg += "\n((Date)second.get(i)).getTime()=" + ((Date)second.get(i)).getTime();
+ }
+
+ debugDetail.append(msg);
+ throw new RuntimeException(debugDetail.toString());
+ }
return c;
}
}
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java Fri Feb 14 10:53:31 2014
@@ -28,20 +28,32 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Timestamp;
import java.util.ArrayList;
+import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hive.hcatalog.common.HCatException;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils;
import junit.framework.Assert;
import junit.framework.TestCase;
+import org.apache.pig.parser.AliasMasker;
public class TestDefaultHCatRecord extends TestCase {
+ /**
+ * test that we properly serialize/deserialize HCatRecordS
+ * @throws IOException
+ */
public void testRYW() throws IOException {
File f = new File("binary.dat");
@@ -65,7 +77,9 @@ public class TestDefaultHCatRecord exten
for (int i = 0; i < recs.length; i++) {
HCatRecord rec = new DefaultHCatRecord();
rec.readFields(inpStream);
- Assert.assertTrue(HCatDataCheckUtil.recordsEqual(recs[i], rec));
+ StringBuilder msg = new StringBuilder("recs[" + i + "]='" + recs[i] + "' rec='" + rec + "'");
+ boolean isEqual = HCatDataCheckUtil.recordsEqual(recs[i], rec, msg);
+ Assert.assertTrue(msg.toString(), isEqual);
}
Assert.assertEquals(fInStream.available(), 0);
@@ -134,6 +148,21 @@ public class TestDefaultHCatRecord exten
Assert.assertTrue(HCatDataCheckUtil.recordsEqual(newRec, inpRec));
}
+ /**
+ * Test type specific get/set methods on HCatRecord types added in Hive 13
+ * @throws HCatException
+ */
+ public void testGetSetByType3() throws HCatException {
+ HCatRecord inpRec = getHCat13TypesRecord();
+ HCatRecord newRec = new DefaultHCatRecord(inpRec.size());
+ HCatSchema hsch = HCatSchemaUtils.getHCatSchema(
+ "a:decimal(5,2),b:char(10),c:varchar(20),d:date,e:timestamp");
+ newRec.setDecimal("a", hsch, inpRec.getDecimal("a", hsch));
+ newRec.setChar("b", hsch, inpRec.getChar("b", hsch));
+ newRec.setVarchar("c", hsch, inpRec.getVarchar("c", hsch));
+ newRec.setDate("d", hsch, inpRec.getDate("d", hsch));
+ newRec.setTimestamp("e", hsch, inpRec.getTimestamp("e", hsch));
+ }
private HCatRecord getGetSet2InpRec() {
List<Object> rlist = new ArrayList<Object>();
@@ -238,9 +267,32 @@ public class TestDefaultHCatRecord exten
rec_6.add(getList());
HCatRecord tup_6 = new DefaultHCatRecord(rec_6);
-
- return new HCatRecord[]{tup_1, tup_2, tup_3, tup_4, tup_5, tup_6};
-
+ return new HCatRecord[]{tup_1, tup_2, tup_3, tup_4, tup_5, tup_6, getHCat13TypesRecord(),
+ getHCat13TypesComplexRecord()};
+ }
+ private static HCatRecord getHCat13TypesRecord() {
+ List<Object> rec_hcat13types = new ArrayList<Object>(5);
+ rec_hcat13types.add(HiveDecimal.create(new BigDecimal("123.45")));//prec 5, scale 2
+ rec_hcat13types.add(new HiveChar("hive_char", 10));
+ rec_hcat13types.add(new HiveVarchar("hive_varchar", 20));
+ rec_hcat13types.add(Date.valueOf("2014-01-06"));
+ rec_hcat13types.add(new Timestamp(System.currentTimeMillis()));
+ return new DefaultHCatRecord(rec_hcat13types);
+ }
+ private static HCatRecord getHCat13TypesComplexRecord() {
+ List<Object> rec_hcat13ComplexTypes = new ArrayList<Object>();
+ Map<HiveDecimal, String> m = new HashMap<HiveDecimal, String>();
+ m.put(HiveDecimal.create(new BigDecimal("1234.12")), "1234.12");
+ m.put(HiveDecimal.create(new BigDecimal("1234.13")), "1234.13");
+ rec_hcat13ComplexTypes.add(m);
+
+ Map<Timestamp, List<Object>> m2 = new HashMap<Timestamp, List<Object>>();
+ List<Object> list = new ArrayList<Object>();
+ list.add(Date.valueOf("2014-01-05"));
+ list.add(new HashMap<HiveDecimal, String>(m));
+ m2.put(new Timestamp(System.currentTimeMillis()), list);
+ rec_hcat13ComplexTypes.add(m2);
+ return new DefaultHCatRecord(rec_hcat13ComplexTypes);
}
private Object getList() {
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java Fri Feb 14 10:53:31 2014
@@ -18,6 +18,9 @@
*/
package org.apache.hive.hcatalog.data;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -27,6 +30,9 @@ import java.util.Properties;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.io.Text;
@@ -85,6 +91,11 @@ public class TestJsonSerDe extends TestC
c1_1.add(i2);
c1.add(c1_1);
rlist.add(c1);
+ rlist.add(HiveDecimal.create(new BigDecimal("123.45")));//prec 5, scale 2
+ rlist.add(new HiveChar("hive_char", 10));
+ rlist.add(new HiveVarchar("hive_varchar", 20));
+ rlist.add(Date.valueOf("2014-01-07"));
+ rlist.add(new Timestamp(System.currentTimeMillis()));
List<Object> nlist = new ArrayList<Object>(13);
nlist.add(null); // tinyint
@@ -100,20 +111,26 @@ public class TestJsonSerDe extends TestC
nlist.add(null); // map
nlist.add(null); // bool
nlist.add(null); // complex
+ nlist.add(null); //decimal(5,2)
+ nlist.add(null); //char(10)
+ nlist.add(null); //varchar(20)
+ nlist.add(null); //date
+ nlist.add(null); //timestamp
String typeString =
"tinyint,smallint,int,bigint,double,float,string,string,"
+ "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean,"
- + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>";
+ + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>," +
+ "decimal(5,2),char(10),varchar(20),date,timestamp";
Properties props = new Properties();
- props.put(serdeConstants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1");
+ props.put(serdeConstants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts");
props.put(serdeConstants.LIST_COLUMN_TYPES, typeString);
// props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
// props.put(Constants.SERIALIZATION_FORMAT, "1");
- data.add(new Pair(props, new DefaultHCatRecord(rlist)));
- data.add(new Pair(props, new DefaultHCatRecord(nlist)));
+ data.add(new Pair<Properties, HCatRecord>(props, new DefaultHCatRecord(rlist)));
+ data.add(new Pair<Properties, HCatRecord>(props, new DefaultHCatRecord(nlist)));
return data;
}
@@ -137,14 +154,17 @@ public class TestJsonSerDe extends TestC
LOG.info("ONE:{}", s);
Object o1 = hrsd.deserialize(s);
- assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o1));
+ StringBuilder msg = new StringBuilder();
+ boolean isEqual = HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o1);
+ assertTrue(msg.toString(), isEqual);
Writable s2 = jsde.serialize(o1, hrsd.getObjectInspector());
LOG.info("TWO:{}", s2);
Object o2 = jsde.deserialize(s2);
LOG.info("deserialized TWO : {} ", o2);
-
- assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o2));
+ msg.setLength(0);
+ isEqual = HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o2, msg);
+ assertTrue(msg.toString(), isEqual);
}
}
@@ -153,7 +173,7 @@ public class TestJsonSerDe extends TestC
/**
* This test has been added to account for HCATALOG-436
* We write out columns with "internal column names" such
- * as "_col0", but try to read with retular column names.
+ * as "_col0", but try to read with regular column names.
*/
Configuration conf = new Configuration();
@@ -190,7 +210,9 @@ public class TestJsonSerDe extends TestC
Object o2 = rjsd.deserialize(s);
LOG.info("deserialized TWO : {} ", o2);
- assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o2));
+ StringBuilder msg = new StringBuilder();
+ boolean isEqual = HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o2, msg);
+ assertTrue(msg.toString(), isEqual);
}
}
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchema.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchema.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchema.java Fri Feb 14 10:53:31 2014
@@ -19,6 +19,7 @@
package org.apache.hive.hcatalog.data.schema;
import junit.framework.TestCase;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hive.hcatalog.common.HCatException;
import java.util.ArrayList;
@@ -62,6 +63,14 @@ public class TestHCatSchema extends Test
HCatFieldSchema memberID2 = new HCatFieldSchema("memberID", HCatFieldSchema.Type.INT, "as a number");
assertTrue("Expected objects to be equal", memberID1.equals(memberID2));
assertTrue("Expected hash codes to be equal", memberID1.hashCode() == memberID2.hashCode());
+ memberID1 = new HCatFieldSchema("memberID", TypeInfoFactory.getDecimalTypeInfo(5,2), "decimal(5,2)");
+ memberID2 = new HCatFieldSchema("memberID", TypeInfoFactory.getDecimalTypeInfo(5,3), "decimal(5)");
+ assertFalse("Expected objects to be unequal", memberID1.equals(memberID2));
+ assertFalse("Expected hash codes to be unequal", memberID1.hashCode() == memberID2.hashCode());
+ memberID1 = new HCatFieldSchema("memberID", TypeInfoFactory.getVarcharTypeInfo(5), "varchar(5)");
+ memberID2 = new HCatFieldSchema("memberID", TypeInfoFactory.getVarcharTypeInfo(5), "varchar(5)");
+ assertTrue("Expected objects to be equal", memberID1.equals(memberID2));
+ assertTrue("Expected hash codes to be equal", memberID1.hashCode() == memberID2.hashCode());
}
public void testCannotInstantiateSchemaWithRepeatedFieldNames() throws HCatException {
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java Fri Feb 14 10:53:31 2014
@@ -26,7 +26,9 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.hcatalog.common.HCatUtil;
+import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -35,6 +37,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Properties;
/**
* Simplify writing HCatalog tests that require a HiveMetaStore.
@@ -81,7 +84,26 @@ public class HCatBaseTest {
}
protected void logAndRegister(PigServer server, String query) throws IOException {
+ logAndRegister(server, query, 1);
+ }
+ protected void logAndRegister(PigServer server, String query, int lineNumber) throws IOException {
+ assert lineNumber > 0 : "(lineNumber > 0) is false";
LOG.info("Registering pig query: " + query);
- server.registerQuery(query);
+ server.registerQuery(query, lineNumber);
+ }
+
+ /**
+ * creates PigServer in LOCAL mode.
+ * http://pig.apache.org/docs/r0.12.0/perf.html#error-handling
+ * @param stopOnFailure equivalent of "-stop_on_failure" command line arg, setting to 'true' makes
+ * debugging easier
+ */
+ public static PigServer createPigServer(boolean stopOnFailure) throws ExecException {
+ if(stopOnFailure) {
+ Properties p = new Properties();
+ p.put("stop.on.failure", Boolean.TRUE.toString());
+ return new PigServer(ExecType.LOCAL, p);
+ }
+ return new PigServer(ExecType.LOCAL);
}
}
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java Fri Feb 14 10:53:31 2014
@@ -84,6 +84,7 @@ public abstract class HCatMapReduceTest
protected abstract List<FieldSchema> getTableColumns();
private static FileSystem fs;
+ private String externalTableLocation = null;
protected Boolean isTableExternal() {
return false;
@@ -123,6 +124,12 @@ public abstract class HCatMapReduceTest
String databaseName = (dbName == null) ? MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName;
client.dropTable(databaseName, tableName);
+ // in case of external table, drop the table contents as well
+ if (isTableExternal() && (externalTableLocation != null)) {
+ if (fs.exists(new Path(externalTableLocation))) {
+ fs.delete(new Path(externalTableLocation), true);
+ }
+ }
} catch (Exception e) {
e.printStackTrace();
throw e;
@@ -167,6 +174,9 @@ public abstract class HCatMapReduceTest
sd.setOutputFormat(outputFormat());
Map<String, String> tableParams = new HashMap<String, String>();
+ if (isTableExternal()) {
+ tableParams.put("EXTERNAL", "TRUE");
+ }
tbl.setParameters(tableParams);
client.createTable(tbl);
@@ -234,7 +244,8 @@ public abstract class HCatMapReduceTest
Job runMRCreate(Map<String, String> partitionValues,
List<HCatFieldSchema> partitionColumns, List<HCatRecord> records,
int writeCount, boolean assertWrite) throws Exception {
- return runMRCreate(partitionValues, partitionColumns, records, writeCount, assertWrite, true);
+ return runMRCreate(partitionValues, partitionColumns, records, writeCount, assertWrite,
+ true, null);
}
/**
@@ -250,7 +261,8 @@ public abstract class HCatMapReduceTest
*/
Job runMRCreate(Map<String, String> partitionValues,
List<HCatFieldSchema> partitionColumns, List<HCatRecord> records,
- int writeCount, boolean assertWrite, boolean asSingleMapTask) throws Exception {
+ int writeCount, boolean assertWrite, boolean asSingleMapTask,
+ String customDynamicPathPattern) throws Exception {
writeRecords = records;
MapCreate.writeCount = 0;
@@ -283,6 +295,9 @@ public abstract class HCatMapReduceTest
job.setOutputFormatClass(HCatOutputFormat.class);
OutputJobInfo outputJobInfo = OutputJobInfo.create(dbName, tableName, partitionValues);
+ if (customDynamicPathPattern != null) {
+ job.getConfiguration().set(HCatConstants.HCAT_DYNAMIC_CUSTOM_PATTERN, customDynamicPathPattern);
+ }
HCatOutputFormat.setOutput(job, outputJobInfo);
job.setMapOutputKeyClass(BytesWritable.class);
@@ -313,6 +328,10 @@ public abstract class HCatMapReduceTest
Assert.assertEquals(writeCount, MapCreate.writeCount);
}
+ if (isTableExternal()) {
+ externalTableLocation = outputJobInfo.getTableInfo().getTableLocation();
+ }
+
return job;
}
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java Fri Feb 14 10:53:31 2014
@@ -101,7 +101,7 @@ public class TestHCatDynamicPartitioned
*/
@Test
public void testHCatDynamicPartitionedTable() throws Exception {
- runHCatDynamicPartitionedTable(true);
+ runHCatDynamicPartitionedTable(true, null);
}
/**
@@ -110,12 +110,13 @@ public class TestHCatDynamicPartitioned
*/
@Test
public void testHCatDynamicPartitionedTableMultipleTask() throws Exception {
- runHCatDynamicPartitionedTable(false);
+ runHCatDynamicPartitionedTable(false, null);
}
- protected void runHCatDynamicPartitionedTable(boolean asSingleMapTask) throws Exception {
+ protected void runHCatDynamicPartitionedTable(boolean asSingleMapTask,
+ String customDynamicPathPattern) throws Exception {
generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
- runMRCreate(null, dataColumns, writeRecords, NUM_RECORDS, true, asSingleMapTask);
+ runMRCreate(null, dataColumns, writeRecords, NUM_RECORDS, true, asSingleMapTask, customDynamicPathPattern);
runMRRead(NUM_RECORDS);
@@ -142,7 +143,8 @@ public class TestHCatDynamicPartitioned
IOException exc = null;
try {
generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
- Job job = runMRCreate(null, dataColumns, writeRecords, NUM_RECORDS, false);
+ Job job = runMRCreate(null, dataColumns, writeRecords, NUM_RECORDS, false,
+ true, customDynamicPathPattern);
if (HCatUtil.isHadoop23()) {
Assert.assertTrue(job.isSuccessful()==false);
Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java Fri Feb 14 10:53:31 2014
@@ -19,6 +19,9 @@
package org.apache.hive.hcatalog.mapreduce;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
public class TestHCatExternalDynamicPartitioned extends TestHCatDynamicPartitioned {
@Override
@@ -26,4 +29,20 @@ public class TestHCatExternalDynamicPart
return true;
}
+ @BeforeClass
+ public static void generateInputData() throws Exception {
+ tableName = "testHCatExternalDynamicPartitionedTable";
+ generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
+ generateDataColumns();
+ }
+
+ /**
+ * Run the external dynamic partitioning test but with single map task
+ * @throws Exception
+ */
+ @Test
+ public void testHCatExternalDynamicCustomLocation() throws Exception {
+ runHCatDynamicPartitionedTable(true, "mapred/externalDynamicOutput/${p1}");
+ }
+
}
Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/pom.xml?rev=1568236&r1=1568235&r2=1568236&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/pom.xml (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/pom.xml Fri Feb 14 10:53:31 2014
@@ -70,6 +70,13 @@
<artifactId>pig</artifactId>
<version>${pig.version}</version>
</dependency>
+ <dependency>
+ <!--this should be automatically brought in by Pig, it's not in Pig 0.12 due to a bug
+ in Pig which requires it This is fixed in Pig's pom file in ASF trunk (pig 13)-->
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.2</version>
+ </dependency>
</dependencies>
</profile>
<profile>
@@ -91,6 +98,13 @@
<version>${pig.version}</version>
<classifier>h2</classifier>
</dependency>
+ <dependency>
+ <!--this should be automatically brought in by Pig, it's not in Pig 0.12 due to a bug
+ in Pig which requires it This is fixed in Pig's pom file in ASF trunk (pig 13)-->
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.2</version>
+ </dependency>
</dependencies>
</profile>
</profiles>