You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/07/02 04:03:16 UTC
svn commit: r790435 [1/3] - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/java/org/apache/hadoop/hive/ql/plan/
ql/src/test/org/apache/hadoop/hive/ql/io/ ql/src/test/queries/cli...
Author: namit
Date: Thu Jul 2 02:03:14 2009
New Revision: 790435
URL: http://svn.apache.org/viewvc?rev=790435&view=rev
Log:
HIVE-136. escaping in lazy serde
(Zheng Shao via namit)
Added:
hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
hadoop/hive/trunk/serde/if/serde.thrift
hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java
hadoop/hive/trunk/serde/src/gen-php/serde_constants.php
hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazyListObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazyMapObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazySimpleStructObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveJavaObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveWritableObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaStringObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableStringObjectInspector.java
hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java
hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Jul 2 02:03:14 2009
@@ -56,6 +56,9 @@
HIVE-567. jdbc: integrate hive with pentaho report designer
(Raghotham Murthy via namit)
+ HIVE-136. escaping in lazy serde
+ (Zheng Shao via namit)
+
IMPROVEMENTS
HIVE-389. Option to build without ivy (jssarma)
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Thu Jul 2 02:03:14 2009
@@ -31,6 +31,7 @@
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -673,10 +674,10 @@
if (crtTbl.getSerName() != null) {
tbl.setSerializationLib(crtTbl.getSerName());
if (crtTbl.getMapProp() != null) {
- Iterator<Map.Entry<String, String>> iter = crtTbl.getMapProp()
+ Iterator<Entry<String, String>> iter = crtTbl.getMapProp()
.entrySet().iterator();
while (iter.hasNext()) {
- Map.Entry<String, String> m = (Map.Entry<String, String>) iter.next();
+ Entry<String, String> m = (Entry<String, String>) iter.next();
tbl.setSerdeParam(m.getKey(), m.getValue());
}
}
@@ -686,11 +687,13 @@
tbl.setSerdeParam(Constants.SERIALIZATION_FORMAT, crtTbl
.getFieldDelim());
}
+ if (crtTbl.getFieldEscape() != null) {
+ tbl.setSerdeParam(Constants.ESCAPE_CHAR, crtTbl.getFieldEscape());
+ }
if (crtTbl.getCollItemDelim() != null)
- tbl
- .setSerdeParam(Constants.COLLECTION_DELIM, crtTbl
- .getCollItemDelim());
+ tbl.setSerdeParam(Constants.COLLECTION_DELIM,
+ crtTbl.getCollItemDelim());
if (crtTbl.getMapKeyDelim() != null)
tbl.setSerdeParam(Constants.MAPKEY_DELIM, crtTbl.getMapKeyDelim());
if (crtTbl.getLineDelim() != null)
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java Thu Jul 2 02:03:14 2009
@@ -78,7 +78,7 @@
} else if (mainInspector.getCategory() == Category.MAP) {
MapObjectInspector moi = (MapObjectInspector)mainInspector;
Object indexObject;
- if (((PrimitiveObjectInspector)moi.getMapKeyObjectInspector()).isWritable()) {
+ if (((PrimitiveObjectInspector)moi.getMapKeyObjectInspector()).preferWritable()) {
indexObject = indexInspector.getPrimitiveWritableObject(index);
} else {
indexObject = indexInspector.getPrimitiveJavaObject(index);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Thu Jul 2 02:03:14 2009
@@ -33,11 +33,11 @@
import org.apache.hadoop.hive.ql.plan.aggregationDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
import org.apache.hadoop.hive.ql.plan.groupByDesc;
-import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -68,6 +68,7 @@
transient protected ExprNodeEvaluator[][] aggregationParameterFields;
transient protected ObjectInspector[][] aggregationParameterObjectInspectors;
+ transient protected ObjectInspector[][] aggregationParameterStandardObjectInspectors;
transient protected Object[][] aggregationParameterObjects;
// In the future, we may allow both count(DISTINCT a) and sum(DISTINCT a) in the same SQL clause,
// so aggregationIsDistinct is a boolean array instead of a single number.
@@ -99,6 +100,11 @@
transient long numRowsCompareHashAggr;
transient float minReductionHashAggr;
+ // current Key ObjectInspectors are standard ObjectInspectors
+ transient protected ObjectInspector[] currentKeyObjectInspectors;
+ // new Key ObjectInspectors are objectInspectors from the parent
+ transient StructObjectInspector newKeyObjectInspector;
+ transient StructObjectInspector currentKeyObjectInspector;
/**
* This is used to store the position and field names for variable length fields.
@@ -149,18 +155,23 @@
}
newKeys = new ArrayList<Object>(keyFields.length);
+ currentKeyObjectInspectors = new ObjectInspector[conf.getKeys().size()];
+
// init aggregationParameterFields
aggregationParameterFields = new ExprNodeEvaluator[conf.getAggregators().size()][];
aggregationParameterObjectInspectors = new ObjectInspector[conf.getAggregators().size()][];
+ aggregationParameterStandardObjectInspectors = new ObjectInspector[conf.getAggregators().size()][];
aggregationParameterObjects = new Object[conf.getAggregators().size()][];
for (int i = 0; i < aggregationParameterFields.length; i++) {
ArrayList<exprNodeDesc> parameters = conf.getAggregators().get(i).getParameters();
aggregationParameterFields[i] = new ExprNodeEvaluator[parameters.size()];
aggregationParameterObjectInspectors[i] = new ObjectInspector[parameters.size()];
+ aggregationParameterStandardObjectInspectors[i] = new ObjectInspector[parameters.size()];
aggregationParameterObjects[i] = new Object[parameters.size()];
for (int j = 0; j < parameters.size(); j++) {
aggregationParameterFields[i][j] = ExprNodeEvaluatorFactory.get(parameters.get(j));
aggregationParameterObjectInspectors[i][j] = null;
+ aggregationParameterStandardObjectInspectors[i][j] = null;
aggregationParameterObjects[i][j] = null;
}
}
@@ -248,10 +259,22 @@
for (int i = 0; i < keyFields.length; i++) {
if (keyObjectInspectors[i] == null) {
keyObjectInspectors[i] = keyFields[i].initialize(inputObjInspector[0]);
+ currentKeyObjectInspectors[i] = ObjectInspectorUtils.getStandardObjectInspector(keyObjectInspectors[i],
+ ObjectInspectorCopyOption.WRITABLE);
}
- objectInspectors.set(i, keyObjectInspectors[i]);
+ objectInspectors.set(i, currentKeyObjectInspectors[i]);
}
+ // Generate key names
+ ArrayList<String> keyNames = new ArrayList<String>(keyFields.length);
+ for (int i = 0; i < keyFields.length; i++) {
+ keyNames.add(fieldNames.get(i));
+ }
+ newKeyObjectInspector =
+ ObjectInspectorFactory.getStandardStructObjectInspector(keyNames, Arrays.asList(keyObjectInspectors));
+ currentKeyObjectInspector =
+ ObjectInspectorFactory.getStandardStructObjectInspector(keyNames, Arrays.asList(currentKeyObjectInspectors));
+
outputObjectInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, objectInspectors);
@@ -418,6 +441,8 @@
if (oi == null) {
oi = aggregationParameterFields[ai][pi].initialize(rowInspector);
aggregationParameterObjectInspectors[ai][pi] = oi;
+ aggregationParameterStandardObjectInspectors[ai][pi]
+ = ObjectInspectorUtils.getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE);
}
PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
o[pi] = poi.getPrimitiveWritableObject(
@@ -457,7 +482,7 @@
}
for (int pi=0; pi<o.length; pi++) {
lastInvoke[ai][pi] = ObjectInspectorUtils.copyToStandardObject(o[pi],
- aggregationParameterObjectInspectors[ai][pi]);
+ aggregationParameterStandardObjectInspectors[ai][pi], ObjectInspectorCopyOption.WRITABLE);
}
}
}
@@ -512,16 +537,18 @@
}
}
- private static ArrayList<Object> deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors) {
+ private static ArrayList<Object> deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors,
+ ObjectInspectorCopyOption copyOption) {
ArrayList<Object> result = new ArrayList<Object>(keys.length);
- deepCopyElements(keys, keyObjectInspectors, result);
+ deepCopyElements(keys, keyObjectInspectors, result, copyOption);
return result;
}
- private static void deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors, ArrayList<Object> result) {
+ private static void deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors, ArrayList<Object> result,
+ ObjectInspectorCopyOption copyOption) {
result.clear();
for (int i=0; i<keys.length; i++) {
- result.add(ObjectInspectorUtils.copyToStandardObject(keys[i], keyObjectInspectors[i]));
+ result.add(ObjectInspectorUtils.copyToStandardObject(keys[i], keyObjectInspectors[i], copyOption));
}
}
@@ -531,10 +558,11 @@
boolean newEntry = false;
// hash-based aggregations
- aggs = hashAggregations.get(newKeys);
+ ArrayList<Object> newDefaultKeys = deepCopyElements(keyObjects, keyObjectInspectors, ObjectInspectorCopyOption.WRITABLE);
+ aggs = hashAggregations.get(newDefaultKeys);
if (aggs == null) {
aggs = newAggregations();
- hashAggregations.put(deepCopyElements(keyObjects, keyObjectInspectors), aggs);
+ hashAggregations.put(newDefaultKeys, aggs);
newEntry = true;
numRowsHashTbl++; // new entry in the hash table
}
@@ -552,7 +580,9 @@
// Prepare aggs for updating
UDAFEvaluator[] aggs = null;
Object[][] lastInvoke = null;
- boolean keysAreEqual = newKeys.equals(currentKeys);
+ boolean keysAreEqual = ObjectInspectorUtils.compare(
+ newKeys, newKeyObjectInspector,
+ currentKeys, currentKeyObjectInspector) == 0;
// Forward the current keys if needed for sort-based aggregation
if (currentKeys != null && !keysAreEqual)
@@ -563,7 +593,7 @@
if (currentKeys == null) {
currentKeys = new ArrayList<Object>(keyFields.length);
}
- deepCopyElements(keyObjects, keyObjectInspectors, currentKeys);
+ deepCopyElements(keyObjects, keyObjectInspectors, currentKeys, ObjectInspectorCopyOption.WRITABLE);
// Init aggregations
for(UDAFEvaluator aggregation: aggregations)
@@ -596,7 +626,7 @@
if (key != null) {
if (key instanceof String) {
totalVariableSize += ((String)key).length();
- } else {
+ } else if (key instanceof Text) {
totalVariableSize += ((Text)key).getLength();
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Thu Jul 2 02:03:14 2009
@@ -155,6 +155,7 @@
List<Order> sortCols = null;
int numBuckets = -1;
String fieldDelim = null;
+ String fieldEscape = null;
String collItemDelim = null;
String mapKeyDelim = null;
String lineDelim = null;
@@ -216,6 +217,9 @@
switch (rowChild.getToken().getType()) {
case HiveParser.TOK_TABLEROWFORMATFIELD:
fieldDelim = unescapeSQLString(rowChild.getChild(0).getText());
+ if (rowChild.getChildCount()>=2) {
+ fieldEscape = unescapeSQLString(rowChild.getChild(1).getText());
+ }
break;
case HiveParser.TOK_TABLEROWFORMATCOLLITEMS:
collItemDelim = unescapeSQLString(rowChild.getChild(0).getText());
@@ -269,7 +273,8 @@
createTableDesc crtTblDesc =
new createTableDesc(tableName, isExt, cols, partCols, bucketCols,
sortCols, numBuckets,
- fieldDelim, collItemDelim, mapKeyDelim, lineDelim,
+ fieldDelim, fieldEscape,
+ collItemDelim, mapKeyDelim, lineDelim,
comment, inputFormat, outputFormat, location, serde,
mapProp, ifNotExists);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Thu Jul 2 02:03:14 2009
@@ -383,8 +383,8 @@
@init { msgs.push("table row format's field separator"); }
@after { msgs.pop(); }
:
- KW_FIELDS KW_TERMINATED KW_BY fldIdnt=StringLiteral
- -> ^(TOK_TABLEROWFORMATFIELD $fldIdnt)
+ KW_FIELDS KW_TERMINATED KW_BY fldIdnt=StringLiteral (KW_ESCAPED KW_BY fldEscape=StringLiteral)?
+ -> ^(TOK_TABLEROWFORMATFIELD $fldIdnt $fldEscape?)
;
tableRowFormatCollItemsIdentifier
@@ -1185,6 +1185,7 @@
KW_DELIMITED: 'DELIMITED';
KW_FIELDS: 'FIELDS';
KW_TERMINATED: 'TERMINATED';
+KW_ESCAPED: 'ESCAPED';
KW_COLLECTION: 'COLLECTION';
KW_ITEMS: 'ITEMS';
KW_KEYS: 'KEYS';
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java Thu Jul 2 02:03:14 2009
@@ -38,6 +38,7 @@
List<Order> sortCols;
int numBuckets;
String fieldDelim;
+ String fieldEscape;
String collItemDelim;
String mapKeyDelim;
String lineDelim;
@@ -53,6 +54,7 @@
List<FieldSchema> cols, List<FieldSchema> partCols,
List<String> bucketCols, List<Order> sortCols,
int numBuckets, String fieldDelim,
+ String fieldEscape,
String collItemDelim,
String mapKeyDelim, String lineDelim,
String comment, String inputFormat, String outputFormat,
@@ -66,6 +68,7 @@
this.cols = cols;
this.comment = comment;
this.fieldDelim = fieldDelim;
+ this.fieldEscape = fieldEscape;
this.inputFormat = inputFormat;
this.outputFormat = outputFormat;
this.lineDelim = lineDelim;
@@ -149,6 +152,15 @@
this.fieldDelim = fieldDelim;
}
+ @explain(displayName="field escape")
+ public String getFieldEscape() {
+ return fieldEscape;
+ }
+
+ public void setFieldEscape(String fieldEscape) {
+ this.fieldEscape = fieldEscape;
+ }
+
@explain(displayName="collection delimiter")
public String getCollItemDelim() {
return collItemDelim;
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Thu Jul 2 02:03:14 2009
@@ -38,8 +38,10 @@
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -176,10 +178,12 @@
assertEquals(8, fieldRefs.size());
for (int j = 0; j < fieldRefs.size(); j++) {
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(j));
+ Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData,
+ fieldRefs.get(j).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE);
if (i == 0)
- assertEquals("Field " + i, fieldData, expectedRecord_1[j]);
+ assertEquals("Field " + i, standardWritableData, expectedRecord_1[j]);
else
- assertEquals("Field " + i, fieldData, expectedRecord_2[j]);
+ assertEquals("Field " + i, standardWritableData, expectedRecord_2[j]);
}
}
@@ -300,7 +304,9 @@
assertEquals(8, fieldRefs.size());
for (int i = 0; i < fieldRefs.size(); i++) {
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
- assertEquals("Field " + i, fieldData, expectedFieldsData[i]);
+ Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData,
+ fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE);
+ assertEquals("Field " + i, standardWritableData, expectedFieldsData[i]);
}
// Serialize
assertEquals(BytesRefArrayWritable.class, serDe.getSerializedClass());
@@ -336,7 +342,9 @@
for (int i = 0; i < fieldRefs.size(); i++) {
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
- assertEquals("Field " + i, fieldData, expectedPartitalFieldsData[i]);
+ Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData,
+ fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE);
+ assertEquals("Field " + i, standardWritableData, expectedPartitalFieldsData[i]);
}
assertEquals(BytesRefArrayWritable.class, serDe.getSerializedClass());
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q?rev=790435&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q Thu Jul 2 02:03:14 2009
@@ -0,0 +1,16 @@
+DROP TABLE table1;
+DROP TABLE table2;
+DROP TABLE table3;
+
+CREATE TABLE table1 (a STRING, b STRING)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\\'
+STORED AS TEXTFILE;
+
+DESCRIBE table1;
+DESCRIBE EXTENDED table1;
+
+INSERT OVERWRITE TABLE table1 SELECT key, '\\\t\\' FROM src WHERE key = 86;
+
+SELECT * FROM table1;
+
+DROP TABLE table1;
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out?rev=790435&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out Thu Jul 2 02:03:14 2009
@@ -0,0 +1,22 @@
+query: DROP TABLE table1
+query: DROP TABLE table2
+query: DROP TABLE table3
+query: CREATE TABLE table1 (a STRING, b STRING)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\\'
+STORED AS TEXTFILE
+query: DESCRIBE table1
+a string
+b string
+query: DESCRIBE EXTENDED table1
+a string
+b string
+
+Detailed Table Information Table(tableName:table1,dbName:default,owner:zshao,createTime:1245330503,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:string,comment:null), FieldSchema(name:b,type:string,comment:null)],location:file:/data/users/zshao/tools/136-trunk-apache-hive/build/ql/test/data/warehouse/table1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,parameters:{serialization.format=9,escape.delim=\,field.delim= }),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{})
+query: INSERT OVERWRITE TABLE table1 SELECT key, '\\\t\\' FROM src WHERE key = 86
+Input: default/src
+Output: default/table1
+query: SELECT * FROM table1
+Input: default/table1
+Output: file:/data/users/zshao/tools/136-trunk-apache-hive/build/ql/tmp/822936101/10000
+86 \ \
+query: DROP TABLE table1
Modified: hadoop/hive/trunk/serde/if/serde.thrift
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/if/serde.thrift?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/if/serde.thrift (original)
+++ hadoop/hive/trunk/serde/if/serde.thrift Thu Jul 2 02:03:14 2009
@@ -17,6 +17,7 @@
const string LINE_DELIM = "line.delim"
const string MAPKEY_DELIM = "mapkey.delim"
const string QUOTE_CHAR = "quote.delim"
+const string ESCAPE_CHAR = "escape.delim"
typedef string PrimitiveType
typedef string CollectionType
Modified: hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (original)
+++ hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java Thu Jul 2 02:03:14 2009
@@ -39,6 +39,8 @@
public static final String QUOTE_CHAR = "quote.delim";
+ public static final String ESCAPE_CHAR = "escape.delim";
+
public static final String VOID_TYPE_NAME = "void";
public static final String BOOLEAN_TYPE_NAME = "boolean";
Modified: hadoop/hive/trunk/serde/src/gen-php/serde_constants.php
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-php/serde_constants.php?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-php/serde_constants.php (original)
+++ hadoop/hive/trunk/serde/src/gen-php/serde_constants.php Thu Jul 2 02:03:14 2009
@@ -32,6 +32,8 @@
$GLOBALS['serde_CONSTANTS']['QUOTE_CHAR'] = 'quote.delim';
+$GLOBALS['serde_CONSTANTS']['ESCAPE_CHAR'] = 'escape.delim';
+
$GLOBALS['serde_CONSTANTS']['VOID_TYPE_NAME'] = 'void';
$GLOBALS['serde_CONSTANTS']['BOOLEAN_TYPE_NAME'] = 'boolean';
Modified: hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py (original)
+++ hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py Thu Jul 2 02:03:14 2009
@@ -31,6 +31,8 @@
QUOTE_CHAR = 'quote.delim'
+ESCAPE_CHAR = 'escape.delim'
+
VOID_TYPE_NAME = 'void'
BOOLEAN_TYPE_NAME = 'boolean'
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java Thu Jul 2 02:03:14 2009
@@ -25,9 +25,18 @@
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.io.Text;
public class SerDeUtils {
@@ -96,6 +105,9 @@
return true;
}
+ /**
+ * Escape a String in JSON format.
+ */
public static String escapeString(String str) {
int length = str.length();
StringBuilder escape = new StringBuilder(length + 16);
@@ -181,7 +193,7 @@
return sb.toString();
}
-
+
static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) {
switch(oi.getCategory()) {
@@ -189,17 +201,46 @@
PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
if (o == null) {
sb.append("null");
- } else if (oi.getTypeName().equals(Constants.STRING_TYPE_NAME)) {
- String s = (String)poi.getPrimitiveJavaObject(o);
- sb.append(QUOTE);
- sb.append(escapeString(s));
- sb.append(QUOTE);
- } else if (oi.getTypeName().equals(Constants.BOOLEAN_TYPE_NAME)) {
- Boolean b = (Boolean)poi.getPrimitiveJavaObject(o);
- sb.append(b.booleanValue() ? "True" : "False");
} else {
- // it's a number - so doesn't need to be escaped.
- sb.append(o.toString());
+ switch (poi.getPrimitiveCategory()) {
+ case BOOLEAN: {
+ boolean b = ((BooleanObjectInspector)poi).get(o);
+ sb.append(b ? "true" : "false");
+ break;
+ }
+ case BYTE: {
+ sb.append(((ByteObjectInspector)poi).get(o));
+ break;
+ }
+ case SHORT: {
+ sb.append(((ShortObjectInspector)poi).get(o));
+ break;
+ }
+ case INT: {
+ sb.append(((IntObjectInspector)poi).get(o));
+ break;
+ }
+ case LONG: {
+ sb.append(((LongObjectInspector)poi).get(o));
+ break;
+ }
+ case FLOAT: {
+ sb.append(((FloatObjectInspector)poi).get(o));
+ break;
+ }
+ case DOUBLE: {
+ sb.append(((DoubleObjectInspector)poi).get(o));
+ break;
+ }
+ case STRING: {
+ sb.append('"');
+ sb.append(escapeString(((StringObjectInspector)poi).getPrimitiveJavaObject(o)));
+ sb.append('"');
+ break;
+ }
+ default:
+ throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
+ }
}
break;
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java Thu Jul 2 02:03:14 2009
@@ -84,13 +84,15 @@
serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, getClass()
.getName());
- cachedLazyStruct = new ColumnarStruct(serdeParams.getRowTypeInfo());
// Create the ObjectInspectors for the fields. Note: Currently
// ColumnarObject uses same ObjectInpector as LazyStruct
cachedObjectInspector = LazyFactory.createColumnarStructInspector(
serdeParams.getColumnNames(), serdeParams.getColumnTypes(), serdeParams
- .getSeparators(), serdeParams.getNullSequence());
+ .getSeparators(), serdeParams.getNullSequence(), serdeParams.isEscaped(),
+ serdeParams.getEscapeChar());
+ cachedLazyStruct = new ColumnarStruct(cachedObjectInspector);
+
int size = serdeParams.getColumnTypes().size();
field = new BytesRefWritable[size];
for (int i = 0; i < size; i++) {
@@ -198,13 +200,17 @@
&& (declaredFields == null || declaredFields.get(i)
.getFieldObjectInspector().getCategory().equals(
Category.PRIMITIVE))) {
- LazySimpleSerDe.serialize(serializeStream, SerDeUtils.getJSONString(
- f, foi),
+ LazySimpleSerDe.serialize(serializeStream,
+ SerDeUtils.getJSONString(f, foi),
PrimitiveObjectInspectorFactory.javaStringObjectInspector,
- serdeParams.getSeparators(), 1, serdeParams.getNullSequence());
+ serdeParams.getSeparators(), 1, serdeParams.getNullSequence(),
+ serdeParams.isEscaped(), serdeParams.getEscapeChar(),
+ serdeParams.getNeedsEscape());
} else {
- LazySimpleSerDe.serialize(serializeStream, f, foi, serdeParams
- .getSeparators(), 1, serdeParams.getNullSequence());
+ LazySimpleSerDe.serialize(serializeStream, f, foi,
+ serdeParams.getSeparators(), 1, serdeParams.getNullSequence(),
+ serdeParams.isEscaped(), serdeParams.getEscapeChar(),
+ serdeParams.getNeedsEscape());
}
field[i].set(serializeStream.getData(), count, serializeStream
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java Thu Jul 2 02:03:14 2009
@@ -25,8 +25,9 @@
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
import org.apache.hadoop.hive.serde2.lazy.LazyObject;
import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.Text;
/**
@@ -39,17 +40,6 @@
public class ColumnarStruct {
/**
- * The TypeInfo for this LazyNonPrimitive.
- */
- TypeInfo typeInfo;
- List<TypeInfo> fieldTypeInfos;
-
- /**
- * Whether the data is already parsed or not.
- */
- boolean parsed;
-
- /**
* The fields of the struct.
*/
LazyObject[] fields;
@@ -58,18 +48,17 @@
* Construct a ColumnarStruct object with the TypeInfo. It creates the first
* level object at the first place
*
- * @param typeInfo
- * the TypeInfo representing the type of this LazyStruct.
+ * @param oi
+ * the ObjectInspector representing the type of this LazyStruct.
*/
- public ColumnarStruct(TypeInfo typeInfo) {
- this.typeInfo = typeInfo;
- fieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
- int num = fieldTypeInfos.size();
+ public ColumnarStruct(ObjectInspector oi) {
+ List<? extends StructField> fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs();
+ int num = fieldRefs.size();
fields = new LazyObject[num];
cachedByteArrayRef = new ByteArrayRef[num];
fieldIsNull = new boolean[num];
for (int i = 0; i < num; i++) {
- fields[i] = LazyFactory.createLazyObject(fieldTypeInfos.get(i));
+ fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector());
cachedByteArrayRef[i] = new ByteArrayRef();
fieldIsNull[i] = false;
}
@@ -147,8 +136,6 @@
}
for (; fieldIndex < fields.length; fieldIndex++)
fieldIsNull[fieldIndex] = true;
-
- parsed = true;
}
ArrayList<Object> cachedList;
@@ -171,4 +158,4 @@
}
return cachedList;
}
-}
\ No newline at end of file
+}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java Thu Jul 2 02:03:14 2009
@@ -21,17 +21,17 @@
import java.util.Arrays;
import java.util.List;
-import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.io.Text;
/**
* LazyArray stores an array of Lazy Objects.
*
* LazyArray does not deal with the case of a NULL array. That is handled
- * by LazyArrayObjectInspector.
+ * by the parent LazyObject.
*/
-public class LazyArray extends LazyNonPrimitive {
+public class LazyArray extends LazyNonPrimitive<LazyListObjectInspector> {
/**
* Whether the data is already parsed or not.
@@ -65,11 +65,12 @@
LazyObject[] arrayElements;
/**
- * Construct a LazyArray object with the TypeInfo.
- * @param typeInfo the TypeInfo representing the type of this LazyArray.
+ * Construct a LazyArray object with the ObjectInspector.
+ * @param oi the oi representing the type of this LazyArray as well as meta
+ * information like separator etc.
*/
- protected LazyArray(TypeInfo typeInfo) {
- super(typeInfo);
+ protected LazyArray(LazyListObjectInspector oi) {
+ super(oi);
}
/**
@@ -86,7 +87,7 @@
* Enlarge the size of arrays storing information for the elements inside
* the array.
*/
- protected void enlargeArrays() {
+ private void enlargeArrays() {
if (startPosition == null) {
int initialSize = 2;
startPosition = new int[initialSize];
@@ -102,9 +103,13 @@
/**
* Parse the bytes and fill arrayLength and startPosition.
*/
- private void parse(byte separator, Text nullSequence) {
+ private void parse() {
parsed = true;
+ byte separator = oi.getSeparator();
+ boolean isEscaped = oi.isEscaped();
+ byte escapeChar = oi.getEscapeChar();
+
// empty array?
if (length == 0) {
arrayLength = 0;
@@ -131,7 +136,14 @@
arrayLength++;
elementByteBegin = elementByteEnd + 1;
}
- elementByteEnd++;
+ if (isEscaped && bytes[elementByteEnd] == escapeChar
+ && elementByteEnd+1 < arrayByteEnd) {
+ // ignore the char after escape_char
+ elementByteEnd += 2;
+ } else {
+ elementByteEnd ++;
+ }
+
}
// Store arrayByteEnd+1 in startPosition[arrayLength]
// so that we can use the same formula to compute the length of
@@ -148,21 +160,22 @@
* Returns the actual primitive object at the index position
* inside the array represented by this LazyObject.
*/
- public Object getListElementObject(int index, byte separator,
- Text nullSequence) {
+ public Object getListElementObject(int index) {
if (!parsed) {
- parse(separator, nullSequence);
+ parse();
}
if (index < 0 || index >= arrayLength) {
return null;
}
- return uncheckedGetElement(index, nullSequence);
+ return uncheckedGetElement(index);
}
/**
- * Get the element without checking parsed or out-of-bound index.
+ * Get the element without checking out-of-bound index.
*/
- private Object uncheckedGetElement(int index, Text nullSequence) {
+ private Object uncheckedGetElement(int index) {
+ Text nullSequence = oi.getNullSequence();
+
int elementLength = startPosition[index+1] - startPosition[index] - 1;
if (elementLength == nullSequence.getLength()
&& 0 == LazyUtils.compare(bytes.getData(), startPosition[index],
@@ -174,7 +187,7 @@
elementInited[index] = true;
if (arrayElements[index] == null) {
arrayElements[index] = LazyFactory.createLazyObject(
- ((ListTypeInfo)typeInfo).getListElementTypeInfo());
+ ((ListObjectInspector)oi).getListElementObjectInspector());
}
arrayElements[index].init(bytes, startPosition[index],
elementLength);
@@ -185,9 +198,9 @@
/** Returns -1 for null array.
*/
- public int getListLength(byte separator, Text nullSequence) {
+ public int getListLength() {
if (!parsed) {
- parse(separator, nullSequence);
+ parse();
}
return arrayLength;
}
@@ -200,9 +213,9 @@
/** Returns the List of actual primitive objects.
* Returns null for null array.
*/
- public List<Object> getList(byte separator, Text nullSequence) {
+ public List<Object> getList() {
if (!parsed) {
- parse(separator, nullSequence);
+ parse();
}
if (arrayLength == -1) {
return null;
@@ -213,7 +226,7 @@
cachedList.clear();
}
for (int index=0; index<arrayLength; index++) {
- cachedList.add(uncheckedGetElement(index, nullSequence));
+ cachedList.add(uncheckedGetElement(index));
}
return cachedList;
}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java?rev=790435&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java Thu Jul 2 02:03:14 2009
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector;
+import org.apache.hadoop.io.BooleanWritable;
+
+/**
+ * LazyObject for storing a value of boolean.
+ *
+ * <p>
+ * Part of the code is adapted from Apache Harmony Project.
+ *
+ * As with the specification, this implementation relied on code laid out in <a
+ * href="http://www.hackersdelight.org/">Henry S. Warren, Jr.'s Hacker's
+ * Delight, (Addison Wesley, 2002)</a> as well as <a
+ * href="http://aggregate.org/MAGIC/">The Aggregate's Magic Algorithms</a>.
+ * </p>
+ *
+ */
+public class LazyBoolean extends LazyPrimitive<LazyBooleanObjectInspector, BooleanWritable> {
+
+ public LazyBoolean(LazyBooleanObjectInspector oi) {
+ super(oi);
+ data = new BooleanWritable();
+ }
+
+ public LazyBoolean(LazyBoolean copy) {
+ super(copy);
+ data = new BooleanWritable(copy.data.get());
+ }
+
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ if (length == 4
+ && Character.toUpperCase(bytes.getData()[start]) == 'T'
+ && Character.toUpperCase(bytes.getData()[start+1]) == 'R'
+ && Character.toUpperCase(bytes.getData()[start+2]) == 'U'
+ && Character.toUpperCase(bytes.getData()[start+3]) == 'E') {
+ data.set(true);
+ isNull = false;
+ } else if (length == 5
+ && Character.toUpperCase(bytes.getData()[start]) == 'F'
+ && Character.toUpperCase(bytes.getData()[start+1]) == 'A'
+ && Character.toUpperCase(bytes.getData()[start+2]) == 'L'
+ && Character.toUpperCase(bytes.getData()[start+3]) == 'S'
+ && Character.toUpperCase(bytes.getData()[start+4]) == 'E') {
+ data.set(false);
+ isNull = false;
+ } else {
+ isNull = true;
+ }
+ }
+
+
+}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java Thu Jul 2 02:03:14 2009
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.serde2.lazy;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector;
/**
* LazyObject for storing a value of Byte.
@@ -32,12 +33,18 @@
* </p>
*
*/
-public class LazyByte extends LazyPrimitive<ByteWritable> {
+public class LazyByte extends LazyPrimitive<LazyByteObjectInspector, ByteWritable> {
- public LazyByte() {
+ public LazyByte(LazyByteObjectInspector oi) {
+ super(oi);
data = new ByteWritable();
}
+ public LazyByte(LazyByte copy) {
+ super(copy);
+ data = new ByteWritable(copy.data.get());
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java Thu Jul 2 02:03:14 2009
@@ -20,17 +20,24 @@
import java.nio.charset.CharacterCodingException;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector;
import org.apache.hadoop.io.Text;
/**
* LazyObject for storing a value of Double.
*
*/
-public class LazyDouble extends LazyPrimitive<DoubleWritable> {
+public class LazyDouble extends LazyPrimitive<LazyDoubleObjectInspector, DoubleWritable> {
- public LazyDouble() {
+ public LazyDouble(LazyDoubleObjectInspector oi) {
+ super(oi);
data = new DoubleWritable();
}
+
+ public LazyDouble(LazyDouble copy) {
+ super(copy);
+ data = new DoubleWritable(copy.data.get());
+ }
@Override
public void init(ByteArrayRef bytes, int start, int length) {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java Thu Jul 2 02:03:14 2009
@@ -20,11 +20,32 @@
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyListObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyIntObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyLongObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
@@ -37,29 +58,32 @@
/**
* Create a lazy primitive class given the type name.
*/
- public static LazyPrimitive<?> createLazyPrimitiveClass(String typeName) {
- PrimitiveCategory p = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveCategory;
+ public static LazyPrimitive<?,?> createLazyPrimitiveClass(PrimitiveObjectInspector oi) {
+ PrimitiveCategory p = oi.getPrimitiveCategory();
switch(p) {
+ case BOOLEAN: {
+ return new LazyBoolean((LazyBooleanObjectInspector)oi);
+ }
case BYTE: {
- return new LazyByte();
+ return new LazyByte((LazyByteObjectInspector)oi);
}
case SHORT: {
- return new LazyShort();
+ return new LazyShort((LazyShortObjectInspector)oi);
}
case INT: {
- return new LazyInteger();
+ return new LazyInteger((LazyIntObjectInspector)oi);
}
case LONG: {
- return new LazyLong();
+ return new LazyLong((LazyLongObjectInspector)oi);
}
case FLOAT: {
- return new LazyFloat();
+ return new LazyFloat((LazyFloatObjectInspector)oi);
}
case DOUBLE: {
- return new LazyDouble();
+ return new LazyDouble((LazyDoubleObjectInspector)oi);
}
case STRING: {
- return new LazyString();
+ return new LazyString((LazyStringObjectInspector)oi);
}
default: {
throw new RuntimeException("Internal error: no LazyObject for " + p);
@@ -70,17 +94,17 @@
/**
* Create a hierarchical LazyObject based on the given typeInfo.
*/
- public static LazyObject createLazyObject(TypeInfo typeInfo) {
- ObjectInspector.Category c = typeInfo.getCategory();
+ public static LazyObject createLazyObject(ObjectInspector oi) {
+ ObjectInspector.Category c = oi.getCategory();
switch(c) {
case PRIMITIVE:
- return createLazyPrimitiveClass(typeInfo.getTypeName());
+ return createLazyPrimitiveClass((PrimitiveObjectInspector)oi);
case MAP:
- return new LazyMap(typeInfo);
+ return new LazyMap((LazyMapObjectInspector)oi);
case LIST:
- return new LazyArray(typeInfo);
+ return new LazyArray((LazyListObjectInspector)oi);
case STRUCT:
- return new LazyStruct(typeInfo);
+ return new LazyStruct((LazySimpleStructObjectInspector)oi);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
@@ -99,24 +123,27 @@
* @return The ObjectInspector
*/
public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator,
- int separatorIndex, Text nullSequence) {
+ int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar) {
ObjectInspector.Category c = typeInfo.getCategory();
switch(c) {
case PRIMITIVE:
- return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
- ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory());
+ return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(
+ ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory(), escaped, escapeChar);
case MAP:
- return ObjectInspectorFactory.getLazySimpleMapObjectInspector(
- createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapKeyTypeInfo(), separator, separatorIndex+2, nullSequence),
- createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapValueTypeInfo(), separator, separatorIndex+2, nullSequence),
+ return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(
+ createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapKeyTypeInfo(),
+ separator, separatorIndex+2, nullSequence, escaped, escapeChar),
+ createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapValueTypeInfo(),
+ separator, separatorIndex+2, nullSequence, escaped, escapeChar),
separator[separatorIndex],
separator[separatorIndex+1],
- nullSequence);
+ nullSequence, escaped, escapeChar);
case LIST:
- return ObjectInspectorFactory.getLazySimpleListObjectInspector(
- createLazyObjectInspector(((ListTypeInfo)typeInfo).getListElementTypeInfo(), separator, separatorIndex+1, nullSequence),
+ return LazyObjectInspectorFactory.getLazySimpleListObjectInspector(
+ createLazyObjectInspector(((ListTypeInfo)typeInfo).getListElementTypeInfo(),
+ separator, separatorIndex+1, nullSequence, escaped, escapeChar),
separator[separatorIndex],
- nullSequence);
+ nullSequence, escaped, escapeChar);
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;
List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
@@ -124,14 +151,15 @@
List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
for(int i=0; i<fieldTypeInfos.size(); i++) {
fieldObjectInspectors.add(
- createLazyObjectInspector(fieldTypeInfos.get(i), separator, separatorIndex+1, nullSequence));
+ createLazyObjectInspector(fieldTypeInfos.get(i), separator,
+ separatorIndex+1, nullSequence, escaped, escapeChar));
}
- return ObjectInspectorFactory.getLazySimpleStructObjectInspector(
+ return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
fieldNames,
fieldObjectInspectors,
separator[separatorIndex],
nullSequence,
- false);
+ false, escaped, escapeChar);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
@@ -143,36 +171,36 @@
*
* @param lastColumnTakesRest whether the last column of the struct should take
* the rest of the row if there are extra fields.
- * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text)
+ * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
*/
public static ObjectInspector createLazyStructInspector(List<String> columnNames,
List<TypeInfo> typeInfos, byte[] separators,
- Text nullSequence, boolean lastColumnTakesRest) {
+ Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
ArrayList<ObjectInspector> columnObjectInspectors =
new ArrayList<ObjectInspector>(typeInfos.size());
for (int i=0; i<typeInfos.size(); i++) {
columnObjectInspectors.add(
- LazyFactory.createLazyObjectInspector(typeInfos.get(i), separators, 1, nullSequence));
+ LazyFactory.createLazyObjectInspector(typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar));
}
return
- ObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames,
- columnObjectInspectors, separators[0], nullSequence, lastColumnTakesRest);
+ LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames,
+ columnObjectInspectors, separators[0], nullSequence, lastColumnTakesRest, escaped, escapeChar);
}
/**
* Create a hierarchical ObjectInspector for ColumnarStruct with the given
* columnNames and columnTypeInfos.
*
- * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text)
+ * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
*/
public static ObjectInspector createColumnarStructInspector(
List<String> columnNames, List<TypeInfo> columnTypes, byte[] separators,
- Text nullSequence) {
+ Text nullSequence, boolean escaped, byte escapeChar) {
ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>(
columnTypes.size());
for (int i = 0; i < columnTypes.size(); i++) {
columnObjectInspectors.add(LazyFactory.createLazyObjectInspector(
- columnTypes.get(i), separators, 1, nullSequence));
+ columnTypes.get(i), separators, 1, nullSequence, escaped, escapeChar));
}
return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames,
columnObjectInspectors, nullSequence);
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java Thu Jul 2 02:03:14 2009
@@ -19,6 +19,7 @@
import java.nio.charset.CharacterCodingException;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.Text;
@@ -26,12 +27,17 @@
* LazyObject for storing a value of Double.
*
*/
-public class LazyFloat extends LazyPrimitive<FloatWritable> {
+public class LazyFloat extends LazyPrimitive<LazyFloatObjectInspector, FloatWritable> {
- public LazyFloat() {
+ public LazyFloat(LazyFloatObjectInspector oi) {
+ super(oi);
data = new FloatWritable();
}
+ public LazyFloat(LazyFloat copy) {
+ super(copy);
+ data = new FloatWritable(copy.data.get());
+ }
@Override
public void init(ByteArrayRef bytes, int start, int length) {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java Thu Jul 2 02:03:14 2009
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.OutputStream;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyIntObjectInspector;
import org.apache.hadoop.io.IntWritable;
/**
@@ -35,12 +36,19 @@
* </p>
*
*/
-public class LazyInteger extends LazyPrimitive<IntWritable> {
+public class LazyInteger extends LazyPrimitive<LazyIntObjectInspector, IntWritable> {
- public LazyInteger() {
+ public LazyInteger(LazyIntObjectInspector oi) {
+ super(oi);
data = new IntWritable();
}
+ public LazyInteger(LazyInteger copy) {
+ super(copy);
+ data = new IntWritable(copy.data.get());
+ }
+
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java Thu Jul 2 02:03:14 2009
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.OutputStream;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyLongObjectInspector;
import org.apache.hadoop.io.LongWritable;
/**
@@ -35,13 +36,19 @@
* </p>
*
*/
-public class LazyLong extends LazyPrimitive<LongWritable> {
+public class LazyLong extends LazyPrimitive<LazyLongObjectInspector, LongWritable> {
- public LazyLong() {
+ public LazyLong(LazyLongObjectInspector oi) {
+ super(oi);
data = new LongWritable();
}
+ public LazyLong(LazyLong copy) {
+ super(copy);
+ data = new LongWritable(copy.data.get());
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Thu Jul 2 02:03:14 2009
@@ -19,10 +19,12 @@
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
-import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.io.Text;
/**
@@ -30,9 +32,9 @@
* Note that the keys of the map cannot contain null.
*
* LazyMap does not deal with the case of a NULL map. That is handled
- * by LazyMapObjectInspector.
+ * by the parent LazyObject.
*/
-public class LazyMap extends LazyNonPrimitive {
+public class LazyMap extends LazyNonPrimitive<LazyMapObjectInspector> {
/**
* Whether the data is already parsed or not.
@@ -63,7 +65,7 @@
/**
* The keys are stored in an array of LazyPrimitives.
*/
- LazyPrimitive<?>[] keyObjects;
+ LazyPrimitive<?,?>[] keyObjects;
/**
* Whether init() is called on keyObjects[i].
*/
@@ -80,11 +82,10 @@
boolean[] valueInited;
/**
- * Construct a LazyMap object with the TypeInfo.
- * @param typeInfo the TypeInfo representing the type of this LazyMap.
+ * Construct a LazyMap object with the ObjectInspector.
*/
- protected LazyMap(TypeInfo typeInfo) {
- super(typeInfo);
+ protected LazyMap(LazyMapObjectInspector oi) {
+ super(oi);
}
/**
@@ -106,7 +107,7 @@
int initialSize = 2;
keyStart = new int[initialSize];
keyEnd = new int[initialSize];
- keyObjects = new LazyPrimitive<?>[initialSize];
+ keyObjects = new LazyPrimitive<?,?>[initialSize];
valueObjects = new LazyObject[initialSize];
keyInited = new boolean[initialSize];
valueInited = new boolean[initialSize];
@@ -122,14 +123,15 @@
/**
* Parse the byte[] and fill keyStart, keyEnd.
- * @param itemSeparator The separator between different entries.
- * @param keyValueSeparator The separator between key and value.
- * @param nullSequence The byte sequence representing NULL.
*/
- private void parse(byte itemSeparator, byte keyValueSeparator,
- Text nullSequence) {
+ private void parse() {
parsed = true;
+ byte itemSeparator = oi.getItemSeparator();
+ byte keyValueSeparator = oi.getKeyValueSeparator();
+ boolean isEscaped = oi.isEscaped();
+ byte escapeChar = oi.getEscapeChar();
+
// empty array?
if (length == 0) {
mapSize = 0;
@@ -167,7 +169,13 @@
&& bytes[elementByteEnd] == keyValueSeparator) {
keyValueSeparatorPosition = elementByteEnd;
}
- elementByteEnd++;
+ if (isEscaped && bytes[elementByteEnd] == escapeChar
+ && elementByteEnd+1 < arrayByteEnd) {
+ // ignore the char after escape_char
+ elementByteEnd += 2;
+ } else {
+ elementByteEnd ++;
+ }
}
// This makes sure we can use the same formula to compute the
@@ -191,28 +199,23 @@
* most cases, user only wants to get one or two values out of the map, and
* the cost of building up a HashMap is substantially higher.
*
- * @param itemSeparator The separator between different entries.
- * @param keyValueSeparator The separator between key and value.
- * @param nullSequence The byte sequence representing NULL.
* @param key The key object that we are looking for.
* @return The corresponding value object, or NULL if not found
*/
- public Object getMapValueElement(byte itemSeparator, byte keyValueSeparator,
- Text nullSequence, Object key) {
+ public Object getMapValueElement(Object key) {
if (!parsed) {
- parse(itemSeparator, keyValueSeparator, nullSequence);
+ parse();
}
-
// search for the key
for (int i=0; i<mapSize; i++) {
- LazyPrimitive<?> lazyKeyI = uncheckedGetKey(i, nullSequence);
+ LazyPrimitive<?,?> lazyKeyI = uncheckedGetKey(i);
if (lazyKeyI == null) continue;
- // getObject() will convert LazyPrimitive to actual primitive objects.
- Object keyI = lazyKeyI.getObject();
+ // getWritableObject() will convert LazyPrimitive to actual primitive writable objects.
+ Object keyI = lazyKeyI.getWritableObject();
if (keyI == null) continue;
if (keyI.equals(key)) {
// Got a match, return the value
- LazyObject v = uncheckedGetValue(i, nullSequence);
+ LazyObject v = uncheckedGetValue(i);
return v == null ? v : v.getObject();
}
}
@@ -225,7 +228,8 @@
* @param index The index into the array starting from 0
* @param nullSequence The byte sequence representing the NULL value
*/
- private LazyObject uncheckedGetValue(int index, Text nullSequence) {
+ private LazyObject uncheckedGetValue(int index) {
+ Text nullSequence = oi.getNullSequence();
int valueIBegin = keyEnd[index] + 1;
int valueILength = keyStart[index+1] - 1 - valueIBegin;
if (valueILength < 0 ||
@@ -238,7 +242,7 @@
valueInited[index] = true;
if (valueObjects[index] == null) {
valueObjects[index] = LazyFactory.createLazyObject(
- ((MapTypeInfo)typeInfo).getMapValueTypeInfo());
+ ((MapObjectInspector)oi).getMapValueObjectInspector());
}
valueObjects[index].init(bytes, valueIBegin, valueILength);
}
@@ -250,7 +254,8 @@
* @param index The index into the array starting from 0
* @param nullSequence The byte sequence representing the NULL value
*/
- private LazyPrimitive<?> uncheckedGetKey(int index, Text nullSequence) {
+ private LazyPrimitive<?,?> uncheckedGetKey(int index) {
+ Text nullSequence = oi.getNullSequence();
int keyIBegin = keyStart[index];
int keyILength = keyEnd[index] - keyStart[index];
if (keyILength < 0 ||
@@ -264,7 +269,7 @@
if (keyObjects[index] == null) {
// Keys are always primitive
keyObjects[index] = LazyFactory.createLazyPrimitiveClass(
- ((MapTypeInfo)typeInfo).getMapKeyTypeInfo().getTypeName());
+ (PrimitiveObjectInspector)((MapObjectInspector)oi).getMapKeyObjectInspector());
}
keyObjects[index].init(bytes, keyIBegin, keyILength);
}
@@ -276,34 +281,32 @@
* But each LazyMap has a separate cachedMap so we won't overwrite the
* data by accident.
*/
- HashMap<Object, Object> cachedMap;
+ LinkedHashMap<Object, Object> cachedMap;
/**
* Return the map object representing this LazyMap.
- * Note that the keyObjects will be Java primitive objects.
- * @param itemSeparator The separator between different entries.
- * @param keyValueSeparator The separator between key and value.
- * @param nullSequence The byte sequence representing NULL.
+ * Note that the keyObjects will be Writable primitive objects.
* @return the map object
*/
- public Map<Object, Object> getMap(byte itemSeparator, byte keyValueSeparator,
- Text nullSequence) {
+ public Map<Object, Object> getMap() {
if (!parsed) {
- parse(itemSeparator, keyValueSeparator, nullSequence);
+ parse();
}
if (cachedMap == null) {
- cachedMap = new HashMap<Object, Object>();
+ // Use LinkedHashMap to provide deterministic order
+ cachedMap = new LinkedHashMap<Object, Object>();
+ } else {
+ cachedMap.clear();
}
- cachedMap.clear();
// go through each element of the map
for (int i = 0; i < mapSize; i++) {
- LazyPrimitive<?> lazyKey = uncheckedGetKey(i, nullSequence);
+ LazyPrimitive<?,?> lazyKey = uncheckedGetKey(i);
if (lazyKey == null) continue;
Object key = lazyKey.getObject();
// do not overwrite if there are duplicate keys
if (key != null && !cachedMap.containsKey(key)) {
- LazyObject lazyValue = uncheckedGetValue(i, nullSequence);
+ LazyObject lazyValue = uncheckedGetValue(i);
Object value = (lazyValue == null ? null : lazyValue.getObject());
cachedMap.put(key, value);
}
@@ -313,15 +316,11 @@
/**
* Get the size of the map represented by this LazyMap.
- * @param itemSeparator The separator between different entries.
- * @param keyValueSeparator The separator between key and value.
- * @param nullSequence The byte sequence representing NULL.
* @return The size of the map, -1 for NULL map.
*/
- public int getMapSize(byte itemSeparator, byte keyValueSeparator,
- Text nullSequence) {
+ public int getMapSize() {
if (!parsed) {
- parse(itemSeparator, keyValueSeparator, nullSequence);
+ parse();
}
return mapSize;
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java Thu Jul 2 02:03:14 2009
@@ -17,25 +17,28 @@
*/
package org.apache.hadoop.hive.serde2.lazy;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* LazyPrimitive stores a primitive Object in a LazyObject.
*/
-public abstract class LazyNonPrimitive implements LazyObject {
+public abstract class LazyNonPrimitive<OI extends ObjectInspector> extends LazyObject<OI> {
protected ByteArrayRef bytes;
protected int start;
protected int length;
-
+
/**
- * The TypeInfo for this LazyNonPrimitive.
+ * Create a LazyNonPrimitive object with the specified ObjectInspector.
+ * @param oi The ObjectInspector would have to have a hierarchy of
+ * LazyObjectInspectors with the leaf nodes being
+ * WritableObjectInspectors. It's used both for accessing the
+ * type hierarchy of the complex object, as well as getting
+ * meta information (separator, nullSequence, etc) when parsing
+ * the lazy object.
*/
- TypeInfo typeInfo;
-
- protected LazyNonPrimitive(TypeInfo typeInfo) {
- this.typeInfo = typeInfo;
+ protected LazyNonPrimitive(OI oi) {
+ super(oi);
bytes = null;
start = 0;
length = 0;
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java Thu Jul 2 02:03:14 2009
@@ -17,14 +17,27 @@
*/
package org.apache.hadoop.hive.serde2.lazy;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
/**
* LazyObject stores an object in a range of bytes in a byte[].
*
* A LazyObject can represent any primitive object or hierarchical object
* like array, map or struct.
*/
-public interface LazyObject {
+public abstract class LazyObject<OI extends ObjectInspector> {
+ OI oi;
+
+ /**
+ * Create a LazyObject.
+ * @param oi Derived classes can access meta information about this Lazy
+ * Object (e.g, separator, nullSequence, escaper) from it.
+ */
+ protected LazyObject(OI oi) {
+ this.oi = oi;
+ }
+
/**
* Set the data for this LazyObject.
* We take ByteArrayRef instead of byte[] so that we will be able to drop
@@ -35,12 +48,12 @@
* @param length The length of the data, starting from "start"
* @see ByteArrayRef
*/
- void init(ByteArrayRef bytes, int start, int length);
+ public abstract void init(ByteArrayRef bytes, int start, int length);
/**
* If the LazyObject is a primitive Object, then deserialize it and return
* the actual primitive Object.
* Otherwise (array, map, struct), return this.
*/
- public Object getObject();
+ public abstract Object getObject();
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java Thu Jul 2 02:03:14 2009
@@ -17,20 +17,40 @@
*/
package org.apache.hadoop.hive.serde2.lazy;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
/**
* LazyPrimitive stores a primitive Object in a LazyObject.
*/
-public abstract class LazyPrimitive<T> implements LazyObject {
+public abstract class LazyPrimitive<OI extends ObjectInspector,
+ T extends Writable> extends LazyObject<OI> {
+
+ LazyPrimitive(OI oi) {
+ super(oi);
+ }
+
+ LazyPrimitive(LazyPrimitive<OI, T> copy) {
+ super(copy.oi);
+ isNull = copy.isNull;
+ }
T data;
boolean isNull = false;
/**
- * Returns the actual primitive object represented by this LazyObject.
+ * Returns the primitive object represented by this LazyObject.
+ * This is useful because it can make sure we have "null" for null objects.
*/
- public T getObject() {
- return isNull ? null : data;
+ public Object getObject() {
+ return isNull ? null : this;
}
+ public T getWritableObject() {
+ return isNull ? null : data;
+ }
+
+ public String toString() {
+ return isNull ? null : data.toString();
+ }
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java Thu Jul 2 02:03:14 2009
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.serde2.lazy;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObjectInspector;
/**
* LazyObject for storing a value of Short.
@@ -32,12 +33,18 @@
* </p>
*
*/
-public class LazyShort extends LazyPrimitive<ShortWritable> {
+public class LazyShort extends LazyPrimitive<LazyShortObjectInspector, ShortWritable> {
- public LazyShort() {
+ public LazyShort(LazyShortObjectInspector oi) {
+ super(oi);
data = new ShortWritable();
}
+ public LazyShort(LazyShort copy) {
+ super(copy);
+ data = new ShortWritable(copy.data.get());
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {