You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/05/13 20:23:15 UTC
svn commit: r1594315 [3/3] - in /hive/trunk: hbase-handler/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/queries/positive/
hbase-handler/src/test/results/positive/ ites...
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Tue May 13 18:23:14 2014
@@ -26,8 +26,8 @@ import org.apache.hadoop.hive.ql.exec.Ex
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -260,12 +260,59 @@ public class ExprNodeDescUtils {
return new ExprNodeDesc[] {expr1, expr2};
}
if (expr1 instanceof ExprNodeConstantDesc && expr2 instanceof ExprNodeColumnDesc) {
- return new ExprNodeDesc[] {expr2, expr1, null}; // add null as a marker (inverted order)
+ return new ExprNodeDesc[] {expr1, expr2};
+ }
+ // handles cases where the query has a predicate "column-name=constant"
+ if (expr1 instanceof ExprNodeFieldDesc && expr2 instanceof ExprNodeConstantDesc) {
+ ExprNodeColumnDesc columnDesc = extractColumn(expr1);
+ return columnDesc != null ? new ExprNodeDesc[] {columnDesc, expr2, expr1} : null;
+ }
+ // handles cases where the query has a predicate "constant=column-name"
+ if (expr1 instanceof ExprNodeConstantDesc && expr2 instanceof ExprNodeFieldDesc) {
+ ExprNodeColumnDesc columnDesc = extractColumn(expr2);
+ return columnDesc != null ? new ExprNodeDesc[] {expr1, columnDesc, expr2} : null;
}
// todo: constant op constant
return null;
}
+ /**
+ * Extract fields from the given {@link ExprNodeFieldDesc node descriptor}
+ * */
+ public static String[] extractFields(ExprNodeFieldDesc expr) {
+ return extractFields(expr, new ArrayList<String>()).toArray(new String[0]);
+ }
+
+ /*
+ * Recursively extract fields from ExprNodeDesc. Deeply nested structs can have multiple levels of
+ * fields in them
+ */
+ private static List<String> extractFields(ExprNodeDesc expr, List<String> fields) {
+ if (expr instanceof ExprNodeFieldDesc) {
+ ExprNodeFieldDesc field = (ExprNodeFieldDesc)expr;
+ fields.add(field.getFieldName());
+ return extractFields(field.getDesc(), fields);
+ }
+ if (expr instanceof ExprNodeColumnDesc) {
+ return fields;
+ }
+ throw new IllegalStateException(
+ "Unexpected exception while extracting fields from ExprNodeDesc");
+ }
+
+ /*
+ * Extract column from the given ExprNodeDesc
+ */
+ private static ExprNodeColumnDesc extractColumn(ExprNodeDesc expr) {
+ if (expr instanceof ExprNodeColumnDesc) {
+ return (ExprNodeColumnDesc)expr;
+ }
+ if (expr instanceof ExprNodeFieldDesc) {
+ return extractColumn(((ExprNodeFieldDesc)expr).getDesc());
+ }
+ return null;
+ }
+
// from IndexPredicateAnalyzer
private static ExprNodeDesc extractConstant(ExprNodeDesc expr) {
if (!(expr instanceof ExprNodeGenericFuncDesc)) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java Tue May 13 18:23:14 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.plan;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -61,6 +62,7 @@ public class TableScanDesc extends Abstr
private int maxStatsKeyPrefixLength = -1;
private ExprNodeGenericFuncDesc filterExpr;
+ private transient Serializable filterObject;
public static final String FILTER_EXPR_CONF_STR =
"hive.io.filter.expr.serialized";
@@ -68,6 +70,9 @@ public class TableScanDesc extends Abstr
public static final String FILTER_TEXT_CONF_STR =
"hive.io.filter.text";
+ public static final String FILTER_OBJECT_CONF_STR =
+ "hive.io.filter.object";
+
// input file name (big) to bucket number
private Map<String, Integer> bucketFileNameMapping;
@@ -112,6 +117,14 @@ public class TableScanDesc extends Abstr
this.filterExpr = filterExpr;
}
+ public Serializable getFilterObject() {
+ return filterObject;
+ }
+
+ public void setFilterObject(Serializable filterObject) {
+ this.filterObject = filterObject;
+ }
+
public void setAlias(String alias) {
this.alias = alias;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Tue May 13 18:23:14 2014
@@ -843,6 +843,8 @@ public final class OpProcFactory {
}
}
tableScanDesc.setFilterExpr(decomposed.pushedPredicate);
+ tableScanDesc.setFilterObject(decomposed.pushedPredicateObject);
+
return (ExprNodeGenericFuncDesc)decomposed.residualPredicate;
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Tue May 13 18:23:14 2014
@@ -216,6 +216,11 @@ public class TestInputOutputFormat {
}
@Override
+ public int getFieldID() {
+ return id;
+ }
+
+ @Override
public String toString() {
return "field " + id + " " + fieldName;
}
Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java?rev=1594315&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java Tue May 13 18:23:14 2014
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class BaseStructObjectInspector extends StructObjectInspector {
+
+ protected static class MyField implements StructField {
+
+ protected final int fieldID;
+ protected final String fieldName;
+ protected final String fieldComment;
+ protected final ObjectInspector fieldObjectInspector;
+
+ public MyField(int fieldID, String fieldName,
+ ObjectInspector fieldObjectInspector, String fieldComment) {
+ this.fieldID = fieldID;
+ this.fieldName = fieldName.toLowerCase();
+ this.fieldObjectInspector = fieldObjectInspector;
+ this.fieldComment = fieldComment;
+ }
+
+ public MyField(int fieldID, StructField field) {
+ this.fieldID = fieldID;
+ this.fieldName = field.getFieldName().toLowerCase();
+ this.fieldObjectInspector = field.getFieldObjectInspector();
+ this.fieldComment = field.getFieldComment();
+ }
+
+ public int getFieldID() {
+ return fieldID;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+
+ @Override
+ public ObjectInspector getFieldObjectInspector() {
+ return fieldObjectInspector;
+ }
+
+ @Override
+ public String getFieldComment() {
+ return fieldComment;
+ }
+
+ @Override
+ public String toString() {
+ return fieldID + ":" + fieldName;
+ }
+ }
+
+ protected final List<MyField> fields = new ArrayList<MyField>();
+
+ protected BaseStructObjectInspector() {
+ super();
+ }
+
+ /**
+ * Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead.
+ */
+ public BaseStructObjectInspector(List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors) {
+ init(structFieldNames, structFieldObjectInspectors, null);
+ }
+
+ public BaseStructObjectInspector(List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments) {
+ init(structFieldNames, structFieldObjectInspectors, structFieldComments);
+ }
+
+ protected void init(List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors,
+ List<String> structFieldComments) {
+ assert (structFieldNames.size() == structFieldObjectInspectors.size());
+ assert (structFieldComments == null ||
+ (structFieldNames.size() == structFieldComments.size()));
+
+ for (int i = 0; i < structFieldNames.size(); i++) {
+ fields.add(createField(i,
+ structFieldNames.get(i), structFieldObjectInspectors.get(i),
+ structFieldComments == null ? null : structFieldComments.get(i)));
+ }
+ }
+
+ protected void init(List<StructField> structFields) {
+ for (int i = 0; i < structFields.size(); i++) {
+ fields.add(new MyField(i, structFields.get(i)));
+ }
+ }
+
+ // override this for using extended FieldObject
+ protected MyField createField(int index, String fieldName, ObjectInspector fieldOI, String comment) {
+ return new MyField(index, fieldName, fieldOI, comment);
+ }
+
+ @Override
+ public String getTypeName() {
+ return ObjectInspectorUtils.getStandardStructTypeName(this);
+ }
+
+ @Override
+ public final Category getCategory() {
+ return Category.STRUCT;
+ }
+
+ @Override
+ public StructField getStructFieldRef(String fieldName) {
+ return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
+ }
+
+ @Override
+ public List<? extends StructField> getAllStructFieldRefs() {
+ return fields;
+ }
+}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java Tue May 13 18:23:14 2014
@@ -46,6 +46,11 @@ public class NullStructSerDe extends Abs
}
@Override
+ public int getFieldID() {
+ return -1;
+ }
+
+ @Override
public String getFieldComment() {
return "";
}
Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java?rev=1594315&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java Tue May 13 18:23:14 2014
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.util.List;
+
+public interface StructObject {
+
+ Object getField(int fieldID);
+
+ List<Object> getFieldsAsList();
+}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java Tue May 13 18:23:14 2014
@@ -23,13 +23,14 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.serde2.SerDeStatsStruct;
+import org.apache.hadoop.hive.serde2.StructObject;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-public abstract class ColumnarStructBase implements SerDeStatsStruct {
+public abstract class ColumnarStructBase implements StructObject, SerDeStatsStruct {
class FieldInfo {
LazyObjectBase field;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java Tue May 13 18:23:14 2014
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.serde2.obj
* A LazyObject can represent any primitive object or hierarchical object like
* array, map or struct.
*/
-public abstract class LazyObject<OI extends ObjectInspector> extends LazyObjectBase {
+public abstract class LazyObject<OI extends ObjectInspector> implements LazyObjectBase {
protected OI oi;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java Tue May 13 18:23:14 2014
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.serde2.lazy;
-public abstract class LazyObjectBase {
+public interface LazyObjectBase {
/**
* Set the data for this LazyObjectBase. We take ByteArrayRef instead of byte[] so
@@ -33,12 +33,12 @@ public abstract class LazyObjectBase {
* The length of the data, starting from "start"
* @see ByteArrayRef
*/
- public abstract void init(ByteArrayRef bytes, int start, int length);
+ void init(ByteArrayRef bytes, int start, int length);
/**
* If the LazyObjectBase is a primitive Object, then deserialize it and return the
* actual primitive Object. Otherwise (array, map, struct), return this.
*/
- public abstract Object getObject();
+ Object getObject();
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java Tue May 13 18:23:14 2014
@@ -23,10 +23,11 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeStatsStruct;
+import org.apache.hadoop.hive.serde2.StructObject;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.Text;
/**
@@ -36,8 +37,8 @@ import org.apache.hadoop.io.Text;
* LazyStruct does not deal with the case of a NULL struct. That is handled by
* the parent LazyObject.
*/
-public class LazyStruct extends LazyNonPrimitive<LazySimpleStructObjectInspector> implements
- SerDeStatsStruct {
+public class LazyStruct extends LazyNonPrimitive<LazySimpleStructObjectInspector>
+ implements StructObject, SerDeStatsStruct {
private static Log LOG = LogFactory.getLog(LazyStruct.class.getName());
@@ -62,7 +63,7 @@ public class LazyStruct extends LazyNonP
/**
* The fields of the struct.
*/
- LazyObject[] fields;
+ LazyObjectBase[] fields;
/**
* Whether init() has been called on the field or not.
*/
@@ -101,17 +102,7 @@ public class LazyStruct extends LazyNonP
byte escapeChar = oi.getEscapeChar();
if (fields == null) {
- List<? extends StructField> fieldRefs = ((StructObjectInspector) oi)
- .getAllStructFieldRefs();
- fields = new LazyObject[fieldRefs.size()];
- for (int i = 0; i < fields.length; i++) {
- fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i)
- .getFieldObjectInspector());
- }
- fieldInited = new boolean[fields.length];
- // Extra element to make sure we have the same formula to compute the
- // length of each element of the array.
- startPosition = new int[fields.length + 1];
+ initLazyFields(oi.getAllStructFieldRefs());
}
int structByteEnd = start + length;
@@ -172,6 +163,25 @@ public class LazyStruct extends LazyNonP
parsed = true;
}
+ protected final void initLazyFields(List<? extends StructField> fieldRefs) {
+ fields = new LazyObjectBase[fieldRefs.size()];
+ for (int i = 0; i < fields.length; i++) {
+ try {
+ fields[i] = createLazyField(i, fieldRefs.get(i));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ fieldInited = new boolean[fields.length];
+ // Extra element to make sure we have the same formula to compute the
+ // length of each element of the array.
+ startPosition = new int[fields.length + 1];
+ }
+
+ protected LazyObjectBase createLazyField(int fieldID, StructField fieldRef) throws SerDeException {
+ return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector());
+ }
+
/**
* Get one field out of the struct.
*
@@ -221,14 +231,14 @@ public class LazyStruct extends LazyNonP
return fields[fieldID].getObject();
}
- ArrayList<Object> cachedList;
+ List<Object> cachedList;
/**
* Get the values of the fields as an ArrayList.
*
* @return The values of the fields as an ArrayList.
*/
- public ArrayList<Object> getFieldsAsList() {
+ public List<Object> getFieldsAsList() {
if (!parsed) {
parse();
}
@@ -256,7 +266,7 @@ public class LazyStruct extends LazyNonP
this.parsed = parsed;
}
- protected LazyObject[] getFields() {
+ protected LazyObjectBase[] getFields() {
return fields;
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java Tue May 13 18:23:14 2014
@@ -18,16 +18,12 @@
package org.apache.hadoop.hive.serde2.lazy.objectinspector;
-import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
+import org.apache.hadoop.hive.serde2.BaseStructObjectInspector;
+import org.apache.hadoop.hive.serde2.StructObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.Text;
/**
@@ -40,55 +36,8 @@ import org.apache.hadoop.io.Text;
* Always use the ObjectInspectorFactory to create new ObjectInspector objects,
* instead of directly creating an instance of this class.
*/
-public class LazySimpleStructObjectInspector extends StructObjectInspector {
+public class LazySimpleStructObjectInspector extends BaseStructObjectInspector {
- public static final Log LOG = LogFactory
- .getLog(LazySimpleStructObjectInspector.class.getName());
-
- protected static class MyField implements StructField {
- protected int fieldID;
- protected String fieldName;
- protected ObjectInspector fieldObjectInspector;
- protected String fieldComment;
-
- protected MyField() {
- super();
- }
- public MyField(int fieldID, String fieldName,
- ObjectInspector fieldObjectInspector) {
- this.fieldID = fieldID;
- this.fieldName = fieldName.toLowerCase();
- this.fieldObjectInspector = fieldObjectInspector;
- }
-
- public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector, String fieldComment) {
- this(fieldID, fieldName, fieldObjectInspector);
- this.fieldComment = fieldComment;
- }
-
- public int getFieldID() {
- return fieldID;
- }
-
- public String getFieldName() {
- return fieldName;
- }
-
- public ObjectInspector getFieldObjectInspector() {
- return fieldObjectInspector;
- }
-
- public String getFieldComment() {
- return fieldComment;
- }
-
- @Override
- public String toString() {
- return "" + fieldID + ":" + fieldName;
- }
- }
-
- private List<MyField> fields;
private byte separator;
private Text nullSequence;
private boolean lastColumnTakesRest;
@@ -98,9 +47,18 @@ public class LazySimpleStructObjectInspe
protected LazySimpleStructObjectInspector() {
super();
}
+
+ protected LazySimpleStructObjectInspector(
+ List<StructField> fields, byte separator, Text nullSequence) {
+ init(fields);
+ this.separator = separator;
+ this.nullSequence = nullSequence;
+ }
+
/**
* Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead.
*/
+ @Deprecated
protected LazySimpleStructObjectInspector(List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors, byte separator,
Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
@@ -122,60 +80,12 @@ public class LazySimpleStructObjectInspe
List<String> structFieldComments, byte separator,
Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
byte escapeChar) {
- assert (structFieldNames.size() == structFieldObjectInspectors.size());
- assert (structFieldComments == null ||
- structFieldNames.size() == structFieldComments.size());
-
+ init(structFieldNames, structFieldObjectInspectors, structFieldComments);
this.separator = separator;
this.nullSequence = nullSequence;
this.lastColumnTakesRest = lastColumnTakesRest;
this.escaped = escaped;
this.escapeChar = escapeChar;
-
- fields = new ArrayList<MyField>(structFieldNames.size());
- for (int i = 0; i < structFieldNames.size(); i++) {
- fields.add(new MyField(i, structFieldNames.get(i),
- structFieldObjectInspectors.get(i),
- structFieldComments == null ? null : structFieldComments.get(i)));
- }
- }
-
- protected LazySimpleStructObjectInspector(List<StructField> fields,
- byte separator, Text nullSequence) {
- init(fields, separator, nullSequence);
- }
-
- protected void init(List<StructField> fields, byte separator,
- Text nullSequence) {
- this.separator = separator;
- this.nullSequence = nullSequence;
-
- this.fields = new ArrayList<MyField>(fields.size());
- for (int i = 0; i < fields.size(); i++) {
- this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
- .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment()));
- }
- }
-
- @Override
- public String getTypeName() {
- return ObjectInspectorUtils.getStandardStructTypeName(this);
- }
-
- @Override
- public final Category getCategory() {
- return Category.STRUCT;
- }
-
- // Without Data
- @Override
- public StructField getStructFieldRef(String fieldName) {
- return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
- }
-
- @Override
- public List<? extends StructField> getAllStructFieldRefs() {
- return fields;
}
// With Data
@@ -184,7 +94,7 @@ public class LazySimpleStructObjectInspe
if (data == null) {
return null;
}
- LazyStruct struct = (LazyStruct) data;
+ StructObject struct = (StructObject) data;
MyField f = (MyField) fieldRef;
int fieldID = f.getFieldID();
@@ -198,15 +108,8 @@ public class LazySimpleStructObjectInspe
if (data == null) {
return null;
}
-
- // Iterate over all the fields picking up the nested structs within them
- List<Object> result = new ArrayList<Object>(fields.size());
-
- for (MyField myField : fields) {
- result.add(getStructFieldData(data, myField));
- }
-
- return result;
+ StructObject struct = (StructObject) data;
+ return struct.getFieldsAsList();
}
// For LazyStruct
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java Tue May 13 18:23:14 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.serde2.obj
* A LazyBinaryObject can represent any primitive object or hierarchical object
* like string, list, map or struct.
*/
-public abstract class LazyBinaryObject<OI extends ObjectInspector> extends LazyObjectBase {
+public abstract class LazyBinaryObject<OI extends ObjectInspector> implements LazyObjectBase {
OI oi;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java Tue May 13 18:23:14 2014
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.SerDeStatsStruct;
+import org.apache.hadoop.hive.serde2.StructObject;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.RecordInfo;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector;
@@ -42,8 +43,8 @@ import org.apache.hadoop.hive.serde2.obj
* Following B, there is another section A and B. This pattern repeats until the
* all struct fields are serialized.
*/
-public class LazyBinaryStruct extends
- LazyBinaryNonPrimitive<LazyBinaryStructObjectInspector> implements SerDeStatsStruct {
+public class LazyBinaryStruct extends LazyBinaryNonPrimitive<LazyBinaryStructObjectInspector>
+ implements StructObject, SerDeStatsStruct {
private static Log LOG = LogFactory.getLog(LazyBinaryStruct.class.getName());
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java Tue May 13 18:23:14 2014
@@ -18,11 +18,9 @@
package org.apache.hadoop.hive.serde2.objectinspector;
-import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde2.BaseStructObjectInspector;
import org.apache.hadoop.hive.serde2.columnar.ColumnarStructBase;
/**
@@ -35,123 +33,26 @@ import org.apache.hadoop.hive.serde2.col
* Always use the ObjectInspectorFactory to create new ObjectInspector objects,
* instead of directly creating an instance of this class.
*/
-class ColumnarStructObjectInspector extends StructObjectInspector {
-
- public static final Log LOG = LogFactory
- .getLog(ColumnarStructObjectInspector.class.getName());
-
- protected static class MyField implements StructField {
- protected int fieldID;
- protected String fieldName;
- protected ObjectInspector fieldObjectInspector;
- protected String fieldComment;
-
- protected MyField() {
- super();
- }
-
- public MyField(int fieldID, String fieldName,
- ObjectInspector fieldObjectInspector) {
- this.fieldID = fieldID;
- this.fieldName = fieldName.toLowerCase();
- this.fieldObjectInspector = fieldObjectInspector;
- }
-
- public MyField(int fieldID, String fieldName,
- ObjectInspector fieldObjectInspector, String fieldComment) {
- this(fieldID, fieldName, fieldObjectInspector);
- this.fieldComment = fieldComment;
- }
-
- public int getFieldID() {
- return fieldID;
- }
-
- public String getFieldName() {
- return fieldName;
- }
-
- public ObjectInspector getFieldObjectInspector() {
- return fieldObjectInspector;
- }
-
- public String getFieldComment() {
- return fieldComment;
- }
- @Override
- public String toString() {
- return "" + fieldID + ":" + fieldName;
- }
- }
-
- private List<MyField> fields;
+class ColumnarStructObjectInspector extends BaseStructObjectInspector {
protected ColumnarStructObjectInspector() {
super();
}
+
/**
* Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead.
*/
public ColumnarStructObjectInspector(List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
- init(structFieldNames, structFieldObjectInspectors, null);
+ super(structFieldNames, structFieldObjectInspectors);
}
public ColumnarStructObjectInspector(List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors,
List<String> structFieldComments) {
- init(structFieldNames, structFieldObjectInspectors, structFieldComments);
- }
-
- protected void init(List<String> structFieldNames,
- List<ObjectInspector> structFieldObjectInspectors,
- List<String> structFieldComments) {
- assert (structFieldNames.size() == structFieldObjectInspectors.size());
- assert (structFieldComments == null ||
- (structFieldNames.size() == structFieldComments.size()));
-
- fields = new ArrayList<MyField>(structFieldNames.size());
- for (int i = 0; i < structFieldNames.size(); i++) {
- fields.add(new MyField(i, structFieldNames.get(i),
- structFieldObjectInspectors.get(i),
- structFieldComments == null ? null : structFieldComments.get(i)));
- }
- }
-
- protected ColumnarStructObjectInspector(List<StructField> fields) {
- init(fields);
- }
-
- protected void init(List<StructField> fields) {
- this.fields = new ArrayList<MyField>(fields.size());
- for (int i = 0; i < fields.size(); i++) {
- this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
- .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment()));
- }
- }
-
- @Override
- public String getTypeName() {
- return ObjectInspectorUtils.getStandardStructTypeName(this);
- }
-
- @Override
- public final Category getCategory() {
- return Category.STRUCT;
- }
-
- // Without Data
- @Override
- public StructField getStructFieldRef(String fieldName) {
- return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
- }
-
- @Override
- public List<? extends StructField> getAllStructFieldRefs() {
- return fields;
+ super(structFieldNames, structFieldObjectInspectors, structFieldComments);
}
- // With Data
@Override
public Object getStructFieldData(Object data, StructField fieldRef) {
if (data == null) {
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/DelegatedStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/DelegatedStructObjectInspector.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/DelegatedStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/DelegatedStructObjectInspector.java Tue May 13 18:23:14 2014
@@ -56,6 +56,9 @@ public class DelegatedStructObjectInspec
public ObjectInspector getFieldObjectInspector() {
return field.getFieldObjectInspector();
}
+ public int getFieldID() {
+ return field.getFieldID();
+ }
public String getFieldComment() {
return field.getFieldComment();
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java Tue May 13 18:23:14 2014
@@ -42,6 +42,7 @@ public class ReflectionStructObjectInspe
*
*/
public static class MyField implements StructField {
+ protected int fieldID;
protected Field field;
protected ObjectInspector fieldObjectInspector;
@@ -49,7 +50,8 @@ public class ReflectionStructObjectInspe
super();
}
- public MyField(Field field, ObjectInspector fieldObjectInspector) {
+ public MyField(int fieldID, Field field, ObjectInspector fieldObjectInspector) {
+ this.fieldID = fieldID;
this.field = field;
this.fieldObjectInspector = fieldObjectInspector;
}
@@ -62,6 +64,10 @@ public class ReflectionStructObjectInspe
return fieldObjectInspector;
}
+ public int getFieldID() {
+ return fieldID;
+ }
+
public String getFieldComment() {
return null;
}
@@ -123,7 +129,7 @@ public class ReflectionStructObjectInspe
for (int i = 0; i < reflectionFields.length; i++) {
if (!shouldIgnoreField(reflectionFields[i].getName())) {
reflectionFields[i].setAccessible(true);
- fields.add(new MyField(reflectionFields[i], structFieldObjectInspectors
+ fields.add(new MyField(i, reflectionFields[i], structFieldObjectInspectors
.get(used++)));
}
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java Tue May 13 18:23:14 2014
@@ -19,8 +19,6 @@
package org.apache.hadoop.hive.serde2.objectinspector;
/**
- * StructField is an empty interface.
- *
* Classes implementing this interface are considered to represent a field of a
* struct for this serde package.
*/
@@ -37,6 +35,11 @@ public interface StructField {
ObjectInspector getFieldObjectInspector();
/**
+ * Get the fieldID for the field.
+ */
+ int getFieldID();
+
+ /**
* Get the comment for the field. May be null if no comment provided.
*/
String getFieldComment();
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java Tue May 13 18:23:14 2014
@@ -60,6 +60,11 @@ public class UnionStructObjectInspector
return structField.getFieldObjectInspector();
}
+ @Override
+ public int getFieldID() {
+ return structID;
+ }
+
public String getFieldComment() {
return structField.getFieldComment();
}