You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/22 08:10:22 UTC
svn commit: r1525322 [7/7] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/
contrib/src/test/results/clientpositive/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/results/positive/ hcatalog/core/src/mai...
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml Sun Sep 22 06:10:15 2013
@@ -230,6 +230,9 @@
</void>
</object>
</void>
+ <void property="conf">
+ <object class="org.apache.hadoop.hive.ql.plan.TableScanDesc"/>
+ </void>
<void property="counterNames">
<object class="java.util.ArrayList">
<void method="add">
@@ -246,6 +249,26 @@
</void>
</object>
</void>
+ <void property="neededColumnIDs">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <int>0</int>
+ </void>
+ <void method="add">
+ <int>1</int>
+ </void>
+ </object>
+ </void>
+ <void property="neededColumns">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <string>key</string>
+ </void>
+ <void method="add">
+ <string>value</string>
+ </void>
+ </object>
+ </void>
<void property="operatorId">
<string>TS_4</string>
</void>
Modified: hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml Sun Sep 22 06:10:15 2013
@@ -230,6 +230,9 @@
</void>
</object>
</void>
+ <void property="conf">
+ <object class="org.apache.hadoop.hive.ql.plan.TableScanDesc"/>
+ </void>
<void property="counterNames">
<object class="java.util.ArrayList">
<void method="add">
@@ -246,6 +249,26 @@
</void>
</object>
</void>
+ <void property="neededColumnIDs">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <int>0</int>
+ </void>
+ <void method="add">
+ <int>1</int>
+ </void>
+ </object>
+ </void>
+ <void property="neededColumns">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <string>key</string>
+ </void>
+ <void method="add">
+ <string>value</string>
+ </void>
+ </object>
+ </void>
<void property="operatorId">
<string>TS_6</string>
</void>
Modified: hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml Sun Sep 22 06:10:15 2013
@@ -163,6 +163,9 @@
</void>
</object>
</void>
+ <void property="conf">
+ <object class="org.apache.hadoop.hive.ql.plan.TableScanDesc"/>
+ </void>
<void property="counterNames">
<object class="java.util.ArrayList">
<void method="add">
@@ -179,6 +182,26 @@
</void>
</object>
</void>
+ <void property="neededColumnIDs">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <int>0</int>
+ </void>
+ <void method="add">
+ <int>1</int>
+ </void>
+ </object>
+ </void>
+ <void property="neededColumns">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <string>_col0</string>
+ </void>
+ <void method="add">
+ <string>_col1</string>
+ </void>
+ </object>
+ </void>
<void property="operatorId">
<string>TS_6</string>
</void>
Modified: hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/union.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/union.q.xml Sun Sep 22 06:10:15 2013
@@ -163,6 +163,9 @@
</void>
</object>
</void>
+ <void property="conf">
+ <object class="org.apache.hadoop.hive.ql.plan.TableScanDesc"/>
+ </void>
<void property="counterNames">
<object class="java.util.ArrayList">
<void method="add">
@@ -179,6 +182,26 @@
</void>
</object>
</void>
+ <void property="neededColumnIDs">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <int>0</int>
+ </void>
+ <void method="add">
+ <int>1</int>
+ </void>
+ </object>
+ </void>
+ <void property="neededColumns">
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <string>_col0</string>
+ </void>
+ <void method="add">
+ <string>_col1</string>
+ </void>
+ </object>
+ </void>
<void property="operatorId">
<string>TS_11</string>
</void>
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java Sun Sep 22 06:10:15 2013
@@ -32,87 +32,57 @@ public final class ColumnProjectionUtils
public static final String READ_COLUMN_IDS_CONF_STR = "hive.io.file.readcolumn.ids";
public static final String READ_COLUMN_NAMES_CONF_STR = "hive.io.file.readcolumn.names";
+ private static final String READ_COLUMN_IDS_CONF_STR_DEFAULT = "";
+ private static final String READ_ALL_COLUMNS = "hive.io.file.read.all.columns";
+ private static final boolean READ_ALL_COLUMNS_DEFAULT = true;
/**
- * Sets read columns' ids(start from zero) for RCFile's Reader. Once a column
- * is included in the list, RCFile's reader will not skip its value.
- *
+ * Sets the <em>READ_ALL_COLUMNS</em> flag and removes any previously
+ * set column ids.
*/
- public static void setReadColumnIDs(Configuration conf, List<Integer> ids) {
- String id = toReadColumnIDString(ids);
- setReadColumnIDConf(conf, id);
+ public static void setReadAllColumns(Configuration conf) {
+ conf.setBoolean(READ_ALL_COLUMNS, true);
+ setReadColumnIDConf(conf, READ_COLUMN_IDS_CONF_STR_DEFAULT);
}
/**
- * Sets read columns' ids(start from zero) for RCFile's Reader. Once a column
- * is included in the list, RCFile's reader will not skip its value.
- *
+ * Returns the <em>READ_ALL_COLUMNS</em> columns flag.
*/
- public static void appendReadColumnIDs(Configuration conf, List<Integer> ids) {
- String id = toReadColumnIDString(ids);
- if (id != null) {
- String old = conf.get(READ_COLUMN_IDS_CONF_STR, null);
- String newConfStr = id;
- if (old != null) {
- newConfStr = newConfStr + StringUtils.COMMA_STR + old;
- }
-
- setReadColumnIDConf(conf, newConfStr);
- }
- }
-
- public static void appendReadColumnNames(Configuration conf,
- List<String> cols) {
- if (cols != null) {
- String old = conf.get(READ_COLUMN_NAMES_CONF_STR, "");
- StringBuilder result = new StringBuilder(old);
- boolean first = old.isEmpty();
- for(String col: cols) {
- if (first) {
- first = false;
- } else {
- result.append(',');
- }
- result.append(col);
- }
- conf.set(READ_COLUMN_NAMES_CONF_STR, result.toString());
- }
+ public static boolean isReadAllColumns(Configuration conf) {
+ return conf.getBoolean(READ_ALL_COLUMNS, READ_ALL_COLUMNS_DEFAULT);
}
- private static void setReadColumnIDConf(Configuration conf, String id) {
- if (id == null || id.length() <= 0) {
- conf.set(READ_COLUMN_IDS_CONF_STR, "");
- return;
- }
-
- conf.set(READ_COLUMN_IDS_CONF_STR, id);
+ /**
+ * Appends read columns' ids (start from zero). Once a column
+ * is included in the list, a underlying record reader of a columnar file format
+ * (e.g. RCFile and ORC) can know what columns are needed.
+ */
+ public static void appendReadColumns(Configuration conf, List<Integer> ids) {
+ String id = toReadColumnIDString(ids);
+ String old = conf.get(READ_COLUMN_IDS_CONF_STR, null);
+ String newConfStr = id;
+ if (old != null) {
+ newConfStr = newConfStr + StringUtils.COMMA_STR + old;
+ }
+ setReadColumnIDConf(conf, newConfStr);
+ // Set READ_ALL_COLUMNS to false
+ conf.setBoolean(READ_ALL_COLUMNS, false);
}
- private static String toReadColumnIDString(List<Integer> ids) {
- String id = null;
- if (ids != null) {
- for (int i = 0; i < ids.size(); i++) {
- if (i == 0) {
- id = "" + ids.get(i);
- } else {
- id = id + StringUtils.COMMA_STR + ids.get(i);
- }
- }
- }
- return id;
+ public static void appendReadColumns(
+ Configuration conf, List<Integer> ids, List<String> names) {
+ appendReadColumns(conf, ids);
+ appendReadColumnNames(conf, names);
}
/**
* Returns an array of column ids(start from zero) which is set in the given
* parameter <tt>conf</tt>.
*/
- public static ArrayList<Integer> getReadColumnIDs(Configuration conf) {
- if (conf == null) {
- return new ArrayList<Integer>(0);
- }
- String skips = conf.get(READ_COLUMN_IDS_CONF_STR, "");
+ public static List<Integer> getReadColumnIDs(Configuration conf) {
+ String skips = conf.get(READ_COLUMN_IDS_CONF_STR, READ_COLUMN_IDS_CONF_STR_DEFAULT);
String[] list = StringUtils.split(skips);
- ArrayList<Integer> result = new ArrayList<Integer>(list.length);
+ List<Integer> result = new ArrayList<Integer>(list.length);
for (String element : list) {
// it may contain duplicates, remove duplicates
Integer toAdd = Integer.parseInt(element);
@@ -123,11 +93,39 @@ public final class ColumnProjectionUtils
return result;
}
- /**
- * Clears the read column ids set in the conf, and will read all columns.
- */
- public static void setFullyReadColumns(Configuration conf) {
- conf.set(READ_COLUMN_IDS_CONF_STR, "");
+ private static void setReadColumnIDConf(Configuration conf, String id) {
+ if (id.trim().isEmpty()) {
+ conf.set(READ_COLUMN_IDS_CONF_STR, READ_COLUMN_IDS_CONF_STR_DEFAULT);
+ } else {
+ conf.set(READ_COLUMN_IDS_CONF_STR, id);
+ }
+ }
+
+ private static void appendReadColumnNames(Configuration conf, List<String> cols) {
+ String old = conf.get(READ_COLUMN_NAMES_CONF_STR, "");
+ StringBuilder result = new StringBuilder(old);
+ boolean first = old.isEmpty();
+ for(String col: cols) {
+ if (first) {
+ first = false;
+ } else {
+ result.append(',');
+ }
+ result.append(col);
+ }
+ conf.set(READ_COLUMN_NAMES_CONF_STR, result.toString());
+ }
+
+ private static String toReadColumnIDString(List<Integer> ids) {
+ String id = "";
+ for (int i = 0; i < ids.size(); i++) {
+ if (i == 0) {
+ id = id + ids.get(i);
+ } else {
+ id = id + StringUtils.COMMA_STR + ids.get(i);
+ }
+ }
+ return id;
}
private ColumnProjectionUtils() {
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java Sun Sep 22 06:10:15 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.serde2.columnar;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
@@ -26,11 +27,9 @@ import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
@@ -79,9 +78,10 @@ public class ColumnarSerDe extends Colum
*
* @see SerDe#initialize(Configuration, Properties)
*/
- public void initialize(Configuration job, Properties tbl) throws SerDeException {
+ @Override
+ public void initialize(Configuration conf, Properties tbl) throws SerDeException {
- serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, getClass().getName());
+ serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, getClass().getName());
// Create the ObjectInspectors for the fields. Note: Currently
// ColumnarObject uses same ObjectInpector as LazyStruct
@@ -90,14 +90,20 @@ public class ColumnarSerDe extends Colum
.getSeparators(), serdeParams.getNullSequence(), serdeParams
.isEscaped(), serdeParams.getEscapeChar());
- java.util.ArrayList<Integer> notSkipIDs = ColumnProjectionUtils.getReadColumnIDs(job);
-
- cachedLazyStruct = new ColumnarStruct(cachedObjectInspector, notSkipIDs,
- serdeParams.getNullSequence());
-
int size = serdeParams.getColumnTypes().size();
+ List<Integer> notSkipIDs = new ArrayList<Integer>();
+ if (conf == null || ColumnProjectionUtils.isReadAllColumns(conf)) {
+ for (int i = 0; i < size; i++ ) {
+ notSkipIDs.add(i);
+ }
+ } else {
+ notSkipIDs = ColumnProjectionUtils.getReadColumnIDs(conf);
+ }
+ cachedLazyStruct = new ColumnarStruct(
+ cachedObjectInspector, notSkipIDs, serdeParams.getNullSequence());
+
super.initialize(size);
- LOG.debug("ColumnarSerDe initialized with: columnNames="
+ LOG.info("ColumnarSerDe initialized with: columnNames="
+ serdeParams.getColumnNames() + " columnTypes="
+ serdeParams.getColumnTypes() + " separator="
+ Arrays.asList(serdeParams.getSeparators()) + " nullstring="
@@ -114,6 +120,7 @@ public class ColumnarSerDe extends Colum
* @return The serialized Writable object
* @see SerDe#serialize(Object, ObjectInspector)
*/
+ @Override
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
if (objInspector.getCategory() != Category.STRUCT) {
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java Sun Sep 22 06:10:15 2013
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.serde2.columnar;
-import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -49,22 +49,10 @@ public class ColumnarStruct extends Colu
*
* @param oi
* the ObjectInspector representing the type of this LazyStruct.
- */
- public ColumnarStruct(ObjectInspector oi) {
- this(oi, null, null);
- }
-
- /**
- * Construct a ColumnarStruct object with the TypeInfo. It creates the first
- * level object at the first place
- *
- * @param oi
- * the ObjectInspector representing the type of this LazyStruct.
* @param notSkippedColumnIDs
* the column ids that should not be skipped
*/
- public ColumnarStruct(ObjectInspector oi,
- ArrayList<Integer> notSkippedColumnIDs, Text nullSequence) {
+ public ColumnarStruct(ObjectInspector oi, List<Integer> notSkippedColumnIDs, Text nullSequence) {
super(oi, notSkippedColumnIDs);
if (nullSequence != null) {
this.nullSequence = nullSequence;
@@ -84,7 +72,7 @@ public class ColumnarStruct extends Colu
}
return fieldLen;
}
-
+
@Override
protected LazyObjectBase createLazyObjectBase(ObjectInspector objectInspector) {
return LazyFactory.createLazyObject(objectInspector);
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java Sun Sep 22 06:10:15 2013
@@ -122,22 +122,13 @@ public abstract class ColumnarStructBase
private FieldInfo[] fieldInfoList = null;
private ArrayList<Object> cachedList;
- public ColumnarStructBase(ObjectInspector oi,
- ArrayList<Integer> notSkippedColumnIDs) {
+ public ColumnarStructBase(ObjectInspector oi, List<Integer> notSkippedColumnIDs) {
List<? extends StructField> fieldRefs = ((StructObjectInspector) oi)
.getAllStructFieldRefs();
int num = fieldRefs.size();
fieldInfoList = new FieldInfo[num];
- // if no columns is set to be skipped, add all columns in
- // 'notSkippedColumnIDs'
- if (notSkippedColumnIDs == null || notSkippedColumnIDs.size() == 0) {
- for (int i = 0; i < num; i++) {
- notSkippedColumnIDs.add(i);
- }
- }
-
for (int i = 0; i < num; i++) {
ObjectInspector foi = fieldRefs.get(i).getFieldObjectInspector();
fieldInfoList[i] = new FieldInfo(
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java Sun Sep 22 06:10:15 2013
@@ -17,22 +17,23 @@
*/
package org.apache.hadoop.hive.serde2.columnar;
+import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.Writable;
@@ -66,9 +67,17 @@ public class LazyBinaryColumnarSerDe ext
cachedObjectInspector = LazyBinaryFactory.createColumnarStructInspector(
columnNames, columnTypes);
- java.util.ArrayList<Integer> notSkipIDs = ColumnProjectionUtils.getReadColumnIDs(conf);
- cachedLazyStruct = new LazyBinaryColumnarStruct(cachedObjectInspector, notSkipIDs);
int size = columnTypes.size();
+ List<Integer> notSkipIDs = new ArrayList<Integer>();
+ if (conf == null || ColumnProjectionUtils.isReadAllColumns(conf)) {
+ for (int i = 0; i < size; i++ ) {
+ notSkipIDs.add(i);
+ }
+ } else {
+ notSkipIDs = ColumnProjectionUtils.getReadColumnIDs(conf);
+ }
+ cachedLazyStruct = new LazyBinaryColumnarStruct(cachedObjectInspector, notSkipIDs);
+
super.initialize(size);
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java?rev=1525322&r1=1525321&r2=1525322&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java Sun Sep 22 06:10:15 2013
@@ -18,26 +18,22 @@
package org.apache.hadoop.hive.serde2.columnar;
-import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
-import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils;
-import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
public class LazyBinaryColumnarStruct extends ColumnarStructBase {
- public LazyBinaryColumnarStruct(ObjectInspector oi, ArrayList<Integer> notSkippedColumnIDs) {
+ public LazyBinaryColumnarStruct(ObjectInspector oi, List<Integer> notSkippedColumnIDs) {
super(oi, notSkippedColumnIDs);
}
- static VInt vInt = new LazyBinaryUtils.VInt();
-
@Override
protected int getLength(ObjectInspector objectInspector, ByteArrayRef cachedByteArrayRef,
int start, int length) {
@@ -48,8 +44,8 @@ public class LazyBinaryColumnarStruct ex
if (category.equals(Category.PRIMITIVE)) {
PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) objectInspector)
.getPrimitiveCategory();
- if (primitiveCategory.equals(PrimitiveCategory.STRING) && (length == 1) &&
- (cachedByteArrayRef.getData()[start]
+ if (primitiveCategory.equals(PrimitiveCategory.STRING) && (length == 1) &&
+ (cachedByteArrayRef.getData()[start]
== LazyBinaryColumnarSerDe.INVALID_UTF__SINGLE_BYTE[0])) {
return 0;
}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java?rev=1525322&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java Sun Sep 22 06:10:15 2013
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestColumnProjectionUtils {
+
+ private Configuration conf;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ }
+
+ @Test
+ public void testReadAllColumns() {
+ // test that the all columns will be read by default
+ assertTrue(ColumnProjectionUtils.isReadAllColumns(conf));
+ // test that setting read all resets column ids
+ ColumnProjectionUtils.appendReadColumns(conf, Arrays.asList(new Integer[]{0, 1, 2}));
+ ColumnProjectionUtils.setReadAllColumns(conf);
+ assertEquals(Collections.EMPTY_LIST, ColumnProjectionUtils.getReadColumnIDs(conf));
+ assertTrue(ColumnProjectionUtils.isReadAllColumns(conf));
+ }
+
+ @Test
+ public void testReadColumnIds() {
+ List<Integer> columnIds = new ArrayList<Integer>();
+ List<Integer> actual;
+
+ assertEquals(Collections.EMPTY_LIST, ColumnProjectionUtils.getReadColumnIDs(conf));
+
+ // test that read columns are initially an empty list
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ assertEquals(Collections.EMPTY_LIST, actual);
+ // set setting read column ids with an empty list
+ ColumnProjectionUtils.appendReadColumns(conf, columnIds);
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ assertEquals(Collections.EMPTY_LIST, actual);
+ // test that setting read column ids set read all columns to false
+ assertFalse(ColumnProjectionUtils.isReadAllColumns(conf));
+ // add needed columns
+ columnIds.add(1);
+ ColumnProjectionUtils.appendReadColumns(conf, Collections.singletonList(1));
+ assertEquals(columnIds, ColumnProjectionUtils.getReadColumnIDs(conf));
+ columnIds.add(2);
+ ColumnProjectionUtils.appendReadColumns(conf, Collections.singletonList(2));
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ Collections.sort(actual);
+ assertEquals(columnIds, actual);
+ columnIds.add(3);
+ ColumnProjectionUtils.appendReadColumns(conf, Collections.singletonList(3));
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ Collections.sort(actual);
+ assertEquals(columnIds, actual);
+ assertFalse(ColumnProjectionUtils.isReadAllColumns(conf));
+ }
+}