You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/10 23:41:17 UTC
svn commit: r1624140 [3/3] - in /hive/branches/cbo: ./
common/src/java/org/apache/hadoop/hive/conf/ hbase-handler/
hbase-handler/if/ hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/ hbase-...
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Wed Sep 10 21:41:16 2014
@@ -346,48 +346,6 @@ public class ColumnStatsSemanticAnalyzer
return rewrittenTree;
}
- public ColumnStatsSemanticAnalyzer(HiveConf conf, ASTNode tree) throws SemanticException {
- super(conf, false);
- // check if it is no scan. grammar prevents coexit noscan/columns
- super.processNoScanCommand(tree);
- // check if it is partial scan. grammar prevents coexit partialscan/columns
- super.processPartialScanCommand(tree);
- /* Rewrite only analyze table <> column <> compute statistics; Don't rewrite analyze table
- * command - table stats are collected by the table scan operator and is not rewritten to
- * an aggregation.
- */
- if (shouldRewrite(tree)) {
- tbl = getTable(tree);
- colNames = getColumnName(tree);
- // Save away the original AST
- originalTree = tree;
- boolean isPartitionStats = isPartitionLevelStats(tree);
- Map<String,String> partSpec = null;
- checkForPartitionColumns(colNames, Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()));
- validateSpecifiedColumnNames(colNames);
- if (conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && tbl.isPartitioned()) {
- isPartitionStats = true;
- }
-
- if (isPartitionStats) {
- isTableLevel = false;
- partSpec = getPartKeyValuePairsFromAST(tree);
- handlePartialPartitionSpec(partSpec);
- } else {
- isTableLevel = true;
- }
- colType = getColumnTypes(colNames);
- int numBitVectors = getNumBitVectorsForNDVEstimation(conf);
- rewrittenQuery = genRewrittenQuery(colNames, numBitVectors, partSpec, isPartitionStats);
- rewrittenTree = genRewrittenTree(rewrittenQuery);
- } else {
- // Not an analyze table column compute statistics statement - don't do any rewrites
- originalTree = rewrittenTree = tree;
- rewrittenQuery = null;
- isRewritten = false;
- }
- }
-
// fail early if the columns specified for column statistics are not valid
private void validateSpecifiedColumnNames(List<String> specifiedCols)
throws SemanticException {
@@ -421,6 +379,46 @@ public class ColumnStatsSemanticAnalyzer
// initialize QB
init(true);
+ // check if it is no scan. grammar prevents coexit noscan/columns
+ super.processNoScanCommand(ast);
+ // check if it is partial scan. grammar prevents coexit partialscan/columns
+ super.processPartialScanCommand(ast);
+ /* Rewrite only analyze table <> column <> compute statistics; Don't rewrite analyze table
+ * command - table stats are collected by the table scan operator and is not rewritten to
+ * an aggregation.
+ */
+ if (shouldRewrite(ast)) {
+ tbl = getTable(ast);
+ colNames = getColumnName(ast);
+ // Save away the original AST
+ originalTree = ast;
+ boolean isPartitionStats = isPartitionLevelStats(ast);
+ Map<String,String> partSpec = null;
+ checkForPartitionColumns(
+ colNames, Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()));
+ validateSpecifiedColumnNames(colNames);
+ if (conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && tbl.isPartitioned()) {
+ isPartitionStats = true;
+ }
+
+ if (isPartitionStats) {
+ isTableLevel = false;
+ partSpec = getPartKeyValuePairsFromAST(ast);
+ handlePartialPartitionSpec(partSpec);
+ } else {
+ isTableLevel = true;
+ }
+ colType = getColumnTypes(colNames);
+ int numBitVectors = getNumBitVectorsForNDVEstimation(conf);
+ rewrittenQuery = genRewrittenQuery(colNames, numBitVectors, partSpec, isPartitionStats);
+ rewrittenTree = genRewrittenTree(rewrittenQuery);
+ } else {
+ // Not an analyze table column compute statistics statement - don't do any rewrites
+ originalTree = rewrittenTree = ast;
+ rewrittenQuery = null;
+ isRewritten = false;
+ }
+
// Setup the necessary metadata if originating from analyze rewrite
if (isRewritten) {
qb = getQB();
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GlobalLimitCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GlobalLimitCtx.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GlobalLimitCtx.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GlobalLimitCtx.java Wed Sep 10 21:41:16 2014
@@ -25,10 +25,14 @@ import org.apache.hadoop.hive.ql.plan.Li
*/
public class GlobalLimitCtx {
- private boolean enable = false;
- private int globalLimit = -1;
- private boolean hasTransformOrUDTF = false;
- private LimitDesc lastReduceLimitDesc = null;
+ private boolean enable;
+ private int globalLimit;
+ private boolean hasTransformOrUDTF;
+ private LimitDesc lastReduceLimitDesc;
+
+ public GlobalLimitCtx() {
+ reset();
+ }
public int getGlobalLimit() {
return globalLimit;
@@ -64,4 +68,11 @@ public class GlobalLimitCtx {
this.globalLimit = -1;
this.lastReduceLimitDesc = null;
}
+
+ public void reset() {
+ enable = false;
+ globalLimit = -1;
+ hasTransformOrUDTF = false;
+ lastReduceLimitDesc = null;
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Sep 10 21:41:16 2014
@@ -293,6 +293,8 @@ public class SemanticAnalyzer extends Ba
public static final String DUMMY_DATABASE = "_dummy_database";
public static final String DUMMY_TABLE = "_dummy_table";
+ // Max characters when auto generating the column name with func name
+ private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20;
private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
@@ -315,7 +317,7 @@ public class SemanticAnalyzer extends Ba
private HashMap<TableScanOperator, sampleDesc> opToSamplePruner;
private final Map<TableScanOperator, Map<String, ExprNodeDesc>> opToPartToSkewedPruner;
/**
- * a map for the split sampling, from ailias to an instance of SplitSample
+ * a map for the split sampling, from alias to an instance of SplitSample
* that describes percentage and number.
*/
private final HashMap<String, SplitSample> nameToSplitSample;
@@ -326,7 +328,7 @@ public class SemanticAnalyzer extends Ba
private ArrayList<String> viewsExpanded;
private ASTNode viewSelect;
private final UnparseTranslator unparseTranslator;
- private final GlobalLimitCtx globalLimitCtx = new GlobalLimitCtx();
+ private final GlobalLimitCtx globalLimitCtx;
// prefix for column names auto generated by hive
private final String autogenColAliasPrfxLbl;
@@ -337,16 +339,13 @@ public class SemanticAnalyzer extends Ba
// keeps track of aliases for V3, V3:V2, V3:V2:V1.
// This is used when T is added as an input for the query, the parents of T is
// derived from the alias V3:V2:V1:T
- private final Map<String, ReadEntity> viewAliasToInput = new HashMap<String, ReadEntity>();
-
- // Max characters when auto generating the column name with func name
- private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20;
+ private final Map<String, ReadEntity> viewAliasToInput;
// flag for no scan during analyze ... compute statistics
- protected boolean noscan = false;
+ protected boolean noscan;
//flag for partial scan during analyze ... compute statistics
- protected boolean partialscan = false;
+ protected boolean partialscan;
private volatile boolean runCBO = true;
private volatile boolean disableJoinMerge = false;
@@ -371,7 +370,6 @@ public class SemanticAnalyzer extends Ba
}
public SemanticAnalyzer(HiveConf conf) throws SemanticException {
-
super(conf);
opToPartPruner = new HashMap<TableScanOperator, ExprNodeDesc>();
opToPartList = new HashMap<TableScanOperator, PrunedPartitionList>();
@@ -401,6 +399,9 @@ public class SemanticAnalyzer extends Ba
queryProperties = new QueryProperties();
opToPartToSkewedPruner = new HashMap<TableScanOperator, Map<String, ExprNodeDesc>>();
aliasToCTEs = new HashMap<String, ASTNode>();
+ globalLimitCtx = new GlobalLimitCtx();
+ viewAliasToInput = new HashMap<String, ReadEntity>();
+ noscan = partialscan = false;
}
@Override
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Wed Sep 10 21:41:16 2014
@@ -263,7 +263,7 @@ public final class SemanticAnalyzerFacto
return new FunctionSemanticAnalyzer(conf);
case HiveParser.TOK_ANALYZE:
- return new ColumnStatsSemanticAnalyzer(conf, tree);
+ return new ColumnStatsSemanticAnalyzer(conf);
case HiveParser.TOK_CREATEMACRO:
case HiveParser.TOK_DROPMACRO:
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java Wed Sep 10 21:41:16 2014
@@ -69,7 +69,11 @@ public class GroupByDesc extends Abstrac
transient private boolean isDistinct;
private boolean dontResetAggrsDistinct;
+ // Extra parameters only for vectorization.
+ private VectorGroupByDesc vectorDesc;
+
public GroupByDesc() {
+ vectorDesc = new VectorGroupByDesc();
}
public GroupByDesc(
@@ -102,6 +106,7 @@ public class GroupByDesc extends Abstrac
final boolean groupingSetsPresent,
final int groupingSetsPosition,
final boolean isDistinct) {
+ vectorDesc = new VectorGroupByDesc();
this.mode = mode;
this.outputColumnNames = outputColumnNames;
this.keys = keys;
@@ -116,6 +121,14 @@ public class GroupByDesc extends Abstrac
this.isDistinct = isDistinct;
}
+ public void setVectorDesc(VectorGroupByDesc vectorDesc) {
+ this.vectorDesc = vectorDesc;
+ }
+
+ public VectorGroupByDesc getVectorDesc() {
+ return vectorDesc;
+ }
+
public Mode getMode() {
return mode;
}
@@ -268,6 +281,14 @@ public class GroupByDesc extends Abstrac
this.groupingSetPosition = groupingSetPosition;
}
+ public boolean isDontResetAggrsDistinct() {
+ return dontResetAggrsDistinct;
+ }
+
+ public void setDontResetAggrsDistinct(boolean dontResetAggrsDistinct) {
+ this.dontResetAggrsDistinct = dontResetAggrsDistinct;
+ }
+
public boolean isDistinct() {
return isDistinct;
}
@@ -276,11 +297,4 @@ public class GroupByDesc extends Abstrac
this.isDistinct = isDistinct;
}
- public boolean isDontResetAggrsDistinct() {
- return dontResetAggrsDistinct;
- }
-
- public void setDontResetAggrsDistinct(boolean dontResetAggrsDistinct) {
- this.dontResetAggrsDistinct = dontResetAggrsDistinct;
- }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Wed Sep 10 21:41:16 2014
@@ -470,10 +470,7 @@ public class SessionState {
*/
private void createSessionDirs(String userName) throws IOException {
HiveConf conf = getConf();
- // First create the root scratch dir on hdfs (if it doesn't already exist) and make it writable
- Path rootHDFSDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR));
- String rootHDFSDirPermission = "777";
- createPath(conf, rootHDFSDirPath, rootHDFSDirPermission, false, false);
+ Path rootHDFSDirPath = createRootHDFSDir(conf);
// Now create session specific dirs
String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
Path path;
@@ -506,6 +503,30 @@ public class SessionState {
}
/**
+ * Create the root scratch dir on hdfs (if it doesn't already exist) and make it writable
+ * @param conf
+ * @return
+ * @throws IOException
+ */
+ private Path createRootHDFSDir(HiveConf conf) throws IOException {
+ Path rootHDFSDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR));
+ FsPermission expectedHDFSDirPermission = new FsPermission("777");
+ FileSystem fs = rootHDFSDirPath.getFileSystem(conf);
+ if (!fs.exists(rootHDFSDirPath)) {
+ Utilities.createDirsWithPermission(conf, rootHDFSDirPath, expectedHDFSDirPermission, true);
+ }
+ FsPermission currentHDFSDirPermission = fs.getFileStatus(rootHDFSDirPath).getPermission();
+ LOG.debug("HDFS root scratch dir: " + rootHDFSDirPath + ", permission: "
+ + currentHDFSDirPermission);
+ // If the root HDFS scratch dir already exists, make sure the permissions are 777.
+ if (!expectedHDFSDirPermission.equals(fs.getFileStatus(rootHDFSDirPath).getPermission())) {
+ throw new RuntimeException("The root scratch dir: " + rootHDFSDirPath
+ + " on HDFS should be writable. Current permissions are: " + currentHDFSDirPermission);
+ }
+ return rootHDFSDirPath;
+ }
+
+ /**
* Create a given path if it doesn't exist.
*
* @param conf
@@ -624,10 +645,10 @@ public class SessionState {
authorizerV2 = authorizerFactory.createHiveAuthorizer(new HiveMetastoreClientFactoryImpl(),
conf, authenticator, authzContextBuilder.build());
- authorizerV2.applyAuthorizationConfigPolicy(conf);
- // create the create table grants with new config
- createTableGrants = CreateTableAutomaticGrant.create(conf);
+ authorizerV2.applyAuthorizationConfigPolicy(conf);
}
+ // create the create table grants with new config
+ createTableGrants = CreateTableAutomaticGrant.create(conf);
} catch (HiveException e) {
throw new RuntimeException(e);
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java Wed Sep 10 21:41:16 2014
@@ -107,7 +107,7 @@ public class TestVectorizer {
gbyOp.setConf(desc);
Vectorizer v = new Vectorizer();
- Assert.assertTrue(v.validateMapWorkOperator(gbyOp));
+ Assert.assertTrue(v.validateMapWorkOperator(gbyOp, false));
VectorGroupByOperator vectorOp = (VectorGroupByOperator) v.vectorizeOperator(gbyOp, vContext);
Assert.assertEquals(VectorUDAFSumLong.class, vectorOp.getAggregators()[0].getClass());
VectorUDAFSumLong udaf = (VectorUDAFSumLong) vectorOp.getAggregators()[0];
@@ -150,7 +150,7 @@ public class TestVectorizer {
/**
* prepareAbstractMapJoin prepares a join operator descriptor, used as helper by SMB and Map join tests.
*/
- private void prepareAbstractMapJoin(AbstractMapJoinOperator<? extends MapJoinDesc> mop, MapJoinDesc mjdesc) {
+ private void prepareAbstractMapJoin(AbstractMapJoinOperator<? extends MapJoinDesc> map, MapJoinDesc mjdesc) {
mjdesc.setPosBigTable(0);
List<ExprNodeDesc> expr = new ArrayList<ExprNodeDesc>();
expr.add(new ExprNodeColumnDesc(Integer.class, "col1", "T", false));
@@ -180,14 +180,14 @@ public class TestVectorizer {
*/
@Test
public void testValidateMapJoinOperator() {
- MapJoinOperator mop = new MapJoinOperator();
+ MapJoinOperator map = new MapJoinOperator();
MapJoinDesc mjdesc = new MapJoinDesc();
- prepareAbstractMapJoin(mop, mjdesc);
- mop.setConf(mjdesc);
+ prepareAbstractMapJoin(map, mjdesc);
+ map.setConf(mjdesc);
Vectorizer vectorizer = new Vectorizer();
- Assert.assertTrue(vectorizer.validateMapWorkOperator(mop));
+ Assert.assertTrue(vectorizer.validateMapWorkOperator(map, false));
}
@@ -196,13 +196,13 @@ public class TestVectorizer {
*/
@Test
public void testValidateSMBJoinOperator() {
- SMBMapJoinOperator mop = new SMBMapJoinOperator();
+ SMBMapJoinOperator map = new SMBMapJoinOperator();
SMBJoinDesc mjdesc = new SMBJoinDesc();
- prepareAbstractMapJoin(mop, mjdesc);
- mop.setConf(mjdesc);
+ prepareAbstractMapJoin(map, mjdesc);
+ map.setConf(mjdesc);
Vectorizer vectorizer = new Vectorizer();
- Assert.assertTrue(vectorizer.validateMapWorkOperator(mop));
+ Assert.assertTrue(vectorizer.validateMapWorkOperator(map, false));
}
}
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q Wed Sep 10 21:41:16 2014
@@ -150,7 +150,9 @@ insert overwrite table over1k_part_buck_
desc formatted over1k_part_buck_sort2_orc partition(t=27);
desc formatted over1k_part_buck_sort2_orc partition(t="__HIVE_DEFAULT_PARTITION__");
+explain select * from over1k_part_buck_sort2_orc;
select * from over1k_part_buck_sort2_orc;
+explain select count(*) from over1k_part_buck_sort2_orc;
select count(*) from over1k_part_buck_sort2_orc;
set hive.optimize.sort.dynamic.partition=true;
@@ -159,5 +161,7 @@ insert overwrite table over1k_part_buck_
desc formatted over1k_part_buck_sort2_orc partition(t=27);
desc formatted over1k_part_buck_sort2_orc partition(t="__HIVE_DEFAULT_PARTITION__");
+explain select * from over1k_part_buck_sort2_orc;
select * from over1k_part_buck_sort2_orc;
+explain select count(*) from over1k_part_buck_sort2_orc;
select count(*) from over1k_part_buck_sort2_orc;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/input_lazyserde.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/input_lazyserde.q?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/input_lazyserde.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/input_lazyserde.q Wed Sep 10 21:41:16 2014
@@ -30,3 +30,7 @@ CREATE TABLE dest1(a map<string,string>)
INSERT OVERWRITE TABLE dest1 SELECT src_thrift.mstringstring FROM src_thrift DISTRIBUTE BY 1;
SELECT * from dest1;
+CREATE TABLE destBin(a UNIONTYPE<int, double, array<string>, struct<col1:int,col2:string>>) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' STORED AS SEQUENCEFILE;
+INSERT OVERWRITE TABLE destBin SELECT create_union( CASE WHEN key < 100 THEN 0 WHEN key < 200 THEN 1 WHEN key < 300 THEN 2 WHEN key < 400 THEN 3 ELSE 0 END, key, 2.0, array("one","two"), struct(5,"five")) FROM srcbucket2;
+SELECT * from destBin ORDER BY a;
+DROP TABLE destBin;
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out Wed Sep 10 21:41:16 2014
@@ -1951,6 +1951,27 @@ Bucket Columns: [si]
Sort Columns: [Order(col:f, order:1)]
Storage Desc Params:
serialization.format 1
+PREHOOK: query: explain select * from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
PREHOOK: query: select * from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
@@ -1982,6 +2003,56 @@ POSTHOOK: Input: default@over1k_part_buc
409 65536 4294967490 46.97 NULL
374 65560 4294967516 65.43 NULL
473 65720 4294967324 80.74 NULL
+PREHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: select count(*) from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
@@ -2098,6 +2169,27 @@ Bucket Columns: [si]
Sort Columns: [Order(col:f, order:1)]
Storage Desc Params:
serialization.format 1
+PREHOOK: query: explain select * from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
PREHOOK: query: select * from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
@@ -2129,6 +2221,56 @@ POSTHOOK: Input: default@over1k_part_buc
409 65536 4294967490 46.97 NULL
374 65560 4294967516 65.43 NULL
473 65720 4294967324 80.74 NULL
+PREHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: select count(*) from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/input_lazyserde.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/input_lazyserde.q.out?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/input_lazyserde.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/input_lazyserde.q.out Wed Sep 10 21:41:16 2014
@@ -219,3 +219,536 @@ NULL
{"key_7":"value_7"}
{"key_8":"value_8"}
{"key_9":"value_9"}
+PREHOOK: query: CREATE TABLE destBin(a UNIONTYPE<int, double, array<string>, struct<col1:int,col2:string>>) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@destBin
+POSTHOOK: query: CREATE TABLE destBin(a UNIONTYPE<int, double, array<string>, struct<col1:int,col2:string>>) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@destBin
+PREHOOK: query: INSERT OVERWRITE TABLE destBin SELECT create_union( CASE WHEN key < 100 THEN 0 WHEN key < 200 THEN 1 WHEN key < 300 THEN 2 WHEN key < 400 THEN 3 ELSE 0 END, key, 2.0, array("one","two"), struct(5,"five")) FROM srcbucket2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket2
+PREHOOK: Output: default@destbin
+POSTHOOK: query: INSERT OVERWRITE TABLE destBin SELECT create_union( CASE WHEN key < 100 THEN 0 WHEN key < 200 THEN 1 WHEN key < 300 THEN 2 WHEN key < 400 THEN 3 ELSE 0 END, key, 2.0, array("one","two"), struct(5,"five")) FROM srcbucket2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket2
+POSTHOOK: Output: default@destbin
+POSTHOOK: Lineage: destbin.a EXPRESSION [(srcbucket2)srcbucket2.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: SELECT * from destBin ORDER BY a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@destbin
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from destBin ORDER BY a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@destbin
+#### A masked pattern was here ####
+{0:0}
+{0:0}
+{0:0}
+{0:10}
+{0:11}
+{0:12}
+{0:12}
+{0:15}
+{0:15}
+{0:17}
+{0:18}
+{0:18}
+{0:19}
+{0:20}
+{0:24}
+{0:24}
+{0:26}
+{0:26}
+{0:27}
+{0:28}
+{0:2}
+{0:30}
+{0:33}
+{0:34}
+{0:35}
+{0:35}
+{0:35}
+{0:37}
+{0:37}
+{0:400}
+{0:401}
+{0:401}
+{0:401}
+{0:401}
+{0:401}
+{0:402}
+{0:403}
+{0:403}
+{0:403}
+{0:404}
+{0:404}
+{0:406}
+{0:406}
+{0:406}
+{0:406}
+{0:407}
+{0:409}
+{0:409}
+{0:409}
+{0:411}
+{0:413}
+{0:413}
+{0:414}
+{0:414}
+{0:417}
+{0:417}
+{0:417}
+{0:418}
+{0:419}
+{0:41}
+{0:421}
+{0:424}
+{0:424}
+{0:427}
+{0:429}
+{0:429}
+{0:42}
+{0:42}
+{0:430}
+{0:430}
+{0:430}
+{0:431}
+{0:431}
+{0:431}
+{0:432}
+{0:435}
+{0:436}
+{0:437}
+{0:438}
+{0:438}
+{0:438}
+{0:439}
+{0:439}
+{0:43}
+{0:443}
+{0:444}
+{0:446}
+{0:448}
+{0:449}
+{0:44}
+{0:452}
+{0:453}
+{0:454}
+{0:454}
+{0:454}
+{0:455}
+{0:457}
+{0:458}
+{0:458}
+{0:459}
+{0:459}
+{0:460}
+{0:462}
+{0:462}
+{0:463}
+{0:463}
+{0:466}
+{0:466}
+{0:466}
+{0:467}
+{0:468}
+{0:468}
+{0:468}
+{0:468}
+{0:469}
+{0:469}
+{0:469}
+{0:469}
+{0:469}
+{0:470}
+{0:472}
+{0:475}
+{0:477}
+{0:478}
+{0:478}
+{0:479}
+{0:47}
+{0:480}
+{0:480}
+{0:480}
+{0:481}
+{0:482}
+{0:483}
+{0:484}
+{0:485}
+{0:487}
+{0:489}
+{0:489}
+{0:489}
+{0:489}
+{0:490}
+{0:491}
+{0:492}
+{0:492}
+{0:493}
+{0:494}
+{0:495}
+{0:496}
+{0:497}
+{0:498}
+{0:498}
+{0:498}
+{0:4}
+{0:51}
+{0:51}
+{0:53}
+{0:54}
+{0:57}
+{0:58}
+{0:58}
+{0:5}
+{0:5}
+{0:5}
+{0:64}
+{0:65}
+{0:66}
+{0:67}
+{0:67}
+{0:69}
+{0:70}
+{0:70}
+{0:70}
+{0:72}
+{0:72}
+{0:74}
+{0:76}
+{0:76}
+{0:77}
+{0:78}
+{0:80}
+{0:82}
+{0:83}
+{0:83}
+{0:84}
+{0:84}
+{0:85}
+{0:86}
+{0:87}
+{0:8}
+{0:90}
+{0:90}
+{0:90}
+{0:92}
+{0:95}
+{0:95}
+{0:96}
+{0:97}
+{0:97}
+{0:98}
+{0:98}
+{0:9}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+PREHOOK: query: DROP TABLE destBin
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@destbin
+PREHOOK: Output: default@destbin
+POSTHOOK: query: DROP TABLE destBin
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@destbin
+POSTHOOK: Output: default@destbin
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out Wed Sep 10 21:41:16 2014
@@ -2026,6 +2026,25 @@ Bucket Columns: [si]
Sort Columns: [Order(col:f, order:1)]
Storage Desc Params:
serialization.format 1
+PREHOOK: query: explain select * from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Select Operator
+ expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ ListSink
+
PREHOOK: query: select * from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
@@ -2057,6 +2076,63 @@ POSTHOOK: Input: default@over1k_part_buc
409 65536 4294967490 46.97 NULL
374 65560 4294967516 65.43 NULL
473 65720 4294967324 80.74 NULL
+PREHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: select count(*) from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
@@ -2173,6 +2249,25 @@ Bucket Columns: [si]
Sort Columns: [Order(col:f, order:1)]
Storage Desc Params:
serialization.format 1
+PREHOOK: query: explain select * from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Select Operator
+ expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ ListSink
+
PREHOOK: query: select * from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
@@ -2204,6 +2299,63 @@ POSTHOOK: Input: default@over1k_part_buc
409 65536 4294967490 46.97 NULL
374 65560 4294967516 65.43 NULL
473 65720 4294967324 80.74 NULL
+PREHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(*) from over1k_part_buck_sort2_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k_part_buck_sort2_orc
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: select count(*) from over1k_part_buck_sort2_orc
PREHOOK: type: QUERY
PREHOOK: Input: default@over1k_part_buck_sort2_orc
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out Wed Sep 10 21:41:16 2014
@@ -103,6 +103,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out Wed Sep 10 21:41:16 2014
@@ -106,6 +106,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out Wed Sep 10 21:41:16 2014
@@ -114,6 +114,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
Files hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out Wed Sep 10 21:41:16 2014 differ
Modified: hive/branches/cbo/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java (original)
+++ hive/branches/cbo/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java Wed Sep 10 21:41:16 2014
@@ -37,6 +37,8 @@ public class serdeConstants {
public static final String SERIALIZATION_CLASS = "serialization.class";
+ public static final String SERIALIZATION_TYPE = "serialization.type";
+
public static final String SERIALIZATION_FORMAT = "serialization.format";
public static final String SERIALIZATION_DDL = "serialization.ddl";
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java Wed Sep 10 21:41:16 2014
@@ -48,7 +48,8 @@ public interface PredicateLeaf {
STRING, // string, char, varchar
DATE,
DECIMAL,
- TIMESTAMP
+ TIMESTAMP,
+ BOOLEAN
}
/**
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroGenericRecordWritable.java Wed Sep 10 21:41:16 2014
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.serde2.avro;
+import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.DataOutputStream;
@@ -25,6 +26,7 @@ import java.io.InputStream;
import java.rmi.server.UID;
import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileStream;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
@@ -101,6 +103,28 @@ public class AvroGenericRecordWritable i
GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(schema);
record = gdr.read(record, binaryDecoder);
}
+
+ public void readFields(byte[] bytes, int offset, int length, Schema writerSchema, Schema readerSchema) throws IOException {
+ fileSchema = writerSchema;
+ record = new GenericData.Record(writerSchema);
+ binaryDecoder =
+ DecoderFactory.get().binaryDecoder(bytes, offset, length - offset,
+ binaryDecoder);
+ GenericDatumReader<GenericRecord> gdr =
+ new GenericDatumReader<GenericRecord>(writerSchema, readerSchema);
+ record = gdr.read(null, binaryDecoder);
+ }
+
+ public void readFields(byte[] bytes, Schema writerSchema, Schema readerSchema) throws IOException {
+ fileSchema = writerSchema;
+ record = new GenericData.Record(writerSchema);
+ GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>();
+ gdr.setExpected(readerSchema);
+ ByteArrayInputStream is = new ByteArrayInputStream(bytes);
+ DataFileStream<GenericRecord> dfr = new DataFileStream<GenericRecord>(is, gdr);
+ record = dfr.next(record);
+ dfr.close();
+ }
public UID getRecordReaderID() {
return recordReaderID;
@@ -117,5 +141,4 @@ public class AvroGenericRecordWritable i
public void setFileSchema(Schema originalSchema) {
this.fileSchema = originalSchema;
}
-
}
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java Wed Sep 10 21:41:16 2014
@@ -40,7 +40,7 @@ import org.apache.hadoop.hive.serde2.typ
* * A list of those fields equivalent types in Hive
* * An ObjectInspector capable of working with an instance of that datum.
*/
-class AvroObjectInspectorGenerator {
+public class AvroObjectInspectorGenerator {
final private List<String> columnNames;
final private List<TypeInfo> columnTypes;
final private ObjectInspector oi;
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java Wed Sep 10 21:41:16 2014
@@ -57,6 +57,7 @@ public class AvroSerdeUtils {
public static final String EXCEPTION_MESSAGE = "Neither " + SCHEMA_LITERAL + " nor "
+ SCHEMA_URL + " specified, can't determine table schema";
public static final String AVRO_SERDE_SCHEMA = "avro.serde.schema";
+ public static final String SCHEMA_RETRIEVER = "avro.schema.retriever";
/**
* Determine the schema to that's been provided for Avro serde work.
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java Wed Sep 10 21:41:16 2014
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.serde2.laz
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
@@ -217,11 +218,11 @@ public final class LazyFactory {
*/
public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo,
byte[] separator, int separatorIndex, Text nullSequence, boolean escaped,
- byte escapeChar) throws SerDeException {
+ byte escapeChar, ObjectInspectorOptions option) throws SerDeException {
return createLazyObjectInspector(typeInfo, separator, separatorIndex, nullSequence,
- escaped, escapeChar, false);
+ escaped, escapeChar, false, option);
}
-
+
/**
* Create a hierarchical ObjectInspector for LazyObject with the given
* typeInfo.
@@ -236,13 +237,54 @@ public final class LazyFactory {
* delimiting entries, the second one for delimiting key and values.
* @param nullSequence
* The sequence of bytes representing NULL.
+ * @return The ObjectInspector
+ * @throws SerDeException
+ */
+ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo,
+ byte[] separator, int separatorIndex, Text nullSequence, boolean escaped,
+ byte escapeChar) throws SerDeException {
+ return createLazyObjectInspector(typeInfo, separator, separatorIndex, nullSequence,
+ escaped, escapeChar, false, ObjectInspectorOptions.JAVA);
+ }
+
+ /**
+ * Create a hierarchical ObjectInspector for LazyObject with the given typeInfo.
+ *
+ * @param typeInfo The type information for the LazyObject
+ * @param separator The array of separators for delimiting each level
+ * @param separatorIndex The current level (for separators). List(array), struct uses 1 level of
+ * separator, and map uses 2 levels: the first one for delimiting entries, the second one
+ * for delimiting key and values.
+ * @param nullSequence The sequence of bytes representing NULL.
* @param extendedBooleanLiteral whether extended boolean literal set is legal
+ * @param option the {@link ObjectInspectorOption}
* @return The ObjectInspector
* @throws SerDeException
*/
public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo,
byte[] separator, int separatorIndex, Text nullSequence, boolean escaped,
byte escapeChar, boolean extendedBooleanLiteral) throws SerDeException {
+ return createLazyObjectInspector(typeInfo, separator, separatorIndex, nullSequence, escaped,
+ escapeChar, extendedBooleanLiteral, ObjectInspectorOptions.JAVA);
+ }
+
+ /**
+ * Create a hierarchical ObjectInspector for LazyObject with the given typeInfo.
+ *
+ * @param typeInfo The type information for the LazyObject
+ * @param separator The array of separators for delimiting each level
+ * @param separatorIndex The current level (for separators). List(array), struct uses 1 level of
+ * separator, and map uses 2 levels: the first one for delimiting entries, the second one
+ * for delimiting key and values.
+ * @param nullSequence The sequence of bytes representing NULL.
+ * @param extendedBooleanLiteral whether extended boolean literal set is legal
+ * @param option the {@link ObjectInspectorOption}
+ * @return The ObjectInspector
+ * @throws SerDeException
+ */
+ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo,
+ byte[] separator, int separatorIndex, Text nullSequence, boolean escaped,
+ byte escapeChar, boolean extendedBooleanLiteral, ObjectInspectorOptions option) throws SerDeException {
ObjectInspector.Category c = typeInfo.getCategory();
switch (c) {
case PRIMITIVE:
@@ -252,9 +294,9 @@ public final class LazyFactory {
return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(
createLazyObjectInspector(((MapTypeInfo) typeInfo)
.getMapKeyTypeInfo(), separator, separatorIndex + 2,
- nullSequence, escaped, escapeChar, extendedBooleanLiteral), createLazyObjectInspector(
+ nullSequence, escaped, escapeChar, extendedBooleanLiteral, option), createLazyObjectInspector(
((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separator,
- separatorIndex + 2, nullSequence, escaped, escapeChar, extendedBooleanLiteral),
+ separatorIndex + 2, nullSequence, escaped, escapeChar, extendedBooleanLiteral, option),
LazyUtils.getSeparator(separator, separatorIndex),
LazyUtils.getSeparator(separator, separatorIndex+1),
nullSequence, escaped, escapeChar);
@@ -262,7 +304,7 @@ public final class LazyFactory {
return LazyObjectInspectorFactory.getLazySimpleListObjectInspector(
createLazyObjectInspector(((ListTypeInfo) typeInfo)
.getListElementTypeInfo(), separator, separatorIndex + 1,
- nullSequence, escaped, escapeChar, extendedBooleanLiteral), LazyUtils.getSeparator(separator, separatorIndex),
+ nullSequence, escaped, escapeChar, extendedBooleanLiteral, option), LazyUtils.getSeparator(separator, separatorIndex),
nullSequence, escaped, escapeChar);
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
@@ -274,19 +316,20 @@ public final class LazyFactory {
for (int i = 0; i < fieldTypeInfos.size(); i++) {
fieldObjectInspectors.add(createLazyObjectInspector(fieldTypeInfos
.get(i), separator, separatorIndex + 1, nullSequence, escaped,
- escapeChar, extendedBooleanLiteral));
+ escapeChar, extendedBooleanLiteral, option));
}
return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
fieldNames, fieldObjectInspectors,
LazyUtils.getSeparator(separator, separatorIndex),
- nullSequence, false, escaped, escapeChar);
+ nullSequence,
+ false, escaped, escapeChar, option);
case UNION:
UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
List<ObjectInspector> lazyOIs = new ArrayList<ObjectInspector>();
for (TypeInfo uti : unionTypeInfo.getAllUnionObjectTypeInfos()) {
lazyOIs.add(createLazyObjectInspector(uti, separator,
separatorIndex + 1, nullSequence, escaped,
- escapeChar, extendedBooleanLiteral));
+ escapeChar, extendedBooleanLiteral, option));
}
return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs,
LazyUtils.getSeparator(separator, separatorIndex),
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java Wed Sep 10 21:41:16 2014
@@ -342,4 +342,13 @@ public class LazyStruct extends LazyNonP
}
return indexes;
}
-}
+
+ /**
+ * Return the data in bytes corresponding to this given struct. This is useful specifically in
+ * cases where the data is stored in serialized formats like protobufs or thrift and would need
+ * custom deserializers to be deserialized.
+ * */
+ public byte[] getBytes() {
+ return bytes.getData();
+ }
+}
\ No newline at end of file
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUnion.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUnion.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUnion.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUnion.java Wed Sep 10 21:41:16 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.serde2.lazy;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.Text;
/**
@@ -26,8 +25,7 @@ import org.apache.hadoop.io.Text;
* non-primitive.
*
*/
-public class LazyUnion extends
- LazyNonPrimitive<LazyUnionObjectInspector> {
+public class LazyUnion extends LazyNonPrimitive<LazyUnionObjectInspector> {
/**
* Whether the data is already parsed or not.
*/
@@ -41,7 +39,7 @@ public class LazyUnion extends
/**
* The object of the union.
*/
- private LazyObject<? extends ObjectInspector> field;
+ private Object field;
/**
* Tag of the Union
@@ -54,6 +52,16 @@ public class LazyUnion extends
private boolean fieldInited = false;
/**
+ * Whether the tag has been set or not
+ * */
+ private boolean tagSet = false;
+
+ /**
+ * Whether the field has been set or not
+ * */
+ private boolean fieldSet = false;
+
+ /**
* Construct a LazyUnion object with the ObjectInspector.
*/
public LazyUnion(LazyUnionObjectInspector oi) {
@@ -123,6 +131,7 @@ public class LazyUnion extends
*
* @return The value of the field
*/
+ @SuppressWarnings("rawtypes")
private Object uncheckedGetField() {
Text nullSequence = oi.getNullSequence();
int fieldLength = start + length - startPosition;
@@ -134,9 +143,9 @@ public class LazyUnion extends
if (!fieldInited) {
fieldInited = true;
- field.init(bytes, startPosition, fieldLength);
+ ((LazyObject) field).init(bytes, startPosition, fieldLength);
}
- return field.getObject();
+ return ((LazyObject) field).getObject();
}
/**
@@ -145,6 +154,10 @@ public class LazyUnion extends
* @return The field as a LazyObject
*/
public Object getField() {
+ if (fieldSet) {
+ return field;
+ }
+
if (!parsed) {
parse();
}
@@ -157,9 +170,33 @@ public class LazyUnion extends
* @return The tag byte
*/
public byte getTag() {
+ if (tagSet) {
+ return tag;
+ }
+
if (!parsed) {
parse();
}
return tag;
}
-}
+
+ /**
+ * Set the field of the union
+ *
+ * @param field the field to be set
+ * */
+ public void setField(Object field) {
+ this.field = field;
+ fieldSet = true;
+ }
+
+ /**
+ * Set the tag for the union
+ *
+ * @param tag the tag to be set
+ * */
+ public void setTag(byte tag) {
+ this.tag = tag;
+ tagSet = true;
+ }
+}
\ No newline at end of file
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java Wed Sep 10 21:41:16 2014
@@ -22,7 +22,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hadoop.hive.serde2.avro.AvroLazyObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import org.apache.hadoop.io.Text;
/**
@@ -48,14 +50,34 @@ public final class LazyObjectInspectorFa
byte escapeChar) {
return getLazySimpleStructObjectInspector(structFieldNames,
structFieldObjectInspectors, null, separator, nullSequence,
- lastColumnTakesRest, escaped, escapeChar);
+ lastColumnTakesRest, escaped, escapeChar, ObjectInspectorOptions.JAVA);
+ }
+
+ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(
+ List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors, byte separator,
+ Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
+ byte escapeChar, ObjectInspectorOptions option) {
+ return getLazySimpleStructObjectInspector(structFieldNames,
+ structFieldObjectInspectors, null, separator, nullSequence,
+ lastColumnTakesRest, escaped, escapeChar, option);
}
public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(
List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments,
byte separator, Text nullSequence, boolean lastColumnTakesRest,
- boolean escaped,byte escapeChar) {
+ boolean escaped, byte escapeChar) {
+ return getLazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors,
+ structFieldComments, separator, nullSequence, lastColumnTakesRest, escaped, escapeChar,
+ ObjectInspectorOptions.JAVA);
+ }
+
+ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(
+ List<String> structFieldNames,
+ List<ObjectInspector> structFieldObjectInspectors, List<String> structFieldComments,
+ byte separator, Text nullSequence, boolean lastColumnTakesRest,
+ boolean escaped,byte escapeChar, ObjectInspectorOptions option) {
ArrayList<Object> signature = new ArrayList<Object>();
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
@@ -64,15 +86,30 @@ public final class LazyObjectInspectorFa
signature.add(Boolean.valueOf(lastColumnTakesRest));
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
+ signature.add(option);
if(structFieldComments != null) {
signature.add(structFieldComments);
}
LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector
.get(signature);
if (result == null) {
- result = new LazySimpleStructObjectInspector(structFieldNames,
- structFieldObjectInspectors, structFieldComments, separator,
- nullSequence, lastColumnTakesRest, escaped, escapeChar);
+ switch (option) {
+ case JAVA:
+ result =
+ new LazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors,
+ structFieldComments, separator, nullSequence, lastColumnTakesRest, escaped,
+ escapeChar);
+ break;
+ case AVRO:
+ result =
+ new AvroLazyObjectInspector(structFieldNames, structFieldObjectInspectors,
+ structFieldComments, separator, nullSequence, lastColumnTakesRest, escaped,
+ escapeChar);
+ break;
+ default:
+ throw new IllegalArgumentException("Illegal ObjectInspector type [" + option + "]");
+ }
+
cachedLazySimpleStructObjectInspector.put(signature, result);
}
return result;
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java Wed Sep 10 21:41:16 2014
@@ -18,10 +18,13 @@
package org.apache.hadoop.hive.serde2.lazy.objectinspector;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.serde2.BaseStructObjectInspector;
import org.apache.hadoop.hive.serde2.StructObject;
+import org.apache.hadoop.hive.serde2.avro.AvroLazyObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.io.Text;
@@ -100,6 +103,20 @@ public class LazySimpleStructObjectInspe
int fieldID = f.getFieldID();
assert (fieldID >= 0 && fieldID < fields.size());
+ ObjectInspector oi = f.getFieldObjectInspector();
+
+ if (oi instanceof AvroLazyObjectInspector) {
+ return ((AvroLazyObjectInspector) oi).getStructFieldData(data, fieldRef);
+ }
+
+ if (oi instanceof MapObjectInspector) {
+ ObjectInspector valueOI = ((MapObjectInspector) oi).getMapValueObjectInspector();
+
+ if (valueOI instanceof AvroLazyObjectInspector) {
+ return ((AvroLazyObjectInspector) valueOI).getStructFieldData(data, fieldRef);
+ }
+ }
+
return struct.getField(fieldID);
}
@@ -108,8 +125,15 @@ public class LazySimpleStructObjectInspe
if (data == null) {
return null;
}
- StructObject struct = (StructObject) data;
- return struct.getFieldsAsList();
+
+ // Iterate over all the fields picking up the nested structs within them
+ List<Object> result = new ArrayList<Object>(fields.size());
+
+ for (MyField myField : fields) {
+ result.add(getStructFieldData(data, myField));
+ }
+
+ return result;
}
// For LazyStruct
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java Wed Sep 10 21:41:16 2014
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryListObjectInspector;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector;
+import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryUnionObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -106,6 +107,8 @@ public final class LazyBinaryFactory {
return new LazyBinaryArray((LazyBinaryListObjectInspector) oi);
case STRUCT:
return new LazyBinaryStruct((LazyBinaryStructObjectInspector) oi);
+ case UNION:
+ return new LazyBinaryUnion((LazyBinaryUnionObjectInspector) oi);
}
throw new RuntimeException("Hive LazyBinarySerDe Internal error.");
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java Wed Sep 10 21:41:16 2014
@@ -43,8 +43,8 @@ import org.apache.hadoop.hive.serde2.laz
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -281,6 +281,13 @@ public class LazyBinarySerDe extends Abs
}
}
+ private static void serializeUnion(RandomAccessOutput byteStream, Object obj,
+ UnionObjectInspector uoi, BooleanRef warnedOnceNullMapKey) throws SerDeException {
+ byte tag = uoi.getTag(obj);
+ byteStream.write(tag);
+ serialize(byteStream, uoi.getField(obj), uoi.getObjectInspectors().get(tag), false, warnedOnceNullMapKey);
+ }
+
private static void serializeText(
RandomAccessOutput byteStream, Text t, boolean skipLengthPrefix) {
/* write byte size of the string which is a vint */
@@ -544,24 +551,31 @@ public class LazyBinarySerDe extends Abs
}
return;
}
- case STRUCT: {
+ case STRUCT:
+ case UNION:{
int byteSizeStart = 0;
- int structStart = 0;
+ int typeStart = 0;
if (!skipLengthPrefix) {
// 1/ reserve spaces for the byte size of the struct
// which is a integer and takes four bytes
byteSizeStart = byteStream.getLength();
byteStream.reserve(4);
- structStart = byteStream.getLength();
+ typeStart = byteStream.getLength();
+ }
+
+ if (ObjectInspector.Category.STRUCT.equals(objInspector.getCategory()) ) {
+ // 2/ serialize the struct
+ serializeStruct(byteStream, obj, (StructObjectInspector) objInspector, warnedOnceNullMapKey);
+ } else {
+ // 2/ serialize the union
+ serializeUnion(byteStream, obj, (UnionObjectInspector) objInspector, warnedOnceNullMapKey);
}
- // 2/ serialize the struct
- serializeStruct(byteStream, obj, (StructObjectInspector) objInspector, warnedOnceNullMapKey);
if (!skipLengthPrefix) {
// 3/ update the byte size of the struct
- int structEnd = byteStream.getLength();
- int structSize = structEnd - structStart;
- writeSizeAtOffset(byteStream, byteSizeStart, structSize);
+ int typeEnd = byteStream.getLength();
+ int typeSize = typeEnd - typeStart;
+ writeSizeAtOffset(byteStream, byteSizeStart, typeSize);
}
return;
}
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java Wed Sep 10 21:41:16 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.WritableUtils;
@@ -226,6 +227,7 @@ public final class LazyBinaryUtils {
case LIST:
case MAP:
case STRUCT:
+ case UNION:
recordInfo.elementOffset = 4;
recordInfo.elementSize = LazyBinaryUtils.byteArrayToInt(bytes, offset);
break;
@@ -474,6 +476,20 @@ public final class LazyBinaryUtils {
fieldObjectInspectors);
break;
}
+ case UNION: {
+ UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
+ final List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
+ List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(
+ fieldTypeInfos.size());
+ for (int i = 0; i < fieldTypeInfos.size(); i++) {
+ fieldObjectInspectors
+ .add(getLazyBinaryObjectInspectorFromTypeInfo(fieldTypeInfos
+ .get(i)));
+ }
+ result = LazyBinaryObjectInspectorFactory
+ .getLazyBinaryUnionObjectInspector(fieldObjectInspectors);
+ break;
+ }
default: {
result = null;
}
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java Wed Sep 10 21:41:16 2014
@@ -40,6 +40,9 @@ public final class LazyBinaryObjectInspe
static ConcurrentHashMap<ArrayList<Object>, LazyBinaryStructObjectInspector> cachedLazyBinaryStructObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazyBinaryStructObjectInspector>();
+ static ConcurrentHashMap<ArrayList<Object>, LazyBinaryUnionObjectInspector> cachedLazyBinaryUnionObjectInspector =
+ new ConcurrentHashMap<ArrayList<Object>, LazyBinaryUnionObjectInspector>();
+
public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(
List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
@@ -66,6 +69,20 @@ public final class LazyBinaryObjectInspe
return result;
}
+ public static LazyBinaryUnionObjectInspector getLazyBinaryUnionObjectInspector(
+ List<ObjectInspector> unionFieldObjectInspectors) {
+ ArrayList<Object> signature = new ArrayList<Object>(1);
+ signature.add(unionFieldObjectInspectors);
+
+ LazyBinaryUnionObjectInspector result = cachedLazyBinaryUnionObjectInspector
+ .get(signature);
+ if (result == null) {
+ result = new LazyBinaryUnionObjectInspector(unionFieldObjectInspectors);
+ cachedLazyBinaryUnionObjectInspector.put(signature, result);
+ }
+ return result;
+ }
+
static ConcurrentHashMap<ArrayList<Object>, LazyBinaryListObjectInspector> cachedLazyBinaryListObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazyBinaryListObjectInspector>();
Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java?rev=1624140&r1=1624139&r2=1624140&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java Wed Sep 10 21:41:16 2014
@@ -57,7 +57,7 @@ public final class ObjectInspectorFactor
* for the same Java type.
*/
public enum ObjectInspectorOptions {
- JAVA, THRIFT, PROTOCOL_BUFFERS
+ JAVA, THRIFT, PROTOCOL_BUFFERS, AVRO
};
private static ConcurrentHashMap<Type, ObjectInspector> objectInspectorCache = new ConcurrentHashMap<Type, ObjectInspector>();