You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/06/22 06:27:00 UTC
svn commit: r1352771 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/optimizer/physical/
test/queries/clientpositive/ test/results/clientpositive/
Author: namit
Date: Fri Jun 22 04:26:59 2012
New Revision: 1352771
URL: http://svn.apache.org/viewvc?rev=1352771&view=rev
Log:
HIVE-2955 Queries consists of metadata-only-query returns always empty value
(Navis via namit)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
hive/trunk/ql/src/test/queries/clientpositive/metadataonly1.q
hive/trunk/ql/src/test/results/clientpositive/metadataonly1.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1352771&r1=1352770&r2=1352771&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Fri Jun 22 04:26:59 2012
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.io.Hive
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
import org.apache.hadoop.hive.ql.io.IOPrepareCache;
+import org.apache.hadoop.hive.ql.io.OneNullRowInputFormat;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
@@ -778,12 +779,14 @@ public class ExecDriver extends Task<Map
// The input file does not exist, replace it by a empty file
Class<? extends HiveOutputFormat> outFileFormat = null;
boolean nonNative = true;
+ boolean oneRow = false;
Properties props;
if (isEmptyPath) {
PartitionDesc partDesc = work.getPathToPartitionInfo().get(path);
props = partDesc.getProperties();
outFileFormat = partDesc.getOutputFileFormatClass();
nonNative = partDesc.getTableDesc().isNonNative();
+ oneRow = partDesc.getInputFileFormatClass() == OneNullRowInputFormat.class;
} else {
TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
props = tableDesc.getProperties();
@@ -841,6 +844,12 @@ public class ExecDriver extends Task<Map
String onefile = newPath.toString();
RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath,
Text.class, false, props, null);
+ if (oneRow) {
+ // empty files are ommited at CombineHiveInputFormat.
+ // for metadata only query, it effectively makes partition columns disappear..
+ // this could be fixed by other methods, but this seemed to be the most easy (HIVEV-2955)
+ recWriter.write(new Text("empty")); // written via HiveIgnoreKeyTextOutputFormat
+ }
recWriter.close(false);
FileInputFormat.addInputPaths(job, onefile);
return numEmptyPaths;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java?rev=1352771&r1=1352770&r2=1352771&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java Fri Jun 22 04:26:59 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.Gr
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -200,6 +201,7 @@ public class MetadataOnlyOptimizer imple
private PartitionDesc changePartitionToMetadataOnly(PartitionDesc desc) {
if (desc != null) {
desc.setInputFileFormatClass(OneNullRowInputFormat.class);
+ desc.setOutputFileFormatClass(HiveIgnoreKeyTextOutputFormat.class);
desc.setDeserializerClass(NullStructSerDe.class);
desc.setSerdeClassName(NullStructSerDe.class.getName());
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/metadataonly1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/metadataonly1.q?rev=1352771&r1=1352770&r2=1352771&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/metadataonly1.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/metadataonly1.q Fri Jun 22 04:26:59 2012
@@ -33,3 +33,6 @@ set hive.input.format=org.apache.hadoop.
explain extended select max(ds) from TEST1;
select max(ds) from TEST1;
+
+select distinct ds from srcpart;
+select min(ds),max(ds) from srcpart;
\ No newline at end of file
Modified: hive/trunk/ql/src/test/results/clientpositive/metadataonly1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/metadataonly1.q.out?rev=1352771&r1=1352770&r2=1352771&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/metadataonly1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/metadataonly1.q.out Fri Jun 22 04:26:59 2012
@@ -1452,3 +1452,34 @@ POSTHOOK: Input: default@test1@ds=1
POSTHOOK: Input: default@test1@ds=2
#### A masked pattern was here ####
2
+PREHOOK: query: select distinct ds from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct ds from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08
+2008-04-09
+PREHOOK: query: select min(ds),max(ds) from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select min(ds),max(ds) from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 2008-04-09