You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/12/18 07:37:36 UTC
svn commit: r1423289 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/ test/queries/clientnegative/
test/queries/clientpositive/ test/results/clientnegative/
test/results/clientpositive/
Author: namit
Date: Tue Dec 18 06:37:34 2012
New Revision: 1423289
URL: http://svn.apache.org/viewvc?rev=1423289&view=rev
Log:
HIVE-3787 Regression introduced from HIVE-3401
(Navis via namit)
Added:
hive/trunk/ql/src/test/queries/clientnegative/split_sample_wrong_format2.q
hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format2.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/test/queries/clientpositive/split_sample.q
hive/trunk/ql/src/test/results/clientnegative/split_sample_out_of_range.q.out
hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format.q.out
hive/trunk/ql/src/test/results/clientpositive/split_sample.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1423289&r1=1423288&r2=1423289&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Dec 18 06:37:34 2012
@@ -505,34 +505,30 @@ public class SemanticAnalyzer extends Ba
}
}
} else if (splitSamplePresent) {
- // only CombineHiveInputFormat supports this optimize
- String inputFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT);
- if (!inputFormat.equals(
- CombineHiveInputFormat.class.getName())) {
- throw new SemanticException(generateErrorMessage((ASTNode) tabref.getChild(1),
- "Percentage sampling is not supported in " + inputFormat));
- }
ASTNode sampleClause = (ASTNode) tabref.getChild(1);
- String alias_id = getAliasId(alias, qb);
Tree type = sampleClause.getChild(0);
- String numerator = unescapeIdentifier(sampleClause.getChild(1).getText());
+ Tree numerator = sampleClause.getChild(1);
+ String value = unescapeIdentifier(numerator.getText());
+
SplitSample sample;
if (type.getType() == HiveParser.TOK_PERCENT) {
- Double percent = Double.valueOf(numerator).doubleValue();
+ assertCombineInputFormat(numerator, "Percentage");
+ Double percent = Double.valueOf(value).doubleValue();
if (percent < 0 || percent > 100) {
- throw new SemanticException(generateErrorMessage(sampleClause,
+ throw new SemanticException(generateErrorMessage((ASTNode) numerator,
"Sampling percentage should be between 0 and 100"));
}
int seedNum = conf.getIntVar(ConfVars.HIVESAMPLERANDOMNUM);
sample = new SplitSample(percent, seedNum);
} else if (type.getType() == HiveParser.TOK_ROWCOUNT) {
- sample = new SplitSample(Integer.valueOf(numerator));
+ sample = new SplitSample(Integer.valueOf(value));
} else {
assert type.getType() == HiveParser.TOK_LENGTH;
- long length = Integer.valueOf(numerator.substring(0, numerator.length() - 1));
- char last = numerator.charAt(numerator.length() - 1);
+ assertCombineInputFormat(numerator, "Total Length");
+ long length = Integer.valueOf(value.substring(0, value.length() - 1));
+ char last = value.charAt(value.length() - 1);
if (last == 'k' || last == 'K') {
length <<= 10;
} else if (last == 'm' || last == 'M') {
@@ -543,6 +539,7 @@ public class SemanticAnalyzer extends Ba
int seedNum = conf.getIntVar(ConfVars.HIVESAMPLERANDOMNUM);
sample = new SplitSample(length, seedNum);
}
+ String alias_id = getAliasId(alias, qb);
nameToSplitSample.put(alias_id, sample);
}
// Insert this map into the stats
@@ -560,6 +557,14 @@ public class SemanticAnalyzer extends Ba
return alias;
}
+ private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException {
+ String inputFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT);
+ if (!inputFormat.equals(CombineHiveInputFormat.class.getName())) {
+ throw new SemanticException(generateErrorMessage((ASTNode) numerator,
+ message + " sampling is not supported in " + inputFormat));
+ }
+ }
+
private String processSubQuery(QB qb, ASTNode subq) throws SemanticException {
// This is a subquery and must have an alias
Added: hive/trunk/ql/src/test/queries/clientnegative/split_sample_wrong_format2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/split_sample_wrong_format2.q?rev=1423289&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/split_sample_wrong_format2.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/split_sample_wrong_format2.q Tue Dec 18 06:37:34 2012
@@ -0,0 +1,3 @@
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+select key from src tablesample(1K);
Modified: hive/trunk/ql/src/test/queries/clientpositive/split_sample.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/split_sample.q?rev=1423289&r1=1423288&r2=1423289&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/split_sample.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/split_sample.q Tue Dec 18 06:37:34 2012
@@ -106,3 +106,7 @@ select count(1) from ss_src2 tablesample
set hive.fetch.task.conversion=more;
select key from ss_src2 tablesample(200B);
select key from ss_src2 tablesample(10 ROWS);
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+-- ROW type works with other input formats (others, don't)
+select count(1) from ss_src2 tablesample(10 ROWS);
Modified: hive/trunk/ql/src/test/results/clientnegative/split_sample_out_of_range.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/split_sample_out_of_range.q.out?rev=1423289&r1=1423288&r2=1423289&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/split_sample_out_of_range.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/split_sample_out_of_range.q.out Tue Dec 18 06:37:34 2012
@@ -1 +1 @@
-FAILED: SemanticException 0:0 Sampling percentage should be between 0 and 100. Error encountered near token '105'
+FAILED: SemanticException 3:32 Sampling percentage should be between 0 and 100. Error encountered near token '105'
Modified: hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format.q.out?rev=1423289&r1=1423288&r2=1423289&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format.q.out Tue Dec 18 06:37:34 2012
@@ -1 +1 @@
-FAILED: SemanticException 0:0 Percentage sampling is not supported in org.apache.hadoop.hive.ql.io.HiveInputFormat. Error encountered near token '1'
+FAILED: SemanticException 3:32 Percentage sampling is not supported in org.apache.hadoop.hive.ql.io.HiveInputFormat. Error encountered near token '1'
Added: hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format2.q.out?rev=1423289&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/split_sample_wrong_format2.q.out Tue Dec 18 06:37:34 2012
@@ -0,0 +1 @@
+FAILED: SemanticException 3:32 Total Length sampling is not supported in org.apache.hadoop.hive.ql.io.HiveInputFormat. Error encountered near token '1K'
Modified: hive/trunk/ql/src/test/results/clientpositive/split_sample.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/split_sample.q.out?rev=1423289&r1=1423288&r2=1423289&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/split_sample.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/split_sample.q.out Tue Dec 18 06:37:34 2012
@@ -4821,3 +4821,26 @@ POSTHOOK: Lineage: ss_i_part PARTITION(p
278
98
484
+PREHOOK: query: -- ROW type works with other input formats (others, don't)
+select count(1) from ss_src2 tablesample(10 ROWS)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ss_src2
+#### A masked pattern was here ####
+POSTHOOK: query: -- ROW type works with other input formats (others, don't)
+select count(1) from ss_src2 tablesample(10 ROWS)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ss_src2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: ss_i_part PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss_i_part PARTITION(p=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+10