You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by rm...@apache.org on 2009/08/12 08:17:56 UTC
svn commit: r803371 - in /hadoop/hive/branches/branch-0.4: ./
contrib/src/test/results/clientnegative/
contrib/src/test/results/clientpositive/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/queries...
Author: rmurthy
Date: Wed Aug 12 06:17:54 2009
New Revision: 803371
URL: http://svn.apache.org/viewvc?rev=803371&view=rev
Log:
HIVE-743. Let user specify serde for custom sctipts.
(Namit Jain via rmurthy)
Added:
hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input33.q
hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input34.q
hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input35.q
hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input36.q
hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input33.q.out
hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input34.q.out
hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input35.q.out
hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input36.q.out
Modified:
hadoop/hive/branches/branch-0.4/CHANGES.txt
hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientnegative/serde_regex.q.out
hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientpositive/serde_regex.q.out
hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input15.q.out
hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/inputddl3.q.out
Modified: hadoop/hive/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/CHANGES.txt?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.4/CHANGES.txt Wed Aug 12 06:17:54 2009
@@ -122,6 +122,9 @@
HIVE-691. show documentation for functions. (Emil Ibrishimov via namit)
+ HIVE-743. Let user specify serde for custom sctipts.
+ (Namit Jain via rmurthy)
+
IMPROVEMENTS
HIVE-389. Option to build without ivy (jssarma)
Modified: hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientnegative/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientnegative/serde_regex.q.out?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientnegative/serde_regex.q.out (original)
+++ hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientnegative/serde_regex.q.out Wed Aug 12 06:17:54 2009
@@ -17,7 +17,7 @@
)
STORED AS TEXTFILE
ABSTRACT SYNTAX TREE:
- (TOK_CREATETABLE serde_regex TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL host TOK_STRING) (TOK_TABCOL identity TOK_STRING) (TOK_TABCOL user TOK_STRING) (TOK_TABCOL time TOK_STRING) (TOK_TABCOL request TOK_STRING) (TOK_TABCOL status TOK_INT) (TOK_TABCOL size TOK_INT) (TOK_TABCOL referer TOK_STRING) (TOK_TABCOL agent TOK_STRING)) (TOK_TABLESERIALIZER 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' (TOK_TABLEPROPERTIES (TOK_TABLEPROPLIST (TOK_TABLEPROPERTY "input.regex" "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\") ([^ \"]*|\"[^\"]*\"))?") (TOK_TABLEPROPERTY "output.format.string" "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s")))) TOK_TBLTEXTFILE)
+ (TOK_CREATETABLE serde_regex TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL host TOK_STRING) (TOK_TABCOL identity TOK_STRING) (TOK_TABCOL user TOK_STRING) (TOK_TABCOL time TOK_STRING) (TOK_TABCOL request TOK_STRING) (TOK_TABCOL status TOK_INT) (TOK_TABCOL size TOK_INT) (TOK_TABCOL referer TOK_STRING) (TOK_TABCOL agent TOK_STRING)) (TOK_TABLESERIALIZER (TOK_SERDENAME 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' (TOK_TABLEPROPERTIES (TOK_TABLEPROPLIST (TOK_TABLEPROPERTY "input.regex" "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\") ([^ \"]*|\"[^\"]*\"))?") (TOK_TABLEPROPERTY "output.format.string" "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s"))))) TOK_TBLTEXTFILE)
STAGE DEPENDENCIES:
Stage-0 is a root stage
Modified: hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientpositive/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientpositive/serde_regex.q.out?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientpositive/serde_regex.q.out (original)
+++ hadoop/hive/branches/branch-0.4/contrib/src/test/results/clientpositive/serde_regex.q.out Wed Aug 12 06:17:54 2009
@@ -17,7 +17,7 @@
)
STORED AS TEXTFILE
ABSTRACT SYNTAX TREE:
- (TOK_CREATETABLE serde_regex TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL host TOK_STRING) (TOK_TABCOL identity TOK_STRING) (TOK_TABCOL user TOK_STRING) (TOK_TABCOL time TOK_STRING) (TOK_TABCOL request TOK_STRING) (TOK_TABCOL status TOK_STRING) (TOK_TABCOL size TOK_STRING) (TOK_TABCOL referer TOK_STRING) (TOK_TABCOL agent TOK_STRING)) (TOK_TABLESERIALIZER 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' (TOK_TABLEPROPERTIES (TOK_TABLEPROPLIST (TOK_TABLEPROPERTY "input.regex" "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\") ([^ \"]*|\"[^\"]*\"))?") (TOK_TABLEPROPERTY "output.format.string" "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s")))) TOK_TBLTEXTFILE)
+ (TOK_CREATETABLE serde_regex TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL host TOK_STRING) (TOK_TABCOL identity TOK_STRING) (TOK_TABCOL user TOK_STRING) (TOK_TABCOL time TOK_STRING) (TOK_TABCOL request TOK_STRING) (TOK_TABCOL status TOK_STRING) (TOK_TABCOL size TOK_STRING) (TOK_TABCOL referer TOK_STRING) (TOK_TABCOL agent TOK_STRING)) (TOK_TABLESERIALIZER (TOK_SERDENAME 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' (TOK_TABLEPROPERTIES (TOK_TABLEPROPLIST (TOK_TABLEPROPERTY "input.regex" "([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\"[^\"]*\") ([^ \"]*|\"[^\"]*\"))?") (TOK_TABLEPROPERTY "output.format.string" "%1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s"))))) TOK_TBLTEXTFILE)
STAGE DEPENDENCIES:
Stage-0 is a root stage
@@ -59,7 +59,7 @@
query: LOAD DATA LOCAL INPATH "../data/files/apache.access.2.log" INTO TABLE serde_regex
query: SELECT * FROM serde_regex ORDER BY time
Input: default/serde_regex
-Output: file:/data/users/zshao/tools/167-trunk-apache-hive/build/ql/tmp/2091262141/10000
+Output: file:/data/users/njain/hive3/hive3/build/ql/tmp/1069367320/10000
127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 NULL NULL
127.0.0.1 - - [26/May/2009:00:00:00 +0000] "GET /someurl/?track=Blabla(Main) HTTP/1.1" 200 5864 - "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.65 Safari/525.19"
query: DROP TABLE serde_regex
Modified: hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Wed Aug 12 06:17:54 2009
@@ -216,6 +216,8 @@
}
break;
case HiveParser.TOK_TABLEROWFORMAT:
+
+ child = (ASTNode)child.getChild(0);
int numChildRowFormat = child.getChildCount();
for (int numC = 0; numC < numChildRowFormat; numC++)
{
@@ -241,6 +243,8 @@
}
break;
case HiveParser.TOK_TABLESERIALIZER:
+
+ child = (ASTNode)child.getChild(0);
serde = unescapeSQLString(child.getChild(0).getText());
if (child.getChildCount() == 2) {
mapProp = new HashMap<String, String>();
Modified: hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original)
+++ hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Wed Aug 12 06:17:54 2009
@@ -50,6 +50,9 @@
TOK_TRUE;
TOK_FALSE;
TOK_TRANSFORM;
+TOK_SERDE;
+TOK_SERDENAME;
+TOK_SERDEPROPS;
TOK_EXPLIST;
TOK_ALIASLIST;
TOK_GROUPBY;
@@ -132,6 +135,8 @@
TOK_HINT;
TOK_MAPJOIN;
TOK_HINTARGLIST;
+TOK_USERSCRIPTCOLNAMES;
+TOK_USERSCRIPTCOLSCHEMA;
}
@@ -277,8 +282,8 @@
alterStatementSuffixSerdeProperties
@init { msgs.push("alter serdes statement"); }
@after { msgs.pop(); }
- : name=Identifier KW_SET KW_SERDE serde=StringLiteral (KW_WITH KW_SERDEPROPERTIES tableProperties)?
- -> ^(TOK_ALTERTABLE_SERIALIZER $name $serde tableProperties?)
+ : name=Identifier KW_SET KW_SERDE serdeName=StringLiteral (KW_WITH KW_SERDEPROPERTIES tableProperties)?
+ -> ^(TOK_ALTERTABLE_SERIALIZER $name $serdeName tableProperties?)
| name=Identifier KW_SET KW_SERDEPROPERTIES tableProperties
-> ^(TOK_ALTERTABLE_SERDEPROPERTIES $name tableProperties)
;
@@ -361,14 +366,36 @@
-> ^(TOK_TABLEBUCKETS $bucketCols $sortCols? $num)
;
+serde
+@init { msgs.push("serde specification"); }
+@after { msgs.pop(); }
+ : serdeFormat -> ^(TOK_SERDE serdeFormat)
+ | serdePropertiesFormat -> ^(TOK_SERDE serdePropertiesFormat)
+ ;
+
+serdeFormat
+@init { msgs.push("serde format specification"); }
+@after { msgs.pop(); }
+ : KW_ROW KW_FORMAT KW_SERDE name=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)?
+ -> ^(TOK_SERDENAME $name $serdeprops?)
+ ;
+
+serdePropertiesFormat
+@init { msgs.push("serde properties specification"); }
+@after { msgs.pop(); }
+ :
+ KW_ROW KW_FORMAT KW_DELIMITED tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?
+ -> ^(TOK_SERDEPROPS tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?)
+ ;
+
tableRowFormat
@init { msgs.push("table row format specification"); }
@after { msgs.pop(); }
:
- KW_ROW KW_FORMAT KW_DELIMITED tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?
- -> ^(TOK_TABLEROWFORMAT tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?)
- | KW_ROW KW_FORMAT KW_SERDE name=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)?
- -> ^(TOK_TABLESERIALIZER $name $serdeprops?)
+ serdePropertiesFormat
+ -> ^(TOK_TABLEROWFORMAT serdePropertiesFormat)
+ | serdeFormat
+ -> ^(TOK_TABLESERIALIZER serdeFormat)
;
tableProperties
@@ -695,7 +722,7 @@
:
( selectExpression (KW_AS? Identifier)?) -> ^(TOK_SELEXPR selectExpression Identifier?)
;
-
+
trfmClause
@init { msgs.push("transform clause"); }
@after { msgs.pop(); }
@@ -703,9 +730,9 @@
( KW_SELECT KW_TRANSFORM LPAREN selectExpressionList RPAREN
| KW_MAP selectExpressionList
| KW_REDUCE selectExpressionList )
- KW_USING StringLiteral
- (KW_AS (LPAREN aliasList RPAREN | aliasList) )?
- -> ^(TOK_TRANSFORM selectExpressionList StringLiteral aliasList?)
+ inSerde=serde? KW_USING StringLiteral
+ ( KW_AS ((LPAREN (aliasList | columnNameTypeList) RPAREN) | (aliasList | columnNameTypeList)) outSerde=serde?)?
+ -> ^(TOK_TRANSFORM selectExpressionList $inSerde? StringLiteral aliasList? columnNameTypeList? $outSerde?)
;
selectExpression
Modified: hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Aug 12 06:17:54 2009
@@ -34,6 +34,7 @@
import java.util.Vector;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import java.lang.ClassNotFoundException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
@@ -124,6 +125,7 @@
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
@@ -1103,46 +1105,167 @@
return cmd;
}
+ private tableDesc getTableDescFromSerDe(ASTNode child, String cols, boolean defaultCols) throws SemanticException {
+ if (child.getType() == HiveParser.TOK_SERDENAME) {
+ String serdeName = unescapeSQLString(child.getChild(0).getText());
+ Class<? extends Deserializer> serdeClass = null;
+
+ try {
+ serdeClass = (Class<? extends Deserializer>)Class.forName(serdeName);
+ } catch (ClassNotFoundException e) {
+ throw new SemanticException(e);
+ }
+
+ tableDesc tblDesc = PlanUtils.getTableDesc(serdeClass, Integer.toString(Utilities.tabCode), cols, defaultCols);
+ // copy all the properties
+ if (child.getChildCount() == 2) {
+ ASTNode prop = (ASTNode)((ASTNode)child.getChild(1)).getChild(0);
+ for (int propChild = 0; propChild < prop.getChildCount(); propChild++) {
+ String key = unescapeSQLString(prop.getChild(propChild).getChild(0).getText());
+ String value = unescapeSQLString(prop.getChild(propChild).getChild(1).getText());
+ tblDesc.getProperties().setProperty(key,value);
+ }
+ }
+ return tblDesc;
+ }
+ else if (child.getType() == HiveParser.TOK_SERDEPROPS) {
+ tableDesc tblDesc = PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.ctrlaCode), cols, defaultCols);
+ int numChildRowFormat = child.getChildCount();
+ for (int numC = 0; numC < numChildRowFormat; numC++)
+ {
+ ASTNode rowChild = (ASTNode)child.getChild(numC);
+ switch (rowChild.getToken().getType()) {
+ case HiveParser.TOK_TABLEROWFORMATFIELD:
+ String fieldDelim = unescapeSQLString(rowChild.getChild(0).getText());
+ tblDesc.getProperties().setProperty(Constants.FIELD_DELIM, fieldDelim);
+ tblDesc.getProperties().setProperty(Constants.SERIALIZATION_FORMAT, fieldDelim);
+
+ if (rowChild.getChildCount()>=2) {
+ String fieldEscape = unescapeSQLString(rowChild.getChild(1).getText());
+ tblDesc.getProperties().setProperty(Constants.ESCAPE_CHAR, fieldDelim);
+ }
+ break;
+ case HiveParser.TOK_TABLEROWFORMATCOLLITEMS:
+ tblDesc.getProperties().setProperty(Constants.COLLECTION_DELIM, unescapeSQLString(rowChild.getChild(0).getText()));
+ break;
+ case HiveParser.TOK_TABLEROWFORMATMAPKEYS:
+ tblDesc.getProperties().setProperty(Constants.MAPKEY_DELIM, unescapeSQLString(rowChild.getChild(0).getText()));
+ break;
+ case HiveParser.TOK_TABLEROWFORMATLINES:
+ tblDesc.getProperties().setProperty(Constants.LINE_DELIM, unescapeSQLString(rowChild.getChild(0).getText()));
+ break;
+ default: assert false;
+ }
+ }
+
+ return tblDesc;
+ }
+
+ // should never come here
+ return null;
+ }
@SuppressWarnings("nls")
private Operator genScriptPlan(ASTNode trfm, QB qb,
Operator input) throws SemanticException {
// If there is no "AS" clause, the output schema will be "key,value"
- ArrayList<String> outputColList = new ArrayList<String>();
- boolean defaultOutputColList = (trfm.getChildCount() < 3);
+ ArrayList<ColumnInfo> outputCols = new ArrayList<ColumnInfo>();
+ boolean defaultOutputColList = true;
+ int inputSerDeChildNum = -1, outputSerDeChildNum = -1;
+ int outputColumnNamesPos = -1, outputColumnSchemaPos = -1;
+ int execPos = 1;
+
+ // Go over all the children
+ for (int pos = 0; pos < trfm.getChildCount(); pos++) {
+ ASTNode child = (ASTNode)trfm.getChild(pos);
+ if (child.getType() == HiveParser.TOK_ALIASLIST) {
+ defaultOutputColList = false;
+ outputColumnNamesPos = pos;
+ break;
+ }
+ else if (child.getType() == HiveParser.TOK_TABCOLLIST) {
+ defaultOutputColList = false;
+ outputColumnSchemaPos = pos;
+ break;
+ }
+ }
+
+ // input serde specified
+ if ((trfm.getChildCount() > 1) &&
+ (trfm.getChild(1).getType() == HiveParser.TOK_SERDE)) {
+ inputSerDeChildNum = 1;
+ execPos++;
+ }
+
+ // output serde specified
+ int checkChildNum = -1;
+ if (outputColumnNamesPos >= 0)
+ checkChildNum = outputColumnNamesPos + 1;
+ else if (outputColumnSchemaPos >= 0)
+ checkChildNum = outputColumnSchemaPos + 1;
+
+ if (checkChildNum >= 0) {
+ if ((trfm.getChildCount() > (checkChildNum))
+ && (trfm.getChild(checkChildNum).getType() == HiveParser.TOK_SERDE))
+ outputSerDeChildNum = checkChildNum;
+ }
+
+ // If column type is not specified, use a string
if (defaultOutputColList) {
- outputColList.add("key");
- outputColList.add("value");
- } else {
- ASTNode collist = (ASTNode) trfm.getChild(2);
+ outputCols.add(new ColumnInfo("key", TypeInfoFactory.stringTypeInfo, null, false));
+ outputCols.add(new ColumnInfo("value", TypeInfoFactory.stringTypeInfo, null, false));
+ }
+ else if (outputColumnNamesPos >= 0) {
+ ASTNode collist = (ASTNode) trfm.getChild(outputColumnNamesPos);
int ccount = collist.getChildCount();
for (int i=0; i < ccount; ++i) {
- outputColList.add(unescapeIdentifier(((ASTNode)collist.getChild(i)).getText()));
+ outputCols.add(new ColumnInfo(unescapeIdentifier(((ASTNode)collist.getChild(i)).getText()), TypeInfoFactory.stringTypeInfo, null, false));
}
}
-
+ else {
+ assert outputColumnSchemaPos >= 0;
+ ASTNode collist = (ASTNode) trfm.getChild(outputColumnSchemaPos);
+ int ccount = collist.getChildCount();
+ for (int i=0; i < ccount; ++i) {
+ ASTNode child = (ASTNode) collist.getChild(i);
+ assert child.getType() == HiveParser.TOK_TABCOL;
+ outputCols.add(new ColumnInfo(unescapeIdentifier(((ASTNode)child.getChild(0)).getText()),
+ TypeInfoUtils.getTypeInfoFromTypeString(DDLSemanticAnalyzer.getTypeName(((ASTNode)child.getChild(1)).getType())), null, false));
+ }
+ }
+
RowResolver out_rwsch = new RowResolver();
StringBuilder columns = new StringBuilder();
- for (int i = 0; i < outputColList.size(); ++i) {
+ for (int i = 0; i < outputCols.size(); ++i) {
if (i != 0) {
columns.append(",");
}
- columns.append(outputColList.get(i));
+ columns.append(outputCols.get(i).getInternalName());
out_rwsch.put(
qb.getParseInfo().getAlias(),
- outputColList.get(i),
- new ColumnInfo(outputColList.get(i), TypeInfoFactory.stringTypeInfo, null, false) // Script output is always a string
- );
+ outputCols.get(i).getInternalName(),
+ outputCols.get(i));
}
+ tableDesc outInfo;
+ tableDesc inInfo;
+
+ if (inputSerDeChildNum < 0)
+ inInfo = PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), "");
+ else
+ inInfo = getTableDescFromSerDe((ASTNode)(((ASTNode)trfm.getChild(inputSerDeChildNum))).getChild(0), "", false);
+
+ if (outputSerDeChildNum < 0)
+ outInfo = PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), columns.toString(), defaultOutputColList);
+ else
+ outInfo = getTableDescFromSerDe((ASTNode)(((ASTNode)trfm.getChild(outputSerDeChildNum))).getChild(0), columns.toString(), defaultOutputColList);
+
Operator output = putOpInsertMap(OperatorFactory
.getAndMakeChild(
new scriptDesc(
- getFixedCmd(stripQuotes(trfm.getChild(1).getText())),
- PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), columns.toString(), defaultOutputColList),
- PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), "")),
- new RowSchema(
- out_rwsch.getColumnInfos()), input), out_rwsch);
+ getFixedCmd(stripQuotes(trfm.getChild(execPos).getText())),
+ outInfo, inInfo),
+ new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch);
return output;
}
Modified: hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hadoop/hive/branches/branch-0.4/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Wed Aug 12 06:17:54 2009
@@ -40,6 +40,7 @@
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.thrift.protocol.TBinaryProtocol;
@@ -68,6 +69,14 @@
}
/**
+ * Generate the table descriptor of given serde with the separatorCode
+ * and column names (comma separated string).
+ */
+ public static tableDesc getTableDesc(Class<? extends Deserializer> serdeClass, String separatorCode, String columns) {
+ return getTableDesc(serdeClass, separatorCode, columns, false);
+ }
+
+ /**
* Generate the table descriptor of MetadataTypedColumnsetSerDe with the separatorCode
* and column names (comma separated string), and whether the last column should take
* the rest of the line.
@@ -78,12 +87,30 @@
}
/**
+ * Generate the table descriptor of the serde specified with the separatorCode
+ * and column names (comma separated string), and whether the last column should take
+ * the rest of the line.
+ */
+ public static tableDesc getTableDesc(Class<? extends Deserializer> serdeClass,
+ String separatorCode, String columns,
+ boolean lastColumnTakesRestOfTheLine) {
+ return getTableDesc(serdeClass, separatorCode, columns, null, lastColumnTakesRestOfTheLine);
+ }
+
+ /**
* Generate the table descriptor of MetadataTypedColumnsetSerDe with the separatorCode
* and column names (comma separated string), and whether the last column should take
* the rest of the line.
*/
public static tableDesc getDefaultTableDesc(String separatorCode, String columns, String columnTypes,
boolean lastColumnTakesRestOfTheLine) {
+ return getTableDesc(LazySimpleSerDe.class, separatorCode, columns, columnTypes,
+ lastColumnTakesRestOfTheLine);
+ }
+
+ public static tableDesc getTableDesc(Class<? extends Deserializer> serdeClass,
+ String separatorCode, String columns, String columnTypes,
+ boolean lastColumnTakesRestOfTheLine) {
Properties properties = Utilities.makeProperties(
Constants.SERIALIZATION_FORMAT, separatorCode,
Constants.LIST_COLUMNS, columns);
@@ -96,10 +123,10 @@
"true");
}
return new tableDesc(
- LazySimpleSerDe.class,
- TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class,
- properties);
+ serdeClass,
+ TextInputFormat.class,
+ IgnoreKeyTextOutputFormat.class,
+ properties);
}
/**
Added: hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input33.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input33.q?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input33.q (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input33.q Wed Aug 12 06:17:54 2009
@@ -0,0 +1,28 @@
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+EXPLAIN
+FROM (
+ FROM src
+ MAP src.key, src.key
+ USING 'cat'
+ DISTRIBUTE BY key
+ SORT BY key, value
+) tmap
+INSERT OVERWRITE TABLE dest1
+REDUCE tmap.key, tmap.value
+USING '../data/scripts/input20_script'
+AS (key STRING, value STRING);
+
+FROM (
+ FROM src
+ MAP src.key, src.key
+ USING 'cat'
+ DISTRIBUTE BY key
+ SORT BY key, value
+) tmap
+INSERT OVERWRITE TABLE dest1
+REDUCE tmap.key, tmap.value
+USING '../data/scripts/input20_script'
+AS (key STRING, value STRING);
+
+SELECT * FROM dest1 SORT BY key, value;
Added: hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input34.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input34.q?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input34.q (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input34.q Wed Aug 12 06:17:54 2009
@@ -0,0 +1,20 @@
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+SELECT dest1.* FROM dest1;
Added: hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input35.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input35.q?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input35.q (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input35.q Wed Aug 12 06:17:54 2009
@@ -0,0 +1,20 @@
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+SELECT dest1.* FROM dest1;
Added: hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input36.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input36.q?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input36.q (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/queries/clientpositive/input36.q Wed Aug 12 06:17:54 2009
@@ -0,0 +1,20 @@
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\003'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\003'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+SELECT dest1.* FROM dest1;
Modified: hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input15.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input15.q.out?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input15.q.out (original)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input15.q.out Wed Aug 12 06:17:54 2009
@@ -1,7 +1,7 @@
query: EXPLAIN
CREATE TABLE TEST15(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE
ABSTRACT SYNTAX TREE:
- (TOK_CREATETABLE TEST15 TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_TABLEROWFORMATFIELD '\t')) TOK_TBLTEXTFILE)
+ (TOK_CREATETABLE TEST15 TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD '\t'))) TOK_TBLTEXTFILE)
STAGE DEPENDENCIES:
Stage-0 is a root stage
Added: hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input33.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input33.q.out?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input33.q.out (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input33.q.out Wed Aug 12 06:17:54 2009
@@ -0,0 +1,421 @@
+query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+query: EXPLAIN
+FROM (
+ FROM src
+ MAP src.key, src.key
+ USING 'cat'
+ DISTRIBUTE BY key
+ SORT BY key, value
+) tmap
+INSERT OVERWRITE TABLE dest1
+REDUCE tmap.key, tmap.value
+USING '../data/scripts/input20_script'
+AS (key STRING, value STRING)
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) 'cat'))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) '../data/scripts/input20_script' (TOK_TABCOLLIST (TOK_TABCOL key TOK_STRING) (TOK_TABCOL value TOK_STRING)))))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ tmap:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: key
+ type: string
+ outputColumnNames: _col0, _col1
+ Transform Operator
+ command: cat
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: -1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ Transform Operator
+ command: ../data/scripts/input20_script
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: UDFToInteger(key)
+ type: int
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+query: FROM (
+ FROM src
+ MAP src.key, src.key
+ USING 'cat'
+ DISTRIBUTE BY key
+ SORT BY key, value
+) tmap
+INSERT OVERWRITE TABLE dest1
+REDUCE tmap.key, tmap.value
+USING '../data/scripts/input20_script'
+AS (key STRING, value STRING)
+Input: default/src
+Output: default/dest1
+query: SELECT * FROM dest1 SORT BY key, value
+Input: default/dest1
+Output: file:/data/users/njain/hive3/hive3/build/ql/tmp/410502456/10000
+1 105_105
+1 10_10
+1 111_111
+1 114_114
+1 116_116
+1 11_11
+1 126_126
+1 131_131
+1 133_133
+1 136_136
+1 143_143
+1 145_145
+1 150_150
+1 153_153
+1 155_155
+1 156_156
+1 157_157
+1 158_158
+1 160_160
+1 162_162
+1 163_163
+1 166_166
+1 168_168
+1 170_170
+1 177_177
+1 178_178
+1 17_17
+1 180_180
+1 181_181
+1 183_183
+1 186_186
+1 189_189
+1 190_190
+1 192_192
+1 194_194
+1 196_196
+1 19_19
+1 201_201
+1 202_202
+1 20_20
+1 214_214
+1 218_218
+1 222_222
+1 226_226
+1 228_228
+1 235_235
+1 241_241
+1 244_244
+1 247_247
+1 248_248
+1 249_249
+1 252_252
+1 257_257
+1 258_258
+1 260_260
+1 262_262
+1 263_263
+1 266_266
+1 274_274
+1 275_275
+1 27_27
+1 283_283
+1 284_284
+1 285_285
+1 286_286
+1 287_287
+1 289_289
+1 28_28
+1 291_291
+1 292_292
+1 296_296
+1 2_2
+1 302_302
+1 305_305
+1 306_306
+1 308_308
+1 30_30
+1 310_310
+1 315_315
+1 323_323
+1 332_332
+1 335_335
+1 336_336
+1 338_338
+1 339_339
+1 33_33
+1 341_341
+1 345_345
+1 34_34
+1 351_351
+1 356_356
+1 360_360
+1 362_362
+1 364_364
+1 365_365
+1 366_366
+1 368_368
+1 373_373
+1 374_374
+1 375_375
+1 377_377
+1 378_378
+1 379_379
+1 386_386
+1 389_389
+1 392_392
+1 393_393
+1 394_394
+1 400_400
+1 402_402
+1 407_407
+1 411_411
+1 418_418
+1 419_419
+1 41_41
+1 421_421
+1 427_427
+1 432_432
+1 435_435
+1 436_436
+1 437_437
+1 43_43
+1 443_443
+1 444_444
+1 446_446
+1 448_448
+1 449_449
+1 44_44
+1 452_452
+1 453_453
+1 455_455
+1 457_457
+1 460_460
+1 467_467
+1 470_470
+1 472_472
+1 475_475
+1 477_477
+1 479_479
+1 47_47
+1 481_481
+1 482_482
+1 483_483
+1 484_484
+1 485_485
+1 487_487
+1 490_490
+1 491_491
+1 493_493
+1 494_494
+1 495_495
+1 496_496
+1 497_497
+1 4_4
+1 53_53
+1 54_54
+1 57_57
+1 64_64
+1 65_65
+1 66_66
+1 69_69
+1 74_74
+1 77_77
+1 78_78
+1 80_80
+1 82_82
+1 85_85
+1 86_86
+1 87_87
+1 8_8
+1 92_92
+1 96_96
+1 9_9
+2 100_100
+2 103_103
+2 104_104
+2 113_113
+2 118_118
+2 120_120
+2 125_125
+2 129_129
+2 12_12
+2 134_134
+2 137_137
+2 146_146
+2 149_149
+2 152_152
+2 15_15
+2 164_164
+2 165_165
+2 172_172
+2 174_174
+2 175_175
+2 176_176
+2 179_179
+2 18_18
+2 191_191
+2 195_195
+2 197_197
+2 200_200
+2 203_203
+2 205_205
+2 207_207
+2 209_209
+2 213_213
+2 216_216
+2 217_217
+2 219_219
+2 221_221
+2 223_223
+2 224_224
+2 229_229
+2 233_233
+2 237_237
+2 238_238
+2 239_239
+2 242_242
+2 24_24
+2 255_255
+2 256_256
+2 265_265
+2 26_26
+2 272_272
+2 278_278
+2 280_280
+2 281_281
+2 282_282
+2 288_288
+2 307_307
+2 309_309
+2 317_317
+2 321_321
+2 322_322
+2 325_325
+2 331_331
+2 333_333
+2 342_342
+2 344_344
+2 353_353
+2 367_367
+2 37_37
+2 382_382
+2 395_395
+2 397_397
+2 399_399
+2 404_404
+2 413_413
+2 414_414
+2 424_424
+2 429_429
+2 42_42
+2 439_439
+2 458_458
+2 459_459
+2 462_462
+2 463_463
+2 478_478
+2 492_492
+2 51_51
+2 58_58
+2 67_67
+2 72_72
+2 76_76
+2 83_83
+2 84_84
+2 95_95
+2 97_97
+2 98_98
+3 0_0
+3 119_119
+3 128_128
+3 167_167
+3 187_187
+3 193_193
+3 199_199
+3 208_208
+3 273_273
+3 298_298
+3 311_311
+3 316_316
+3 318_318
+3 327_327
+3 35_35
+3 369_369
+3 384_384
+3 396_396
+3 403_403
+3 409_409
+3 417_417
+3 430_430
+3 431_431
+3 438_438
+3 454_454
+3 466_466
+3 480_480
+3 498_498
+3 5_5
+3 70_70
+3 90_90
+4 138_138
+4 169_169
+4 277_277
+4 406_406
+4 468_468
+4 489_489
+5 230_230
+5 348_348
+5 401_401
+5 469_469
Added: hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input34.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input34.q.out?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input34.q.out (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input34.q.out Wed Aug 12 06:17:54 2009
@@ -0,0 +1,614 @@
+query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+query: EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) value)) (TOK_SERDE (TOK_SERDENAME 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe')) '/bin/cat' (TOK_ALIASLIST tkey tvalue) (TOK_SERDE (TOK_SERDENAME 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'))))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL tkey)) (TOK_SELEXPR (TOK_TABLE_OR_COL tvalue)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-4 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ tmap:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Transform Operator
+ command: /bin/cat
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: tkey
+ type: string
+ expr: tvalue
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-4
+ Conditional Operator
+ list of dependent Tasks:
+ Move Operator
+ files:
+ hdfs directory: true
+ destination: file:/data/users/njain/hive3/hive3/build/ql/tmp/1093532122/10000
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/data/users/njain/hive3/hive3/build/ql/tmp/754709366/10002
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns:
+ expr: rand()
+ type: double
+ tag: -1
+ value expressions:
+ expr: key
+ type: int
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+query: FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+Input: default/src
+Output: default/dest1
+query: SELECT dest1.* FROM dest1
+Input: default/dest1
+Output: file:/data/users/njain/hive3/hive3/build/ql/tmp/248092577/10000
+238 val_238
+86 val_86
+311 val_311
+27 val_27
+165 val_165
+409 val_409
+255 val_255
+278 val_278
+98 val_98
+484 val_484
+265 val_265
+193 val_193
+401 val_401
+150 val_150
+273 val_273
+224 val_224
+369 val_369
+66 val_66
+128 val_128
+213 val_213
+146 val_146
+406 val_406
+429 val_429
+374 val_374
+152 val_152
+469 val_469
+145 val_145
+495 val_495
+37 val_37
+327 val_327
+281 val_281
+277 val_277
+209 val_209
+15 val_15
+82 val_82
+403 val_403
+166 val_166
+417 val_417
+430 val_430
+252 val_252
+292 val_292
+219 val_219
+287 val_287
+153 val_153
+193 val_193
+338 val_338
+446 val_446
+459 val_459
+394 val_394
+237 val_237
+482 val_482
+174 val_174
+413 val_413
+494 val_494
+207 val_207
+199 val_199
+466 val_466
+208 val_208
+174 val_174
+399 val_399
+396 val_396
+247 val_247
+417 val_417
+489 val_489
+162 val_162
+377 val_377
+397 val_397
+309 val_309
+365 val_365
+266 val_266
+439 val_439
+342 val_342
+367 val_367
+325 val_325
+167 val_167
+195 val_195
+475 val_475
+17 val_17
+113 val_113
+155 val_155
+203 val_203
+339 val_339
+0 val_0
+455 val_455
+128 val_128
+311 val_311
+316 val_316
+57 val_57
+302 val_302
+205 val_205
+149 val_149
+438 val_438
+345 val_345
+129 val_129
+170 val_170
+20 val_20
+489 val_489
+157 val_157
+378 val_378
+221 val_221
+92 val_92
+111 val_111
+47 val_47
+72 val_72
+4 val_4
+280 val_280
+35 val_35
+427 val_427
+277 val_277
+208 val_208
+356 val_356
+399 val_399
+169 val_169
+382 val_382
+498 val_498
+125 val_125
+386 val_386
+437 val_437
+469 val_469
+192 val_192
+286 val_286
+187 val_187
+176 val_176
+54 val_54
+459 val_459
+51 val_51
+138 val_138
+103 val_103
+239 val_239
+213 val_213
+216 val_216
+430 val_430
+278 val_278
+176 val_176
+289 val_289
+221 val_221
+65 val_65
+318 val_318
+332 val_332
+311 val_311
+275 val_275
+137 val_137
+241 val_241
+83 val_83
+333 val_333
+180 val_180
+284 val_284
+12 val_12
+230 val_230
+181 val_181
+67 val_67
+260 val_260
+404 val_404
+384 val_384
+489 val_489
+353 val_353
+373 val_373
+272 val_272
+138 val_138
+217 val_217
+84 val_84
+348 val_348
+466 val_466
+58 val_58
+8 val_8
+411 val_411
+230 val_230
+208 val_208
+348 val_348
+24 val_24
+463 val_463
+431 val_431
+179 val_179
+172 val_172
+42 val_42
+129 val_129
+158 val_158
+119 val_119
+496 val_496
+0 val_0
+322 val_322
+197 val_197
+468 val_468
+393 val_393
+454 val_454
+100 val_100
+298 val_298
+199 val_199
+191 val_191
+418 val_418
+96 val_96
+26 val_26
+165 val_165
+327 val_327
+230 val_230
+205 val_205
+120 val_120
+131 val_131
+51 val_51
+404 val_404
+43 val_43
+436 val_436
+156 val_156
+469 val_469
+468 val_468
+308 val_308
+95 val_95
+196 val_196
+288 val_288
+481 val_481
+457 val_457
+98 val_98
+282 val_282
+197 val_197
+187 val_187
+318 val_318
+318 val_318
+409 val_409
+470 val_470
+137 val_137
+369 val_369
+316 val_316
+169 val_169
+413 val_413
+85 val_85
+77 val_77
+0 val_0
+490 val_490
+87 val_87
+364 val_364
+179 val_179
+118 val_118
+134 val_134
+395 val_395
+282 val_282
+138 val_138
+238 val_238
+419 val_419
+15 val_15
+118 val_118
+72 val_72
+90 val_90
+307 val_307
+19 val_19
+435 val_435
+10 val_10
+277 val_277
+273 val_273
+306 val_306
+224 val_224
+309 val_309
+389 val_389
+327 val_327
+242 val_242
+369 val_369
+392 val_392
+272 val_272
+331 val_331
+401 val_401
+242 val_242
+452 val_452
+177 val_177
+226 val_226
+5 val_5
+497 val_497
+402 val_402
+396 val_396
+317 val_317
+395 val_395
+58 val_58
+35 val_35
+336 val_336
+95 val_95
+11 val_11
+168 val_168
+34 val_34
+229 val_229
+233 val_233
+143 val_143
+472 val_472
+322 val_322
+498 val_498
+160 val_160
+195 val_195
+42 val_42
+321 val_321
+430 val_430
+119 val_119
+489 val_489
+458 val_458
+78 val_78
+76 val_76
+41 val_41
+223 val_223
+492 val_492
+149 val_149
+449 val_449
+218 val_218
+228 val_228
+138 val_138
+453 val_453
+30 val_30
+209 val_209
+64 val_64
+468 val_468
+76 val_76
+74 val_74
+342 val_342
+69 val_69
+230 val_230
+33 val_33
+368 val_368
+103 val_103
+296 val_296
+113 val_113
+216 val_216
+367 val_367
+344 val_344
+167 val_167
+274 val_274
+219 val_219
+239 val_239
+485 val_485
+116 val_116
+223 val_223
+256 val_256
+263 val_263
+70 val_70
+487 val_487
+480 val_480
+401 val_401
+288 val_288
+191 val_191
+5 val_5
+244 val_244
+438 val_438
+128 val_128
+467 val_467
+432 val_432
+202 val_202
+316 val_316
+229 val_229
+469 val_469
+463 val_463
+280 val_280
+2 val_2
+35 val_35
+283 val_283
+331 val_331
+235 val_235
+80 val_80
+44 val_44
+193 val_193
+321 val_321
+335 val_335
+104 val_104
+466 val_466
+366 val_366
+175 val_175
+403 val_403
+483 val_483
+53 val_53
+105 val_105
+257 val_257
+406 val_406
+409 val_409
+190 val_190
+406 val_406
+401 val_401
+114 val_114
+258 val_258
+90 val_90
+203 val_203
+262 val_262
+348 val_348
+424 val_424
+12 val_12
+396 val_396
+201 val_201
+217 val_217
+164 val_164
+431 val_431
+454 val_454
+478 val_478
+298 val_298
+125 val_125
+431 val_431
+164 val_164
+424 val_424
+187 val_187
+382 val_382
+5 val_5
+70 val_70
+397 val_397
+480 val_480
+291 val_291
+24 val_24
+351 val_351
+255 val_255
+104 val_104
+70 val_70
+163 val_163
+438 val_438
+119 val_119
+414 val_414
+200 val_200
+491 val_491
+237 val_237
+439 val_439
+360 val_360
+248 val_248
+479 val_479
+305 val_305
+417 val_417
+199 val_199
+444 val_444
+120 val_120
+429 val_429
+169 val_169
+443 val_443
+323 val_323
+325 val_325
+277 val_277
+230 val_230
+478 val_478
+178 val_178
+468 val_468
+310 val_310
+317 val_317
+333 val_333
+493 val_493
+460 val_460
+207 val_207
+249 val_249
+265 val_265
+480 val_480
+83 val_83
+136 val_136
+353 val_353
+172 val_172
+214 val_214
+462 val_462
+233 val_233
+406 val_406
+133 val_133
+175 val_175
+189 val_189
+454 val_454
+375 val_375
+401 val_401
+421 val_421
+407 val_407
+384 val_384
+256 val_256
+26 val_26
+134 val_134
+67 val_67
+384 val_384
+379 val_379
+18 val_18
+462 val_462
+492 val_492
+100 val_100
+298 val_298
+9 val_9
+341 val_341
+498 val_498
+146 val_146
+458 val_458
+362 val_362
+186 val_186
+285 val_285
+348 val_348
+167 val_167
+18 val_18
+273 val_273
+183 val_183
+281 val_281
+344 val_344
+97 val_97
+469 val_469
+315 val_315
+84 val_84
+28 val_28
+37 val_37
+448 val_448
+152 val_152
+348 val_348
+307 val_307
+194 val_194
+414 val_414
+477 val_477
+222 val_222
+126 val_126
+90 val_90
+169 val_169
+403 val_403
+400 val_400
+200 val_200
+97 val_97
Added: hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input35.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input35.q.out?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input35.q.out (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input35.q.out Wed Aug 12 06:17:54 2009
@@ -0,0 +1,614 @@
+query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+query: EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) value)) (TOK_SERDE (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD '\002'))) '/bin/cat' (TOK_ALIASLIST tkey tvalue) (TOK_SERDE (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD '\002')))))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL tkey)) (TOK_SELEXPR (TOK_TABLE_OR_COL tvalue)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-4 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ tmap:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Transform Operator
+ command: /bin/cat
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: tkey
+ type: string
+ expr: tvalue
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-4
+ Conditional Operator
+ list of dependent Tasks:
+ Move Operator
+ files:
+ hdfs directory: true
+ destination: file:/data/users/njain/hive3/hive3/build/ql/tmp/525283863/10000
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/data/users/njain/hive3/hive3/build/ql/tmp/84195792/10002
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns:
+ expr: rand()
+ type: double
+ tag: -1
+ value expressions:
+ expr: key
+ type: int
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+query: FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+Input: default/src
+Output: default/dest1
+query: SELECT dest1.* FROM dest1
+Input: default/dest1
+Output: file:/data/users/njain/hive3/hive3/build/ql/tmp/1714970451/10000
+238 val_238
+86 val_86
+311 val_311
+27 val_27
+165 val_165
+409 val_409
+255 val_255
+278 val_278
+98 val_98
+484 val_484
+265 val_265
+193 val_193
+401 val_401
+150 val_150
+273 val_273
+224 val_224
+369 val_369
+66 val_66
+128 val_128
+213 val_213
+146 val_146
+406 val_406
+429 val_429
+374 val_374
+152 val_152
+469 val_469
+145 val_145
+495 val_495
+37 val_37
+327 val_327
+281 val_281
+277 val_277
+209 val_209
+15 val_15
+82 val_82
+403 val_403
+166 val_166
+417 val_417
+430 val_430
+252 val_252
+292 val_292
+219 val_219
+287 val_287
+153 val_153
+193 val_193
+338 val_338
+446 val_446
+459 val_459
+394 val_394
+237 val_237
+482 val_482
+174 val_174
+413 val_413
+494 val_494
+207 val_207
+199 val_199
+466 val_466
+208 val_208
+174 val_174
+399 val_399
+396 val_396
+247 val_247
+417 val_417
+489 val_489
+162 val_162
+377 val_377
+397 val_397
+309 val_309
+365 val_365
+266 val_266
+439 val_439
+342 val_342
+367 val_367
+325 val_325
+167 val_167
+195 val_195
+475 val_475
+17 val_17
+113 val_113
+155 val_155
+203 val_203
+339 val_339
+0 val_0
+455 val_455
+128 val_128
+311 val_311
+316 val_316
+57 val_57
+302 val_302
+205 val_205
+149 val_149
+438 val_438
+345 val_345
+129 val_129
+170 val_170
+20 val_20
+489 val_489
+157 val_157
+378 val_378
+221 val_221
+92 val_92
+111 val_111
+47 val_47
+72 val_72
+4 val_4
+280 val_280
+35 val_35
+427 val_427
+277 val_277
+208 val_208
+356 val_356
+399 val_399
+169 val_169
+382 val_382
+498 val_498
+125 val_125
+386 val_386
+437 val_437
+469 val_469
+192 val_192
+286 val_286
+187 val_187
+176 val_176
+54 val_54
+459 val_459
+51 val_51
+138 val_138
+103 val_103
+239 val_239
+213 val_213
+216 val_216
+430 val_430
+278 val_278
+176 val_176
+289 val_289
+221 val_221
+65 val_65
+318 val_318
+332 val_332
+311 val_311
+275 val_275
+137 val_137
+241 val_241
+83 val_83
+333 val_333
+180 val_180
+284 val_284
+12 val_12
+230 val_230
+181 val_181
+67 val_67
+260 val_260
+404 val_404
+384 val_384
+489 val_489
+353 val_353
+373 val_373
+272 val_272
+138 val_138
+217 val_217
+84 val_84
+348 val_348
+466 val_466
+58 val_58
+8 val_8
+411 val_411
+230 val_230
+208 val_208
+348 val_348
+24 val_24
+463 val_463
+431 val_431
+179 val_179
+172 val_172
+42 val_42
+129 val_129
+158 val_158
+119 val_119
+496 val_496
+0 val_0
+322 val_322
+197 val_197
+468 val_468
+393 val_393
+454 val_454
+100 val_100
+298 val_298
+199 val_199
+191 val_191
+418 val_418
+96 val_96
+26 val_26
+165 val_165
+327 val_327
+230 val_230
+205 val_205
+120 val_120
+131 val_131
+51 val_51
+404 val_404
+43 val_43
+436 val_436
+156 val_156
+469 val_469
+468 val_468
+308 val_308
+95 val_95
+196 val_196
+288 val_288
+481 val_481
+457 val_457
+98 val_98
+282 val_282
+197 val_197
+187 val_187
+318 val_318
+318 val_318
+409 val_409
+470 val_470
+137 val_137
+369 val_369
+316 val_316
+169 val_169
+413 val_413
+85 val_85
+77 val_77
+0 val_0
+490 val_490
+87 val_87
+364 val_364
+179 val_179
+118 val_118
+134 val_134
+395 val_395
+282 val_282
+138 val_138
+238 val_238
+419 val_419
+15 val_15
+118 val_118
+72 val_72
+90 val_90
+307 val_307
+19 val_19
+435 val_435
+10 val_10
+277 val_277
+273 val_273
+306 val_306
+224 val_224
+309 val_309
+389 val_389
+327 val_327
+242 val_242
+369 val_369
+392 val_392
+272 val_272
+331 val_331
+401 val_401
+242 val_242
+452 val_452
+177 val_177
+226 val_226
+5 val_5
+497 val_497
+402 val_402
+396 val_396
+317 val_317
+395 val_395
+58 val_58
+35 val_35
+336 val_336
+95 val_95
+11 val_11
+168 val_168
+34 val_34
+229 val_229
+233 val_233
+143 val_143
+472 val_472
+322 val_322
+498 val_498
+160 val_160
+195 val_195
+42 val_42
+321 val_321
+430 val_430
+119 val_119
+489 val_489
+458 val_458
+78 val_78
+76 val_76
+41 val_41
+223 val_223
+492 val_492
+149 val_149
+449 val_449
+218 val_218
+228 val_228
+138 val_138
+453 val_453
+30 val_30
+209 val_209
+64 val_64
+468 val_468
+76 val_76
+74 val_74
+342 val_342
+69 val_69
+230 val_230
+33 val_33
+368 val_368
+103 val_103
+296 val_296
+113 val_113
+216 val_216
+367 val_367
+344 val_344
+167 val_167
+274 val_274
+219 val_219
+239 val_239
+485 val_485
+116 val_116
+223 val_223
+256 val_256
+263 val_263
+70 val_70
+487 val_487
+480 val_480
+401 val_401
+288 val_288
+191 val_191
+5 val_5
+244 val_244
+438 val_438
+128 val_128
+467 val_467
+432 val_432
+202 val_202
+316 val_316
+229 val_229
+469 val_469
+463 val_463
+280 val_280
+2 val_2
+35 val_35
+283 val_283
+331 val_331
+235 val_235
+80 val_80
+44 val_44
+193 val_193
+321 val_321
+335 val_335
+104 val_104
+466 val_466
+366 val_366
+175 val_175
+403 val_403
+483 val_483
+53 val_53
+105 val_105
+257 val_257
+406 val_406
+409 val_409
+190 val_190
+406 val_406
+401 val_401
+114 val_114
+258 val_258
+90 val_90
+203 val_203
+262 val_262
+348 val_348
+424 val_424
+12 val_12
+396 val_396
+201 val_201
+217 val_217
+164 val_164
+431 val_431
+454 val_454
+478 val_478
+298 val_298
+125 val_125
+431 val_431
+164 val_164
+424 val_424
+187 val_187
+382 val_382
+5 val_5
+70 val_70
+397 val_397
+480 val_480
+291 val_291
+24 val_24
+351 val_351
+255 val_255
+104 val_104
+70 val_70
+163 val_163
+438 val_438
+119 val_119
+414 val_414
+200 val_200
+491 val_491
+237 val_237
+439 val_439
+360 val_360
+248 val_248
+479 val_479
+305 val_305
+417 val_417
+199 val_199
+444 val_444
+120 val_120
+429 val_429
+169 val_169
+443 val_443
+323 val_323
+325 val_325
+277 val_277
+230 val_230
+478 val_478
+178 val_178
+468 val_468
+310 val_310
+317 val_317
+333 val_333
+493 val_493
+460 val_460
+207 val_207
+249 val_249
+265 val_265
+480 val_480
+83 val_83
+136 val_136
+353 val_353
+172 val_172
+214 val_214
+462 val_462
+233 val_233
+406 val_406
+133 val_133
+175 val_175
+189 val_189
+454 val_454
+375 val_375
+401 val_401
+421 val_421
+407 val_407
+384 val_384
+256 val_256
+26 val_26
+134 val_134
+67 val_67
+384 val_384
+379 val_379
+18 val_18
+462 val_462
+492 val_492
+100 val_100
+298 val_298
+9 val_9
+341 val_341
+498 val_498
+146 val_146
+458 val_458
+362 val_362
+186 val_186
+285 val_285
+348 val_348
+167 val_167
+18 val_18
+273 val_273
+183 val_183
+281 val_281
+344 val_344
+97 val_97
+469 val_469
+315 val_315
+84 val_84
+28 val_28
+37 val_37
+448 val_448
+152 val_152
+348 val_348
+307 val_307
+194 val_194
+414 val_414
+477 val_477
+222 val_222
+126 val_126
+90 val_90
+169 val_169
+403 val_403
+400 val_400
+200 val_200
+97 val_97
Added: hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input36.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input36.q.out?rev=803371&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input36.q.out (added)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/input36.q.out Wed Aug 12 06:17:54 2009
@@ -0,0 +1,614 @@
+query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+query: EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\003'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) value)) (TOK_SERDE (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD '\002'))) '/bin/cat' (TOK_ALIASLIST tkey tvalue) (TOK_SERDE (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD '\003')))))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL tkey)) (TOK_SELEXPR (TOK_TABLE_OR_COL tvalue)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-4 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ tmap:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Transform Operator
+ command: /bin/cat
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: tkey
+ type: string
+ expr: tvalue
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-4
+ Conditional Operator
+ list of dependent Tasks:
+ Move Operator
+ files:
+ hdfs directory: true
+ destination: file:/data/users/njain/hive3/hive3/build/ql/tmp/1966676963/10000
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/data/users/njain/hive3/hive3/build/ql/tmp/1028977168/10002
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns:
+ expr: rand()
+ type: double
+ tag: -1
+ value expressions:
+ expr: key
+ type: int
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+query: FROM (
+ FROM src
+ SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\003'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+Input: default/src
+Output: default/dest1
+query: SELECT dest1.* FROM dest1
+Input: default/dest1
+Output: file:/data/users/njain/hive3/hive3/build/ql/tmp/1073316238/10000
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
+NULL NULL
Modified: hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/inputddl3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/inputddl3.q.out?rev=803371&r1=803370&r2=803371&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/inputddl3.q.out (original)
+++ hadoop/hive/branches/branch-0.4/ql/src/test/results/clientpositive/inputddl3.q.out Wed Aug 12 06:17:54 2009
@@ -1,7 +1,7 @@
query: EXPLAIN
CREATE TABLE INPUTDDL3(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE
ABSTRACT SYNTAX TREE:
- (TOK_CREATETABLE INPUTDDL3 TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_TABLEROWFORMATFIELD '\t')) TOK_TBLTEXTFILE)
+ (TOK_CREATETABLE INPUTDDL3 TOK_LIKETABLE (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD '\t'))) TOK_TBLTEXTFILE)
STAGE DEPENDENCIES:
Stage-0 is a root stage