You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2008/12/09 00:50:14 UTC
svn commit: r724555 - in /hadoop/hive/trunk: ./
serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/
serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/
Author: zshao
Date: Mon Dec 8 15:50:13 2008
New Revision: 724555
URL: http://svn.apache.org/viewvc?rev=724555&view=rev
Log:
HIVE-93. Dynamic serde to handle _-prefixed column names in DDL. (Pete Wyckoff through zshao)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jjt
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarConstants.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java
hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Dec 8 15:50:13 2008
@@ -33,6 +33,9 @@
BUG FIXES
+ HIVE-93. Dynamic serde to handle _-prefixed column names in DDL.
+ (Pete Wyckoff through zshao)
+
HIVE-127. Fetch Task to pass the jobconf to SerDe. (Namit through zshao)
HIVE-116. Let test depend on deploy. (Johan Oskarsson through zshao)
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java Mon Dec 8 15:50:13 2008
@@ -1,18 +1,17 @@
-/* Generated By:JJTree: Do not edit this line. /home/pwyckoff/projects/hadoop/trunk/VENDOR/hadoop-0.17/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java */
-
+/* Generated By:JavaCC: Do not edit this line. JJTthrift_grammarState.java Version 4.1 */
package org.apache.hadoop.hive.serde2.dynamic_type;
-class JJTthrift_grammarState {
- private java.util.Stack nodes;
- private java.util.Stack marks;
+public class JJTthrift_grammarState {
+ private java.util.List nodes;
+ private java.util.List marks;
- private int sp; // number of nodes on stack
- private int mk; // current mark
+ private int sp; // number of nodes on stack
+ private int mk; // current mark
private boolean node_created;
- JJTthrift_grammarState() {
- nodes = new java.util.Stack();
- marks = new java.util.Stack();
+ public JJTthrift_grammarState() {
+ nodes = new java.util.ArrayList();
+ marks = new java.util.ArrayList();
sp = 0;
mk = 0;
}
@@ -20,62 +19,62 @@
/* Determines whether the current node was actually closed and
pushed. This should only be called in the final user action of a
node scope. */
- boolean nodeCreated() {
+ public boolean nodeCreated() {
return node_created;
}
/* Call this to reinitialize the node stack. It is called
automatically by the parser's ReInit() method. */
- void reset() {
- nodes.removeAllElements();
- marks.removeAllElements();
+ public void reset() {
+ nodes.clear();
+ marks.clear();
sp = 0;
mk = 0;
}
/* Returns the root node of the AST. It only makes sense to call
this after a successful parse. */
- Node rootNode() {
- return (Node)nodes.elementAt(0);
+ public Node rootNode() {
+ return (Node)nodes.get(0);
}
/* Pushes a node on to the stack. */
- void pushNode(Node n) {
- nodes.push(n);
+ public void pushNode(Node n) {
+ nodes.add(n);
++sp;
}
/* Returns the node on the top of the stack, and remove it from the
stack. */
- Node popNode() {
+ public Node popNode() {
if (--sp < mk) {
- mk = ((Integer)marks.pop()).intValue();
+ mk = ((Integer)marks.remove(marks.size()-1)).intValue();
}
- return (Node)nodes.pop();
+ return (Node)nodes.remove(nodes.size()-1);
}
/* Returns the node currently on the top of the stack. */
- Node peekNode() {
- return (Node)nodes.peek();
+ public Node peekNode() {
+ return (Node)nodes.get(nodes.size()-1);
}
/* Returns the number of children on the stack in the current node
scope. */
- int nodeArity() {
+ public int nodeArity() {
return sp - mk;
}
- void clearNodeScope(Node n) {
+ public void clearNodeScope(Node n) {
while (sp > mk) {
popNode();
}
- mk = ((Integer)marks.pop()).intValue();
+ mk = ((Integer)marks.remove(marks.size()-1)).intValue();
}
- void openNodeScope(Node n) {
- marks.push(new Integer(mk));
+ public void openNodeScope(Node n) {
+ marks.add(new Integer(mk));
mk = sp;
n.jjtOpen();
}
@@ -85,8 +84,8 @@
children. That number of nodes are popped from the stack and
made the children of the definite node. Then the definite node
is pushed on to the stack. */
- void closeNodeScope(Node n, int num) {
- mk = ((Integer)marks.pop()).intValue();
+ public void closeNodeScope(Node n, int num) {
+ mk = ((Integer)marks.remove(marks.size()-1)).intValue();
while (num-- > 0) {
Node c = popNode();
c.jjtSetParent(n);
@@ -100,24 +99,25 @@
/* A conditional node is constructed if its condition is true. All
the nodes that have been pushed since the node was opened are
- made children of the the conditional node, which is then pushed
+ made children of the conditional node, which is then pushed
on to the stack. If the condition is false the node is not
constructed and they are left on the stack. */
- void closeNodeScope(Node n, boolean condition) {
+ public void closeNodeScope(Node n, boolean condition) {
if (condition) {
int a = nodeArity();
- mk = ((Integer)marks.pop()).intValue();
+ mk = ((Integer)marks.remove(marks.size()-1)).intValue();
while (a-- > 0) {
- Node c = popNode();
- c.jjtSetParent(n);
- n.jjtAddChild(c, a);
+ Node c = popNode();
+ c.jjtSetParent(n);
+ n.jjtAddChild(c, a);
}
n.jjtClose();
pushNode(n);
node_created = true;
} else {
- mk = ((Integer)marks.pop()).intValue();
+ mk = ((Integer)marks.remove(marks.size()-1)).intValue();
node_created = false;
}
}
}
+/* JavaCC - OriginalChecksum=67039445e12d18e18e63124a33879cd3 (do not edit this line) */
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java Mon Dec 8 15:50:13 2008
@@ -2142,9 +2142,13 @@
throw new Error("Missing return statement in function");
}
+ /** Generated Token Manager. */
public thrift_grammarTokenManager token_source;
SimpleCharStream jj_input_stream;
- public Token token, jj_nt;
+ /** Current token. */
+ public Token token;
+ /** Next token. */
+ public Token jj_nt;
private int jj_ntk;
private int jj_gen;
final private int[] jj_la1 = new int[33];
@@ -2152,23 +2156,25 @@
static private int[] jj_la1_1;
static private int[] jj_la1_2;
static {
- jj_la1_0();
- jj_la1_1();
- jj_la1_2();
+ jj_la1_init_0();
+ jj_la1_init_1();
+ jj_la1_init_2();
}
- private static void jj_la1_0() {
+ private static void jj_la1_init_0() {
jj_la1_0 = new int[] {0x0,0x100,0xa3fee00,0xa3fee00,0x23fee00,0x100,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xb0000000,0x0,0x0,0x0,0x0,0xa0000000,0x0,0x0,0x0,0x0,0xa0000000,0xb0000000,0xa0000000,};
}
- private static void jj_la1_1() {
+ private static void jj_la1_init_1() {
jj_la1_1 = new int[] {0x18000000,0x18019c20,0x0,0x0,0x0,0x19c20,0x11c20,0x18000000,0x400000,0x80000000,0x18000000,0x2000000,0x18000000,0x18000000,0x22700000,0x18000000,0x22700000,0x18000000,0x22700000,0x22700000,0x4003cf,0x2000,0x18000000,0x200,0x4000,0x5e01cf,0x100000,0x18000000,0xe0000,0x80000000,0x1cf,0x4001cf,0x4001cf,};
}
- private static void jj_la1_2() {
+ private static void jj_la1_init_2() {
jj_la1_2 = new int[] {0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x1,0x0,0x1,0x1,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,};
}
+ /** Constructor with InputStream. */
public thrift_grammar(java.io.InputStream stream) {
this(stream, null);
}
+ /** Constructor with InputStream and supplied encoding */
public thrift_grammar(java.io.InputStream stream, String encoding) {
try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
token_source = new thrift_grammarTokenManager(jj_input_stream);
@@ -2178,9 +2184,11 @@
for (int i = 0; i < 33; i++) jj_la1[i] = -1;
}
+ /** Reinitialise. */
public void ReInit(java.io.InputStream stream) {
ReInit(stream, null);
}
+ /** Reinitialise. */
public void ReInit(java.io.InputStream stream, String encoding) {
try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
token_source.ReInit(jj_input_stream);
@@ -2191,6 +2199,7 @@
for (int i = 0; i < 33; i++) jj_la1[i] = -1;
}
+ /** Constructor. */
public thrift_grammar(java.io.Reader stream) {
jj_input_stream = new SimpleCharStream(stream, 1, 1);
token_source = new thrift_grammarTokenManager(jj_input_stream);
@@ -2200,6 +2209,7 @@
for (int i = 0; i < 33; i++) jj_la1[i] = -1;
}
+ /** Reinitialise. */
public void ReInit(java.io.Reader stream) {
jj_input_stream.ReInit(stream, 1, 1);
token_source.ReInit(jj_input_stream);
@@ -2210,6 +2220,7 @@
for (int i = 0; i < 33; i++) jj_la1[i] = -1;
}
+ /** Constructor with generated Token Manager. */
public thrift_grammar(thrift_grammarTokenManager tm) {
token_source = tm;
token = new Token();
@@ -2218,6 +2229,7 @@
for (int i = 0; i < 33; i++) jj_la1[i] = -1;
}
+ /** Reinitialise. */
public void ReInit(thrift_grammarTokenManager tm) {
token_source = tm;
token = new Token();
@@ -2227,7 +2239,7 @@
for (int i = 0; i < 33; i++) jj_la1[i] = -1;
}
- final private Token jj_consume_token(int kind) throws ParseException {
+ private Token jj_consume_token(int kind) throws ParseException {
Token oldToken;
if ((oldToken = token).next != null) token = token.next;
else token = token.next = token_source.getNextToken();
@@ -2241,6 +2253,8 @@
throw generateParseException();
}
+
+/** Get the next Token. */
final public Token getNextToken() {
if (token.next != null) token = token.next;
else token = token.next = token_source.getNextToken();
@@ -2249,6 +2263,7 @@
return token;
}
+/** Get the specific Token. */
final public Token getToken(int index) {
Token t = token;
for (int i = 0; i < index; i++) {
@@ -2258,23 +2273,21 @@
return t;
}
- final private int jj_ntk() {
+ private int jj_ntk() {
if ((jj_nt=token.next) == null)
return (jj_ntk = (token.next=token_source.getNextToken()).kind);
else
return (jj_ntk = jj_nt.kind);
}
- private java.util.Vector jj_expentries = new java.util.Vector();
+ private java.util.List jj_expentries = new java.util.ArrayList();
private int[] jj_expentry;
private int jj_kind = -1;
+ /** Generate ParseException. */
public ParseException generateParseException() {
- jj_expentries.removeAllElements();
+ jj_expentries.clear();
boolean[] la1tokens = new boolean[71];
- for (int i = 0; i < 71; i++) {
- la1tokens[i] = false;
- }
if (jj_kind >= 0) {
la1tokens[jj_kind] = true;
jj_kind = -1;
@@ -2298,19 +2311,21 @@
if (la1tokens[i]) {
jj_expentry = new int[1];
jj_expentry[0] = i;
- jj_expentries.addElement(jj_expentry);
+ jj_expentries.add(jj_expentry);
}
}
int[][] exptokseq = new int[jj_expentries.size()][];
for (int i = 0; i < jj_expentries.size(); i++) {
- exptokseq[i] = (int[])jj_expentries.elementAt(i);
+ exptokseq[i] = (int[])jj_expentries.get(i);
}
return new ParseException(token, exptokseq, tokenImage);
}
+ /** Enable tracing. */
final public void enable_tracing() {
}
+ /** Disable tracing. */
final public void disable_tracing() {
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj Mon Dec 8 15:50:13 2008
@@ -1,6 +1,8 @@
-/*@bgen(jjtree) Generated By:JJTree: Do not edit this line. /home/pwyckoff/projects/hadoop/trunk/VENDOR/hadoop-0.17/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj */
-/*@egen*/options {
- STATIC = false;
+/*@bgen(jjtree) Generated By:JJTree: Do not edit this line. /home/pwyckoff/hive-core/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj */
+/*@egen*/options {
+
+ STATIC = false;
+
}
@@ -165,7 +167,7 @@
|
<tok_double_constant: ["+","-"](<DIGIT>)*"."(<DIGIT>)+(["e","E"](["+","-"])?(<DIGIT>)+)?>
|
-<IDENTIFIER: <LETTER>(<LETTER>|<DIGIT>|"."|"_")*>
+<IDENTIFIER: (<DIGIT>|<LETTER>|"_")(<LETTER>|<DIGIT>|"."|"_")*>
|
<#LETTER: (["a"-"z", "A"-"Z" ]) >
|
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jjt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jjt?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jjt (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jjt Mon Dec 8 15:50:13 2008
@@ -163,7 +163,7 @@
|
<tok_double_constant: ["+","-"](<DIGIT>)*"."(<DIGIT>)+(["e","E"](["+","-"])?(<DIGIT>)+)?>
|
-<IDENTIFIER: <LETTER>(<LETTER>|<DIGIT>|"."|"_")*>
+<IDENTIFIER: (<DIGIT>|<LETTER>|"_")(<LETTER>|<DIGIT>|"."|"_")*>
|
<#LETTER: (["a"-"z", "A"-"Z" ]) >
|
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarConstants.java?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarConstants.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarConstants.java Mon Dec 8 15:50:13 2008
@@ -1,63 +1,122 @@
/* Generated By:JJTree&JavaCC: Do not edit this line. thrift_grammarConstants.java */
package org.apache.hadoop.hive.serde2.dynamic_type;
+
+/**
+ * Token literal values and constants.
+ * Generated by org.javacc.parser.OtherFilesGen#start()
+ */
public interface thrift_grammarConstants {
+ /** End of File. */
int EOF = 0;
+ /** RegularExpression Id. */
int tok_const = 8;
+ /** RegularExpression Id. */
int tok_namespace = 9;
+ /** RegularExpression Id. */
int tok_cpp_namespace = 10;
+ /** RegularExpression Id. */
int tok_cpp_include = 11;
+ /** RegularExpression Id. */
int tok_cpp_type = 12;
+ /** RegularExpression Id. */
int tok_java_package = 13;
+ /** RegularExpression Id. */
int tok_cocoa_prefix = 14;
+ /** RegularExpression Id. */
int tok_csharp_namespace = 15;
+ /** RegularExpression Id. */
int tok_php_namespace = 16;
+ /** RegularExpression Id. */
int tok_py_module = 17;
+ /** RegularExpression Id. */
int tok_perl_package = 18;
+ /** RegularExpression Id. */
int tok_ruby_namespace = 19;
+ /** RegularExpression Id. */
int tok_smalltalk_category = 20;
+ /** RegularExpression Id. */
int tok_smalltalk_prefix = 21;
+ /** RegularExpression Id. */
int tok_xsd_all = 22;
+ /** RegularExpression Id. */
int tok_xsd_optional = 23;
+ /** RegularExpression Id. */
int tok_xsd_nillable = 24;
+ /** RegularExpression Id. */
int tok_xsd_namespace = 25;
+ /** RegularExpression Id. */
int tok_xsd_attrs = 26;
+ /** RegularExpression Id. */
int tok_include = 27;
+ /** RegularExpression Id. */
int tok_void = 28;
+ /** RegularExpression Id. */
int tok_bool = 29;
+ /** RegularExpression Id. */
int tok_byte = 30;
+ /** RegularExpression Id. */
int tok_i16 = 31;
+ /** RegularExpression Id. */
int tok_i32 = 32;
+ /** RegularExpression Id. */
int tok_i64 = 33;
+ /** RegularExpression Id. */
int tok_double = 34;
+ /** RegularExpression Id. */
int tok_string = 35;
+ /** RegularExpression Id. */
int tok_slist = 36;
+ /** RegularExpression Id. */
int tok_senum = 37;
+ /** RegularExpression Id. */
int tok_map = 38;
+ /** RegularExpression Id. */
int tok_list = 39;
+ /** RegularExpression Id. */
int tok_set = 40;
+ /** RegularExpression Id. */
int tok_async = 41;
+ /** RegularExpression Id. */
int tok_typedef = 42;
+ /** RegularExpression Id. */
int tok_struct = 43;
+ /** RegularExpression Id. */
int tok_exception = 44;
+ /** RegularExpression Id. */
int tok_extends = 45;
+ /** RegularExpression Id. */
int tok_throws = 46;
+ /** RegularExpression Id. */
int tok_service = 47;
+ /** RegularExpression Id. */
int tok_enum = 48;
+ /** RegularExpression Id. */
int tok_required = 49;
+ /** RegularExpression Id. */
int tok_optional = 50;
+ /** RegularExpression Id. */
int tok_skip = 51;
+ /** RegularExpression Id. */
int tok_int_constant = 52;
+ /** RegularExpression Id. */
int tok_double_constant = 53;
+ /** RegularExpression Id. */
int IDENTIFIER = 54;
+ /** RegularExpression Id. */
int LETTER = 55;
+ /** RegularExpression Id. */
int DIGIT = 56;
+ /** RegularExpression Id. */
int tok_literal = 57;
+ /** RegularExpression Id. */
int tok_st_identifier = 58;
+ /** Lexical state. */
int DEFAULT = 0;
+ /** Literal token values. */
String[] tokenImage = {
"<EOF>",
"\" \"",
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java Mon Dec 8 15:50:13 2008
@@ -7,9 +7,13 @@
import com.facebook.thrift.transport.*;
import org.apache.hadoop.hive.serde2.dynamic_type.*;
+/** Token Manager. */
public class thrift_grammarTokenManager implements thrift_grammarConstants
{
+
+ /** Debug output. */
public java.io.PrintStream debugStream = System.out;
+ /** Set debug output. */
public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1)
{
@@ -182,21 +186,13 @@
{
return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0, active1), pos + 1);
}
-private final int jjStopAtPos(int pos, int kind)
+private int jjStopAtPos(int pos, int kind)
{
jjmatchedKind = kind;
jjmatchedPos = pos;
return pos + 1;
}
-private final int jjStartNfaWithStates_0(int pos, int kind, int state)
-{
- jjmatchedKind = kind;
- jjmatchedPos = pos;
- try { curChar = input_stream.readChar(); }
- catch(java.io.IOException e) { return pos + 1; }
- return jjMoveNfa_0(state, pos + 1);
-}
-private final int jjMoveStringLiteralDfa0_0()
+private int jjMoveStringLiteralDfa0_0()
{
switch(curChar)
{
@@ -262,7 +258,7 @@
return jjMoveNfa_0(0, 0);
}
}
-private final int jjMoveStringLiteralDfa1_0(long active0)
+private int jjMoveStringLiteralDfa1_0(long active0)
{
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) {
@@ -312,7 +308,7 @@
}
return jjStartNfa_0(0, active0, 0L);
}
-private final int jjMoveStringLiteralDfa2_0(long old0, long active0)
+private int jjMoveStringLiteralDfa2_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(0, old0, 0L);
@@ -380,7 +376,7 @@
}
return jjStartNfa_0(1, active0, 0L);
}
-private final int jjMoveStringLiteralDfa3_0(long old0, long active0)
+private int jjMoveStringLiteralDfa3_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(1, old0, 0L);
@@ -440,7 +436,7 @@
}
return jjStartNfa_0(2, active0, 0L);
}
-private final int jjMoveStringLiteralDfa4_0(long old0, long active0)
+private int jjMoveStringLiteralDfa4_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(2, old0, 0L);
@@ -494,7 +490,7 @@
}
return jjStartNfa_0(3, active0, 0L);
}
-private final int jjMoveStringLiteralDfa5_0(long old0, long active0)
+private int jjMoveStringLiteralDfa5_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(3, old0, 0L);
@@ -546,7 +542,7 @@
}
return jjStartNfa_0(4, active0, 0L);
}
-private final int jjMoveStringLiteralDfa6_0(long old0, long active0)
+private int jjMoveStringLiteralDfa6_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(4, old0, 0L);
@@ -596,7 +592,7 @@
}
return jjStartNfa_0(5, active0, 0L);
}
-private final int jjMoveStringLiteralDfa7_0(long old0, long active0)
+private int jjMoveStringLiteralDfa7_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(5, old0, 0L);
@@ -636,7 +632,7 @@
}
return jjStartNfa_0(6, active0, 0L);
}
-private final int jjMoveStringLiteralDfa8_0(long old0, long active0)
+private int jjMoveStringLiteralDfa8_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(6, old0, 0L);
@@ -674,7 +670,7 @@
}
return jjStartNfa_0(7, active0, 0L);
}
-private final int jjMoveStringLiteralDfa9_0(long old0, long active0)
+private int jjMoveStringLiteralDfa9_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(7, old0, 0L);
@@ -708,7 +704,7 @@
}
return jjStartNfa_0(8, active0, 0L);
}
-private final int jjMoveStringLiteralDfa10_0(long old0, long active0)
+private int jjMoveStringLiteralDfa10_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(8, old0, 0L);
@@ -740,7 +736,7 @@
}
return jjStartNfa_0(9, active0, 0L);
}
-private final int jjMoveStringLiteralDfa11_0(long old0, long active0)
+private int jjMoveStringLiteralDfa11_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(9, old0, 0L);
@@ -780,7 +776,7 @@
}
return jjStartNfa_0(10, active0, 0L);
}
-private final int jjMoveStringLiteralDfa12_0(long old0, long active0)
+private int jjMoveStringLiteralDfa12_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(10, old0, 0L);
@@ -810,7 +806,7 @@
}
return jjStartNfa_0(11, active0, 0L);
}
-private final int jjMoveStringLiteralDfa13_0(long old0, long active0)
+private int jjMoveStringLiteralDfa13_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(11, old0, 0L);
@@ -834,7 +830,7 @@
}
return jjStartNfa_0(12, active0, 0L);
}
-private final int jjMoveStringLiteralDfa14_0(long old0, long active0)
+private int jjMoveStringLiteralDfa14_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(12, old0, 0L);
@@ -856,7 +852,7 @@
}
return jjStartNfa_0(13, active0, 0L);
}
-private final int jjMoveStringLiteralDfa15_0(long old0, long active0)
+private int jjMoveStringLiteralDfa15_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(13, old0, 0L);
@@ -882,7 +878,7 @@
}
return jjStartNfa_0(14, active0, 0L);
}
-private final int jjMoveStringLiteralDfa16_0(long old0, long active0)
+private int jjMoveStringLiteralDfa16_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(14, old0, 0L);
@@ -900,7 +896,7 @@
}
return jjStartNfa_0(15, active0, 0L);
}
-private final int jjMoveStringLiteralDfa17_0(long old0, long active0)
+private int jjMoveStringLiteralDfa17_0(long old0, long active0)
{
if (((active0 &= old0)) == 0L)
return jjStartNfa_0(15, old0, 0L);
@@ -920,47 +916,24 @@
}
return jjStartNfa_0(16, active0, 0L);
}
-private final void jjCheckNAdd(int state)
-{
- if (jjrounds[state] != jjround)
- {
- jjstateSet[jjnewStateCnt++] = state;
- jjrounds[state] = jjround;
- }
-}
-private final void jjAddStates(int start, int end)
-{
- do {
- jjstateSet[jjnewStateCnt++] = jjnextStates[start];
- } while (start++ != end);
-}
-private final void jjCheckNAddTwoStates(int state1, int state2)
-{
- jjCheckNAdd(state1);
- jjCheckNAdd(state2);
-}
-private final void jjCheckNAddStates(int start, int end)
-{
- do {
- jjCheckNAdd(jjnextStates[start]);
- } while (start++ != end);
-}
-private final void jjCheckNAddStates(int start)
+private int jjStartNfaWithStates_0(int pos, int kind, int state)
{
- jjCheckNAdd(jjnextStates[start]);
- jjCheckNAdd(jjnextStates[start + 1]);
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return pos + 1; }
+ return jjMoveNfa_0(state, pos + 1);
}
static final long[] jjbitVec0 = {
0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
};
-private final int jjMoveNfa_0(int startState, int curPos)
+private int jjMoveNfa_0(int startState, int curPos)
{
- int[] nextStates;
int startsAt = 0;
jjnewStateCnt = 35;
int i = 1;
jjstateSet[0] = startState;
- int j, kind = 0x7fffffff;
+ int kind = 0x7fffffff;
for (;;)
{
if (++jjround == 0x7fffffff)
@@ -968,7 +941,7 @@
if (curChar < 64)
{
long l = 1L << curChar;
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -989,9 +962,9 @@
case 0:
if ((0x3ff000000000000L & l) != 0L)
{
- if (kind > 52)
- kind = 52;
- jjCheckNAdd(5);
+ if (kind > 54)
+ kind = 54;
+ jjCheckNAdd(7);
}
else if ((0x280000000000L & l) != 0L)
jjCheckNAddStates(0, 2);
@@ -1003,7 +976,13 @@
jjCheckNAddTwoStates(9, 10);
else if (curChar == 35)
jjCheckNAddStates(5, 7);
- if (curChar == 45)
+ if ((0x3ff000000000000L & l) != 0L)
+ {
+ if (kind > 52)
+ kind = 52;
+ jjCheckNAdd(5);
+ }
+ else if (curChar == 45)
{
if (kind > 58)
kind = 58;
@@ -1033,6 +1012,13 @@
kind = 52;
jjCheckNAdd(5);
break;
+ case 6:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 54)
+ kind = 54;
+ jjCheckNAdd(7);
+ break;
case 7:
if ((0x3ff400000000000L & l) == 0L)
break;
@@ -1163,7 +1149,7 @@
else if (curChar < 128)
{
long l = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -1182,24 +1168,24 @@
}
break;
case 0:
+ if ((0x7fffffe87fffffeL & l) != 0L)
+ {
+ if (kind > 54)
+ kind = 54;
+ jjCheckNAdd(7);
+ }
if ((0x7fffffe07fffffeL & l) != 0L)
{
if (kind > 58)
kind = 58;
jjCheckNAdd(15);
}
- if ((0x7fffffe07fffffeL & l) != 0L)
- {
- if (kind > 54)
- kind = 54;
- jjCheckNAdd(7);
- }
break;
case 1:
jjAddStates(5, 7);
break;
case 6:
- if ((0x7fffffe07fffffeL & l) == 0L)
+ if ((0x7fffffe87fffffeL & l) == 0L)
break;
if (kind > 54)
kind = 54;
@@ -1254,7 +1240,7 @@
{
int i2 = (curChar & 0xff) >> 6;
long l2 = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -1304,6 +1290,8 @@
5, 29, 30, 17, 22, 1, 2, 4, 18, 19, 21, 25, 27, 9, 10, 12,
13, 33, 34,
};
+
+/** Token literal values. */
public static final String[] jjstrLiteralImages = {
"", null, null, null, null, null, null, null, "\143\157\156\163\164",
"\156\141\155\145\163\160\141\143\145", "\143\160\160\137\156\141\155\145\163\160\141\143\145",
@@ -1323,8 +1311,10 @@
"\163\145\162\166\151\143\145", "\145\156\165\155", "\162\145\161\165\151\162\145\144",
"\157\160\164\151\157\156\141\154", "\163\153\151\160", null, null, null, null, null, null, null, "\54", "\73",
"\173", "\175", "\75", "\133", "\135", "\72", "\50", "\51", "\74", "\76", };
+
+/** Lexer state names. */
public static final String[] lexStateNames = {
- "DEFAULT",
+ "DEFAULT",
};
static final long[] jjtoToken = {
0xfe7fffffffffff01L, 0x7fL,
@@ -1336,15 +1326,20 @@
private final int[] jjrounds = new int[35];
private final int[] jjstateSet = new int[70];
protected char curChar;
+/** Constructor. */
public thrift_grammarTokenManager(SimpleCharStream stream){
if (SimpleCharStream.staticFlag)
throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
input_stream = stream;
}
+
+/** Constructor. */
public thrift_grammarTokenManager(SimpleCharStream stream, int lexState){
this(stream);
SwitchTo(lexState);
}
+
+/** Reinitialise parser. */
public void ReInit(SimpleCharStream stream)
{
jjmatchedPos = jjnewStateCnt = 0;
@@ -1352,18 +1347,22 @@
input_stream = stream;
ReInitRounds();
}
-private final void ReInitRounds()
+private void ReInitRounds()
{
int i;
jjround = 0x80000001;
for (i = 35; i-- > 0;)
jjrounds[i] = 0x80000000;
}
+
+/** Reinitialise parser. */
public void ReInit(SimpleCharStream stream, int lexState)
{
ReInit(stream);
SwitchTo(lexState);
}
+
+/** Switch to specified lex state. */
public void SwitchTo(int lexState)
{
if (lexState >= 1 || lexState < 0)
@@ -1374,14 +1373,27 @@
protected Token jjFillToken()
{
- Token t = Token.newToken(jjmatchedKind);
- t.kind = jjmatchedKind;
+ final Token t;
+ final String curTokenImage;
+ final int beginLine;
+ final int endLine;
+ final int beginColumn;
+ final int endColumn;
String im = jjstrLiteralImages[jjmatchedKind];
- t.image = (im == null) ? input_stream.GetImage() : im;
- t.beginLine = input_stream.getBeginLine();
- t.beginColumn = input_stream.getBeginColumn();
- t.endLine = input_stream.getEndLine();
- t.endColumn = input_stream.getEndColumn();
+ curTokenImage = (im == null) ? input_stream.GetImage() : im;
+ beginLine = input_stream.getBeginLine();
+ beginColumn = input_stream.getBeginColumn();
+ endLine = input_stream.getEndLine();
+ endColumn = input_stream.getEndColumn();
+ t = Token.newToken(jjmatchedKind);
+ t.kind = jjmatchedKind;
+ t.image = curTokenImage;
+
+ t.beginLine = beginLine;
+ t.endLine = endLine;
+ t.beginColumn = beginColumn;
+ t.endColumn = endColumn;
+
return t;
}
@@ -1392,22 +1404,21 @@
int jjmatchedPos;
int jjmatchedKind;
+/** Get the next Token. */
public Token getNextToken()
{
- int kind;
- Token specialToken = null;
Token matchedToken;
int curPos = 0;
EOFLoop :
for (;;)
- {
- try
- {
+ {
+ try
+ {
curChar = input_stream.BeginToken();
- }
+ }
catch(java.io.IOException e)
- {
+ {
jjmatchedKind = 0;
matchedToken = jjFillToken();
return matchedToken;
@@ -1458,4 +1469,31 @@
}
}
+private void jjCheckNAdd(int state)
+{
+ if (jjrounds[state] != jjround)
+ {
+ jjstateSet[jjnewStateCnt++] = state;
+ jjrounds[state] = jjround;
+ }
+}
+private void jjAddStates(int start, int end)
+{
+ do {
+ jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+ } while (start++ != end);
+}
+private void jjCheckNAddTwoStates(int state1, int state2)
+{
+ jjCheckNAdd(state1);
+ jjCheckNAdd(state2);
+}
+
+private void jjCheckNAddStates(int start, int end)
+{
+ do {
+ jjCheckNAdd(jjnextStates[start]);
+ } while (start++ != end);
+}
+
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java Mon Dec 8 15:50:13 2008
@@ -1,5 +1,4 @@
-/* Generated By:JJTree: Do not edit this line. /home/pwyckoff/projects/hadoop/trunk/VENDOR/hadoop-0.17/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java */
-
+/* Generated By:JavaCC: Do not edit this line. thrift_grammarTreeConstants.java Version 4.1 */
package org.apache.hadoop.hive.serde2.dynamic_type;
public interface thrift_grammarTreeConstants
@@ -103,3 +102,4 @@
"TypeList",
};
}
+/* JavaCC - OriginalChecksum=7edd3e61472739e9fede55c18a336638 (do not edit this line) */
Modified: hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java?rev=724555&r1=724554&r2=724555&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java (original)
+++ hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java Mon Dec 8 15:50:13 2008
@@ -107,7 +107,7 @@
schema.setProperty(Constants.SERIALIZATION_FORMAT, protocol);
schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test");
schema.setProperty(Constants.SERIALIZATION_DDL,
- "struct test { i32 hello, list<string> bye, map<string,i32> another, i32 nhello, double d, double nd}");
+ "struct test { i32 _hello, list<string> 2bye, map<string,i32> another, i32 nhello, double d, double nd}");
schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString());
HashMap<String, String> p = additionalParams.get(pp);
if (p != null) {