You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2009/08/12 00:27:47 UTC
svn commit: r803312 [7/16] - in /hadoop/pig/trunk: ./ contrib/zebra/
contrib/zebra/docs/ contrib/zebra/src/ contrib/zebra/src/java/
contrib/zebra/src/java/org/ contrib/zebra/src/java/org/apache/
contrib/zebra/src/java/org/apache/hadoop/ contrib/zebra/s...
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.java?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.java Tue Aug 11 22:27:44 2009
@@ -0,0 +1,1060 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. TableStorageParser.java */
+package org.apache.hadoop.zebra.types;
+import java.io.*;
+import java.util.*;
+
+public class TableStorageParser implements TableStorageParserConstants {
+ public static void main( String[] args )
+ throws ParseException, TokenMgrError, java.io.FileNotFoundException, java.io.IOException, java.lang.ClassNotFoundException {
+ /*
+ FileInputStream input = new FileInputStream("/tmp/schema");
+ ObjectInputStream objin = new ObjectInputStream(input);
+ Schema schema = (Schema) objin.readObject();
+ objin.close();
+ input = new FileInputStream("store");
+ System.out.println("Table Schema:");
+ System.out.println("--------------------------");
+ System.out.println(schema.toString());
+ Partition partition = new Partition(
+ TableStorageParser parser = new TableStorageParser(schema.toString(), schema) ;
+ ArrayList<CGSchema> schemas = parser.StorageSchema();
+ System.out.println("Column Groups:");
+ System.out.println("--------------------------");
+ for (int i = 0; i < schemas.size(); i++)
+ {
+ System.out.println(schemas.get(i).toString());
+ System.out.println("==========================");
+ }
+ System.out.println("Table Schema with Storage Hints");
+ System.out.println(schema.toString());
+ System.out.println("==========================");
+ System.out.println("Column Map:");
+ System.out.println("--------------------------");
+ Map<String, Schema.ColumnMappingEntry> colmap = schema.getColMap();
+ if (!colmap.isEmpty())
+ {
+ Set<Map.Entry<String, Schema.ColumnMappingEntry>> entrySet = colmap.entrySet();
+ Map.Entry<String, Schema.ColumnMappingEntry>[] array = entrySet.toArray(new Map.Entry[entrySet.size()]);
+ for (int j = 0; j < colmap.size(); j++)
+ {
+ System.out.println("[Column = " + array[j].getKey() +" CG= "+ array[j].getValue().getCGIndex()+"."+array[j].getValue().getColumnIndex()+"]");
+ }
+ }
+ Schema.ColumnMappingEntry cm;
+ if (args.length > 0)
+ {
+ System.out.println("==========================");
+ System.out.println("Column CG Index:");
+ System.out.println("--------------------------");
+ if ((cm = schema.getColMapping(args[0])) != null)
+ System.out.println(args[0]+":CG= "+ cm.getCGIndex()+"."+cm.getColumnIndex());
+ else
+ System.out.println(args[0]+":Column not found !");
+ }
+ Schema.ColumnSchema fs;
+ if (args.length > 0)
+ {
+ System.out.println("==========================");
+ System.out.println("Column Schema:");
+ System.out.println("--------------------------");
+ StringBuilder sb = new StringBuilder();
+ if ((fs = schema.getColumnSchema(args[0])) != null)
+ {
+ // print out a particular column's CG index
+ // System.out.println(args[0]+":Schema = "+ fs.name +":"+ColumnType.genTypeToNameMap().get(fs.type));
+ Schema.stringifySchema(sb, fs.schema, fs.type, false, false);
+ if (sb.toString().isEmpty())
+ System.out.println(args[0]+":Schema = "+ ColumnType.findTypeName(fs.type));
+ else
+ System.out.println(args[0]+":Schema = "+ ColumnType.findTypeName(fs.type)+":"+sb.toString());
+ } else
+ System.out.println(args[0]+":Column not found !");
+ }
+ */
+ }
+ public TableStorageParser(java.io.Reader in, Partition partition, Schema schema) { this(in); mSchema = schema; this.partition = partition;}
+ private Schema mSchema;
+ private int mDefaultCGIndex = -1;
+ private String mCompressor = "lzo2", mSerializer = "pig";
+ private int mCGCount = 0;
+ private Partition partition;
+
+ final public ArrayList<CGSchema> StorageSchema() throws ParseException, ParseException {
+ ArrayList<CGSchema> s = new ArrayList();
+ CGSchema fs;
+ CGSchema defaultSchema;
+ try {
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case 0:
+ jj_consume_token(0);
+ defaultSchema = partition.generateDefaultCGSchema(mCompressor, mSerializer, 0);
+ if (defaultSchema != null)
+ s.add(defaultSchema);
+ {if (true) return s;}
+ break;
+ case COMPRESS:
+ case SERIALIZE:
+ case 18:
+ fs = FieldSchema();
+ mCGCount++; if (fs != null) s.add(fs);
+ label_1:
+ while (true) {
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case 17:
+ ;
+ break;
+ default:
+ jj_la1[0] = jj_gen;
+ break label_1;
+ }
+ jj_consume_token(17);
+ fs = FieldSchema();
+ mCGCount++; if (fs != null) s.add(fs);
+ }
+ jj_consume_token(0);
+ defaultSchema = partition.generateDefaultCGSchema(mCompressor, mSerializer, mDefaultCGIndex == -1 ? mDefaultCGIndex = mCGCount++ : mDefaultCGIndex);
+ if (defaultSchema != null)
+ s.add(defaultSchema);
+ {if (true) return s;}
+ break;
+ default:
+ jj_la1[1] = jj_gen;
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ } catch (TokenMgrError e) {
+ {if (true) throw new ParseException(e.getMessage());}
+ }
+ throw new Error("Missing return statement in function");
+ }
+
+ final public CGSchema FieldSchema() throws ParseException {
+ Token t1 = null, t2 = null;
+ Schema fs = null;
+ CGSchema cs = null;
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case COMPRESS:
+ jj_consume_token(COMPRESS);
+ t1 = jj_consume_token(IDENTIFIER);
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case SERIALIZE:
+ jj_consume_token(SERIALIZE);
+ t2 = jj_consume_token(IDENTIFIER);
+ break;
+ default:
+ jj_la1[2] = jj_gen;
+ ;
+ }
+ break;
+ case SERIALIZE:
+ jj_consume_token(SERIALIZE);
+ t2 = jj_consume_token(IDENTIFIER);
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case COMPRESS:
+ jj_consume_token(COMPRESS);
+ t1 = jj_consume_token(IDENTIFIER);
+ break;
+ default:
+ jj_la1[3] = jj_gen;
+ ;
+ }
+ break;
+ default:
+ jj_la1[4] = jj_gen;
+ if (jj_2_1(100)) {
+ jj_consume_token(18);
+ fs = RecordSchema(null);
+ jj_consume_token(19);
+ jj_consume_token(COMPRESS);
+ t1 = jj_consume_token(COMPRESSOR);
+ jj_consume_token(SERIALIZE);
+ t2 = jj_consume_token(SERIALIZER);
+ } else if (jj_2_2(100)) {
+ jj_consume_token(18);
+ fs = RecordSchema(null);
+ jj_consume_token(19);
+ jj_consume_token(SERIALIZE);
+ t2 = jj_consume_token(SERIALIZER);
+ jj_consume_token(COMPRESS);
+ t1 = jj_consume_token(COMPRESSOR);
+ } else if (jj_2_3(100)) {
+ jj_consume_token(18);
+ fs = RecordSchema(null);
+ jj_consume_token(19);
+ jj_consume_token(COMPRESS);
+ t1 = jj_consume_token(COMPRESSOR);
+ } else if (jj_2_4(100)) {
+ jj_consume_token(18);
+ fs = RecordSchema(null);
+ jj_consume_token(19);
+ jj_consume_token(SERIALIZE);
+ t2 = jj_consume_token(SERIALIZER);
+ } else if (jj_2_5(100)) {
+ jj_consume_token(18);
+ fs = RecordSchema(null);
+ jj_consume_token(19);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ }
+ String compressor, serializer;
+ if (t1 == null)
+ compressor = "lzo2";
+ else {
+ compressor = t1.image;
+ }
+ if (t2 == null)
+ serializer = "pig";
+ else {
+ serializer = t2.image;
+ }
+ if (fs == null)
+ {
+ if (mDefaultCGIndex != -1)
+ {
+ String msg = "Default Storage Information defined more than once";
+ {if (true) throw new ParseException(msg);}
+ }
+ mDefaultCGIndex = mCGCount;
+ mCompressor = compressor;
+ mSerializer = serializer;
+ } else
+ cs = new CGSchema(fs, false, serializer, compressor);
+ {if (true) return cs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema ColumnSchema(int colIndex) throws ParseException, ParseException {
+ Token t1;
+ String name = "";
+ Schema.ColumnSchema fs = null;
+ if (jj_2_6(2147483647)) {
+ fs = SchemaRecord(mSchema, name, colIndex);
+ } else if (jj_2_7(2147483647)) {
+ fs = SchemaMap(mSchema, name, colIndex);
+ } else if (jj_2_8(2147483647)) {
+ fs = AtomSchema(mSchema, name, colIndex);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema AtomSchema(Schema schema, String name, int colIndex) throws ParseException, ParseException {
+ Token t1 = null;
+ Schema.ColumnSchema fs;
+ t1 = jj_consume_token(IDENTIFIER);
+ Schema.ColumnSchema fs0 = schema.getColumn(t1.image);
+ name += t1.image;
+ if (fs0 == null)
+ {
+ String msg = "Column "+name+" not defined in schema";
+ {if (true) throw new ParseException(msg);}
+ }
+ /* create a new field schema using concatenated name */
+ if (!partition.getPartitionInfo().setCGIndex(fs0, mCGCount, colIndex, name))
+ {
+ {if (true) throw new ParseException("Column "+name+" specified more than once!");}
+ }
+ fs = new Schema.ColumnSchema(name, fs0.schema, fs0.type);
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema SchemaMap(Schema schema, String name, int colIndex) throws ParseException, ParseException {
+ Token t1 = null;
+ Schema.ColumnSchema fs;
+ t1 = jj_consume_token(IDENTIFIER);
+ jj_consume_token(20);
+ fs = AnonymousMapSchema(schema.getColumn(t1.image), t1.image, 0, colIndex);
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema SchemaRecord(Schema schema, String name, int colIndex) throws ParseException, ParseException {
+ Token t1 = null;
+ Schema.ColumnSchema fs;
+ t1 = jj_consume_token(IDENTIFIER);
+ jj_consume_token(21);
+ fs = AnonymousRecordSchema(schema.getColumn(t1.image), t1.image, 0, colIndex);
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema AnonymousColumnSchema(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException, ParseException {
+ Token t1;
+ Schema.ColumnSchema fs = null;
+ if (jj_2_9(2147483647)) {
+ fs = AnonymousSchemaRecord(schema, name, cl, colIndex);
+ } else if (jj_2_10(2147483647)) {
+ fs = AnonymousSchemaMap(schema, name, cl, colIndex);
+ } else {
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema AnonymousSchemaRecord(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException, ParseException {
+ Schema.ColumnSchema fs;
+ jj_consume_token(21);
+ fs = AnonymousRecordSchema(schema, name, cl, colIndex);
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema AnonymousSchemaMap(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException, ParseException {
+ Schema.ColumnSchema fs;
+ jj_consume_token(20);
+ fs = AnonymousMapSchema(schema, name, cl, colIndex);
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema RecordSchema(Schema list) throws ParseException, ParseException {
+ if (list == null)
+ list = new Schema();
+ Schema.ColumnSchema fs = null;
+ int colIndex = 0;
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case IDENTIFIER:
+ fs = ColumnSchema(colIndex);
+ list.add(fs);
+ label_2:
+ while (true) {
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case 22:
+ ;
+ break;
+ default:
+ jj_la1[5] = jj_gen;
+ break label_2;
+ }
+ jj_consume_token(22);
+ fs = ColumnSchema(++colIndex);
+ list.add(fs);
+ }
+ break;
+ default:
+ jj_la1[6] = jj_gen;
+
+ list = null;
+ }
+ {if (true) return list;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema AnonymousRecordSchema(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException, ParseException {
+ Schema.ColumnSchema fs = null, fs0 = null;
+ Token t;
+ if (schema == null)
+ {
+ String msg = "no matching column: " + name;
+ throw new ParseException(msg);
+ } else if (schema.type != ColumnType.RECORD && schema.type != ColumnType.COLLECTION) {
+ String msg = "Column " + name + " is not a record or a collection of records";
+ throw new ParseException(msg);
+ } else if (schema.type == ColumnType.COLLECTION) {
+ String msg = "Split of COLLECTION Column, " + name + ", is not supported in this release";
+ throw new ParseException(msg);
+ }
+ if (schema.type == ColumnType.COLLECTION)
+ {
+ cl += 1;
+ if (schema.schema.getNumColumns() == 1 && schema.schema.getColumn(0).type == ColumnType.RECORD && (schema.schema.getColumn(0).name == null || schema.schema.getColumn(0).name.isEmpty()))
+ // an anonymous record inside a collection: go one level lower
+ {
+ schema = schema.schema.getColumn(0);
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.RECORD, name, null, false);
+ }
+ }
+ if (jj_2_11(2)) {
+ t = jj_consume_token(IDENTIFIER);
+ fs = AnonymousSchemaRecord(schema.schema.getColumn(t.image), name + "." + t.image, cl, colIndex);
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.RECORD, name, t.image, true);
+ {if (true) return fs;}
+ } else if (jj_2_12(2)) {
+ t = jj_consume_token(IDENTIFIER);
+ fs = AnonymousSchemaMap(schema.schema.getColumn(t.image), name + "." + t.image, cl, colIndex);
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.MAP, name, t.image, true);
+ {if (true) return fs;}
+ } else {
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case IDENTIFIER:
+ t = jj_consume_token(IDENTIFIER);
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.NONE, name, t.image, false);
+ name = name + "." + t.image;
+ fs0 = schema.schema.getColumn(t.image);
+ if (fs0 == null)
+ {
+ String msg = "no matching column: " + name;
+ {if (true) throw new ParseException(msg);}
+ }
+ if (!partition.setCGIndex(fs0, mCGCount, colIndex, name))
+ {if (true) throw new ParseException("Column "+name+" specified more than once!");}
+ fs = new Schema.ColumnSchema(name, fs0.schema, fs0.type);
+ break;
+ default:
+ jj_la1[7] = jj_gen;
+ jj_consume_token(-1);
+ throw new ParseException();
+ }
+ }
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public Schema.ColumnSchema AnonymousMapSchema(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException, ParseException {
+ Schema.ColumnSchema fs = null;
+ HashSet<String> keys;
+ if (schema == null)
+ {
+ String msg = "no matching column: " + name;
+ throw new ParseException(msg);
+ } else if (schema.type != ColumnType.MAP) {
+ String msg = "Column " + name + " is not a map";
+ throw new ParseException(msg);
+ }
+ partition.setSplit(schema, Partition.SplitType.MAP, Partition.SplitType.RECORD, name, null, false);
+ jj_consume_token(23);
+ keys = hashKeys();
+ jj_consume_token(24);
+ if(!partition.getPartitionInfo().setKeyCGIndex(schema, mCGCount, colIndex, name, keys))
+ {if (true) throw new ParseException("Column "+name+" specified more than once!");}
+ fs = new Schema.ColumnSchema(name, schema.schema, schema.type);
+ {if (true) return fs;}
+ throw new Error("Missing return statement in function");
+ }
+
+ final public HashSet<String> hashKeys() throws ParseException {
+ Token t;
+ HashSet<String> result = new HashSet<String>();
+ t = jj_consume_token(IDENTIFIER);
+ result.add(t.image);
+ label_3:
+ while (true) {
+ switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+ case 25:
+ ;
+ break;
+ default:
+ jj_la1[8] = jj_gen;
+ break label_3;
+ }
+ jj_consume_token(25);
+ t = jj_consume_token(IDENTIFIER);
+ result.add(t.image);
+ }
+ {if (true) return result;}
+ throw new Error("Missing return statement in function");
+ }
+
+ private boolean jj_2_1(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_1(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(0, xla); }
+ }
+
+ private boolean jj_2_2(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_2(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(1, xla); }
+ }
+
+ private boolean jj_2_3(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_3(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(2, xla); }
+ }
+
+ private boolean jj_2_4(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_4(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(3, xla); }
+ }
+
+ private boolean jj_2_5(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_5(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(4, xla); }
+ }
+
+ private boolean jj_2_6(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_6(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(5, xla); }
+ }
+
+ private boolean jj_2_7(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_7(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(6, xla); }
+ }
+
+ private boolean jj_2_8(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_8(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(7, xla); }
+ }
+
+ private boolean jj_2_9(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_9(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(8, xla); }
+ }
+
+ private boolean jj_2_10(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_10(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(9, xla); }
+ }
+
+ private boolean jj_2_11(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_11(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(10, xla); }
+ }
+
+ private boolean jj_2_12(int xla) {
+ jj_la = xla; jj_lastpos = jj_scanpos = token;
+ try { return !jj_3_12(); }
+ catch(LookaheadSuccess ls) { return true; }
+ finally { jj_save(11, xla); }
+ }
+
+ private boolean jj_3R_17() {
+ if (jj_scan_token(IDENTIFIER)) return true;
+ Token xsp;
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3R_21()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3R_16() {
+ if (jj_scan_token(IDENTIFIER)) return true;
+ return false;
+ }
+
+ private boolean jj_3_12() {
+ if (jj_scan_token(IDENTIFIER)) return true;
+ if (jj_3R_9()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_7() {
+ if (jj_scan_token(IDENTIFIER)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_10() {
+ if (jj_3R_14()) return true;
+ Token xsp;
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3R_15()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
+ private boolean jj_3_8() {
+ if (jj_3R_7()) return true;
+ return false;
+ }
+
+ private boolean jj_3_7() {
+ if (jj_3R_6()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_4() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3R_10()) {
+ jj_scanpos = xsp;
+ if (jj_3R_11()) return true;
+ }
+ return false;
+ }
+
+ private boolean jj_3_6() {
+ if (jj_3R_5()) return true;
+ return false;
+ }
+
+ private boolean jj_3_11() {
+ if (jj_scan_token(IDENTIFIER)) return true;
+ if (jj_3R_8()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_5() {
+ if (jj_scan_token(IDENTIFIER)) return true;
+ if (jj_scan_token(21)) return true;
+ if (jj_3R_12()) return true;
+ return false;
+ }
+
+ private boolean jj_3_5() {
+ if (jj_scan_token(18)) return true;
+ if (jj_3R_4()) return true;
+ if (jj_scan_token(19)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_20() {
+ if (jj_3R_7()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_19() {
+ if (jj_3R_6()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_12() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_11()) {
+ jj_scanpos = xsp;
+ if (jj_3_12()) {
+ jj_scanpos = xsp;
+ if (jj_3R_16()) return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_4() {
+ if (jj_scan_token(18)) return true;
+ if (jj_3R_4()) return true;
+ if (jj_scan_token(19)) return true;
+ if (jj_scan_token(SERIALIZE)) return true;
+ if (jj_scan_token(SERIALIZER)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_18() {
+ if (jj_3R_5()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_13() {
+ if (jj_scan_token(23)) return true;
+ if (jj_3R_17()) return true;
+ if (jj_scan_token(24)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_14() {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3R_18()) {
+ jj_scanpos = xsp;
+ if (jj_3R_19()) {
+ jj_scanpos = xsp;
+ if (jj_3R_20()) return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean jj_3_3() {
+ if (jj_scan_token(18)) return true;
+ if (jj_3R_4()) return true;
+ if (jj_scan_token(19)) return true;
+ if (jj_scan_token(COMPRESS)) return true;
+ if (jj_scan_token(COMPRESSOR)) return true;
+ return false;
+ }
+
+ private boolean jj_3_2() {
+ if (jj_scan_token(18)) return true;
+ if (jj_3R_4()) return true;
+ if (jj_scan_token(19)) return true;
+ if (jj_scan_token(SERIALIZE)) return true;
+ if (jj_scan_token(SERIALIZER)) return true;
+ if (jj_scan_token(COMPRESS)) return true;
+ if (jj_scan_token(COMPRESSOR)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_9() {
+ if (jj_scan_token(20)) return true;
+ if (jj_3R_13()) return true;
+ return false;
+ }
+
+ private boolean jj_3_1() {
+ if (jj_scan_token(18)) return true;
+ if (jj_3R_4()) return true;
+ if (jj_scan_token(19)) return true;
+ if (jj_scan_token(COMPRESS)) return true;
+ if (jj_scan_token(COMPRESSOR)) return true;
+ if (jj_scan_token(SERIALIZE)) return true;
+ if (jj_scan_token(SERIALIZER)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_6() {
+ if (jj_scan_token(IDENTIFIER)) return true;
+ if (jj_scan_token(20)) return true;
+ if (jj_3R_13()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_8() {
+ if (jj_scan_token(21)) return true;
+ if (jj_3R_12()) return true;
+ return false;
+ }
+
+ private boolean jj_3_10() {
+ if (jj_3R_9()) return true;
+ return false;
+ }
+
+ private boolean jj_3_9() {
+ if (jj_3R_8()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_21() {
+ if (jj_scan_token(25)) return true;
+ if (jj_scan_token(IDENTIFIER)) return true;
+ return false;
+ }
+
+ private boolean jj_3R_15() {
+ if (jj_scan_token(22)) return true;
+ if (jj_3R_14()) return true;
+ return false;
+ }
+
+ private boolean jj_3R_11() {
+ return false;
+ }
+
+ /** Generated Token Manager. */
+ public TableStorageParserTokenManager token_source;
+ SimpleCharStream jj_input_stream;
+ /** Current token. */
+ public Token token;
+ /** Next token. */
+ public Token jj_nt;
+ private int jj_ntk;
+ private Token jj_scanpos, jj_lastpos;
+ private int jj_la;
+ private int jj_gen;
+ final private int[] jj_la1 = new int[9];
+ static private int[] jj_la1_0;
+ static {
+ jj_la1_init_0();
+ }
+ private static void jj_la1_init_0() {
+ jj_la1_0 = new int[] {0x20000,0x40c01,0x800,0x400,0xc00,0x400000,0x10000,0x10000,0x2000000,};
+ }
+ final private JJCalls[] jj_2_rtns = new JJCalls[12];
+ private boolean jj_rescan = false;
+ private int jj_gc = 0;
+
+ /** Constructor with InputStream. */
+ public TableStorageParser(java.io.InputStream stream) {
+ this(stream, null);
+ }
+ /** Constructor with InputStream and supplied encoding */
+ public TableStorageParser(java.io.InputStream stream, String encoding) {
+ try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
+ token_source = new TableStorageParserTokenManager(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 9; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream stream) {
+ ReInit(stream, null);
+ }
+ /** Reinitialise. */
+ public void ReInit(java.io.InputStream stream, String encoding) {
+ try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
+ token_source.ReInit(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 9; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Constructor. */
+ public TableStorageParser(java.io.Reader stream) {
+ jj_input_stream = new SimpleCharStream(stream, 1, 1);
+ token_source = new TableStorageParserTokenManager(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 9; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(java.io.Reader stream) {
+ jj_input_stream.ReInit(stream, 1, 1);
+ token_source.ReInit(jj_input_stream);
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 9; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Constructor with generated Token Manager. */
+ public TableStorageParser(TableStorageParserTokenManager tm) {
+ token_source = tm;
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 9; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ /** Reinitialise. */
+ public void ReInit(TableStorageParserTokenManager tm) {
+ token_source = tm;
+ token = new Token();
+ jj_ntk = -1;
+ jj_gen = 0;
+ for (int i = 0; i < 9; i++) jj_la1[i] = -1;
+ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+ }
+
+ private Token jj_consume_token(int kind) throws ParseException {
+ Token oldToken;
+ if ((oldToken = token).next != null) token = token.next;
+ else token = token.next = token_source.getNextToken();
+ jj_ntk = -1;
+ if (token.kind == kind) {
+ jj_gen++;
+ if (++jj_gc > 100) {
+ jj_gc = 0;
+ for (int i = 0; i < jj_2_rtns.length; i++) {
+ JJCalls c = jj_2_rtns[i];
+ while (c != null) {
+ if (c.gen < jj_gen) c.first = null;
+ c = c.next;
+ }
+ }
+ }
+ return token;
+ }
+ token = oldToken;
+ jj_kind = kind;
+ throw generateParseException();
+ }
+
+ static private final class LookaheadSuccess extends java.lang.Error { }
+ final private LookaheadSuccess jj_ls = new LookaheadSuccess();
+ private boolean jj_scan_token(int kind) {
+ if (jj_scanpos == jj_lastpos) {
+ jj_la--;
+ if (jj_scanpos.next == null) {
+ jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken();
+ } else {
+ jj_lastpos = jj_scanpos = jj_scanpos.next;
+ }
+ } else {
+ jj_scanpos = jj_scanpos.next;
+ }
+ if (jj_rescan) {
+ int i = 0; Token tok = token;
+ while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; }
+ if (tok != null) jj_add_error_token(kind, i);
+ }
+ if (jj_scanpos.kind != kind) return true;
+ if (jj_la == 0 && jj_scanpos == jj_lastpos) throw jj_ls;
+ return false;
+ }
+
+
+/** Get the next Token. */
+ final public Token getNextToken() {
+ if (token.next != null) token = token.next;
+ else token = token.next = token_source.getNextToken();
+ jj_ntk = -1;
+ jj_gen++;
+ return token;
+ }
+
+/** Get the specific Token. */
+ final public Token getToken(int index) {
+ Token t = token;
+ for (int i = 0; i < index; i++) {
+ if (t.next != null) t = t.next;
+ else t = t.next = token_source.getNextToken();
+ }
+ return t;
+ }
+
+ private int jj_ntk() {
+ if ((jj_nt=token.next) == null)
+ return (jj_ntk = (token.next=token_source.getNextToken()).kind);
+ else
+ return (jj_ntk = jj_nt.kind);
+ }
+
+ private java.util.List jj_expentries = new java.util.ArrayList();
+ private int[] jj_expentry;
+ private int jj_kind = -1;
+ private int[] jj_lasttokens = new int[100];
+ private int jj_endpos;
+
+ private void jj_add_error_token(int kind, int pos) {
+ if (pos >= 100) return;
+ if (pos == jj_endpos + 1) {
+ jj_lasttokens[jj_endpos++] = kind;
+ } else if (jj_endpos != 0) {
+ jj_expentry = new int[jj_endpos];
+ for (int i = 0; i < jj_endpos; i++) {
+ jj_expentry[i] = jj_lasttokens[i];
+ }
+ jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) {
+ int[] oldentry = (int[])(it.next());
+ if (oldentry.length == jj_expentry.length) {
+ for (int i = 0; i < jj_expentry.length; i++) {
+ if (oldentry[i] != jj_expentry[i]) {
+ continue jj_entries_loop;
+ }
+ }
+ jj_expentries.add(jj_expentry);
+ break jj_entries_loop;
+ }
+ }
+ if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind;
+ }
+ }
+
+ /** Generate ParseException. */
+ public ParseException generateParseException() {
+ jj_expentries.clear();
+ boolean[] la1tokens = new boolean[26];
+ if (jj_kind >= 0) {
+ la1tokens[jj_kind] = true;
+ jj_kind = -1;
+ }
+ for (int i = 0; i < 9; i++) {
+ if (jj_la1[i] == jj_gen) {
+ for (int j = 0; j < 32; j++) {
+ if ((jj_la1_0[i] & (1<<j)) != 0) {
+ la1tokens[j] = true;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < 26; i++) {
+ if (la1tokens[i]) {
+ jj_expentry = new int[1];
+ jj_expentry[0] = i;
+ jj_expentries.add(jj_expentry);
+ }
+ }
+ jj_endpos = 0;
+ jj_rescan_token();
+ jj_add_error_token(0, 0);
+ int[][] exptokseq = new int[jj_expentries.size()][];
+ for (int i = 0; i < jj_expentries.size(); i++) {
+ exptokseq[i] = (int[])jj_expentries.get(i);
+ }
+ return new ParseException(token, exptokseq, tokenImage);
+ }
+
+ /** Enable tracing. */
+ final public void enable_tracing() {
+ }
+
+ /** Disable tracing. */
+ final public void disable_tracing() {
+ }
+
+ private void jj_rescan_token() {
+ jj_rescan = true;
+ for (int i = 0; i < 12; i++) {
+ try {
+ JJCalls p = jj_2_rtns[i];
+ do {
+ if (p.gen > jj_gen) {
+ jj_la = p.arg; jj_lastpos = jj_scanpos = p.first;
+ switch (i) {
+ case 0: jj_3_1(); break;
+ case 1: jj_3_2(); break;
+ case 2: jj_3_3(); break;
+ case 3: jj_3_4(); break;
+ case 4: jj_3_5(); break;
+ case 5: jj_3_6(); break;
+ case 6: jj_3_7(); break;
+ case 7: jj_3_8(); break;
+ case 8: jj_3_9(); break;
+ case 9: jj_3_10(); break;
+ case 10: jj_3_11(); break;
+ case 11: jj_3_12(); break;
+ }
+ }
+ p = p.next;
+ } while (p != null);
+ } catch(LookaheadSuccess ls) { }
+ }
+ jj_rescan = false;
+ }
+
+ private void jj_save(int index, int xla) {
+ JJCalls p = jj_2_rtns[index];
+ while (p.gen > jj_gen) {
+ if (p.next == null) { p = p.next = new JJCalls(); break; }
+ p = p.next;
+ }
+ p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla;
+ }
+
+ static final class JJCalls {
+ int gen;
+ Token first;
+ int arg;
+ JJCalls next;
+ }
+
+}
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParser.jjt Tue Aug 11 22:27:44 2009
@@ -0,0 +1,442 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+options {
+ STATIC = false ;
+ IGNORE_CASE = true;
+}
+
+PARSER_BEGIN(TableStorageParser)
+package org.apache.pig.table.types;
+import java.io.*;
+import java.util.*;
+
+public class TableStorageParser {
+ public static void main( String[] args )
+ throws ParseException, TokenMgrError, java.io.FileNotFoundException, java.io.IOException, java.lang.ClassNotFoundException {
+ /*
+ FileInputStream input = new FileInputStream("/tmp/schema");
+ ObjectInputStream objin = new ObjectInputStream(input);
+ Schema schema = (Schema) objin.readObject();
+ objin.close();
+ input = new FileInputStream("store");
+ System.out.println("Table Schema:");
+ System.out.println("--------------------------");
+ System.out.println(schema.toString());
+ Partition partition = new Partition(
+ TableStorageParser parser = new TableStorageParser(schema.toString(), schema) ;
+ ArrayList<CGSchema> schemas = parser.StorageSchema();
+ System.out.println("Column Groups:");
+ System.out.println("--------------------------");
+ for (int i = 0; i < schemas.size(); i++)
+ {
+ System.out.println(schemas.get(i).toString());
+ System.out.println("==========================");
+ }
+ System.out.println("Table Schema with Storage Hints");
+ System.out.println(schema.toString());
+ System.out.println("==========================");
+ System.out.println("Column Map:");
+ System.out.println("--------------------------");
+ Map<String, Schema.ColumnMappingEntry> colmap = schema.getColMap();
+ if (!colmap.isEmpty())
+ {
+ Set<Map.Entry<String, Schema.ColumnMappingEntry>> entrySet = colmap.entrySet();
+ Map.Entry<String, Schema.ColumnMappingEntry>[] array = entrySet.toArray(new Map.Entry[entrySet.size()]);
+ for (int j = 0; j < colmap.size(); j++)
+ {
+ System.out.println("[Column = " + array[j].getKey() +" CG= "+ array[j].getValue().getCGIndex()+"."+array[j].getValue().getColumnIndex()+"]");
+ }
+ }
+ Schema.ColumnMappingEntry cm;
+ if (args.length > 0)
+ {
+ System.out.println("==========================");
+ System.out.println("Column CG Index:");
+ System.out.println("--------------------------");
+ if ((cm = schema.getColMapping(args[0])) != null)
+ System.out.println(args[0]+":CG= "+ cm.getCGIndex()+"."+cm.getColumnIndex());
+ else
+ System.out.println(args[0]+":Column not found !");
+ }
+ Schema.ColumnSchema fs;
+ if (args.length > 0)
+ {
+ System.out.println("==========================");
+ System.out.println("Column Schema:");
+ System.out.println("--------------------------");
+ StringBuilder sb = new StringBuilder();
+ if ((fs = schema.getColumnSchema(args[0])) != null)
+ {
+ // print out a particular column's CG index
+ // System.out.println(args[0]+":Schema = "+ fs.name +":"+ColumnType.genTypeToNameMap().get(fs.type));
+ Schema.stringifySchema(sb, fs.schema, fs.type, false, false);
+ if (sb.toString().isEmpty())
+ System.out.println(args[0]+":Schema = "+ ColumnType.findTypeName(fs.type));
+ else
+ System.out.println(args[0]+":Schema = "+ ColumnType.findTypeName(fs.type)+":"+sb.toString());
+ } else
+ System.out.println(args[0]+":Column not found !");
+ }
+ */
+ }
+ public TableStorageParser(java.io.Reader in, Partition partition, Schema schema) { this(in); mSchema = schema; this.partition = partition;}
+ private Schema mSchema;
+ private int mDefaultCGIndex = -1;
+ private String mCompressor = "lzo2", mSerializer = "pig";
+ private int mCGCount = 0;
+ private Partition partition;
+}
+PARSER_END(TableStorageParser)
+
+// Skip all the new lines, tabs and spaces
+SKIP : { " " | "\r" | "\t" | "\n" }
+
+// Skip comments(single line and multiline)
+SKIP : {
+ <"--"(~["\r","\n"])*>
+| <"/*" (~["*"])* "*" ("*" | (~["*","/"] (~["*"])* "*"))* "/">
+}
+
+TOKEN : { <COMPRESSOR : "lzo2" | "gzip"> }
+TOKEN : { <SERIALIZER : ("pig" | "avro")> }
+
+TOKEN : { <ORDER : "order by"> }
+TOKEN : { <COMPRESS : "compress by"> }
+TOKEN : { <SERIALIZE : "serialize by"> }
+
+TOKEN:
+{
+ <#LETTER : ["a"-"z", "A"-"Z"] >
+| <#DIGIT : ["0"-"9"] >
+| <#SPECIALCHAR : ["_"] >
+| <#FSSPECIALCHAR: ["-", ":", "/"]>
+| <IDENTIFIER: ( <LETTER> )+ ( <DIGIT> | <LETTER> | <SPECIALCHAR> )* >
+}
+
+ArrayList<CGSchema> StorageSchema() throws ParseException :
+{
+ ArrayList<CGSchema> s = new ArrayList();
+ CGSchema fs;
+ CGSchema defaultSchema;
+}
+{
+ try {
+ <EOF>
+ {
+ defaultSchema = partition.generateDefaultCGSchema(mCompressor, mSerializer, 0);
+ if (defaultSchema != null)
+ s.add(defaultSchema);
+ return s;
+ }
+ |
+ fs = FieldSchema() {mCGCount++; if (fs != null) s.add(fs);}
+ (";" fs = FieldSchema() {mCGCount++; if (fs != null) s.add(fs);})* <EOF>
+ {
+ defaultSchema = partition.generateDefaultCGSchema(mCompressor, mSerializer, mDefaultCGIndex == -1 ? mDefaultCGIndex = mCGCount++ : mDefaultCGIndex);
+ if (defaultSchema != null)
+ s.add(defaultSchema);
+ return s;
+ }
+ } catch (TokenMgrError e) {
+ throw new ParseException(e.getMessage());
+ }
+}
+
+CGSchema FieldSchema() :
+{
+ Token t1 = null, t2 = null;
+ Schema fs = null;
+ CGSchema cs = null;
+}
+{
+ (
+ <COMPRESS> t1 = <IDENTIFIER> [ <SERIALIZE> t2 = <IDENTIFIER> ]
+|
+ <SERIALIZE> t2 = <IDENTIFIER> [ <COMPRESS> t1 = <IDENTIFIER> ]
+|
+// one column group schema's length cannot exceed 1000;
+// how good is value 1000?
+ LOOKAHEAD(100) "[" fs = RecordSchema(null) "]" <COMPRESS> t1 = <COMPRESSOR> <SERIALIZE> t2 = <SERIALIZER>
+|
+ LOOKAHEAD(100) "[" fs = RecordSchema(null) "]" <SERIALIZE> t2 = <SERIALIZER> <COMPRESS> t1 = <COMPRESSOR>
+|
+ LOOKAHEAD(100) "[" fs = RecordSchema(null) "]" <COMPRESS> t1 = <COMPRESSOR>
+|
+ LOOKAHEAD(100) "[" fs = RecordSchema(null) "]" <SERIALIZE> t2 = <SERIALIZER>
+|
+ LOOKAHEAD(100) "[" fs = RecordSchema(null) "]"
+ )
+ {
+ String compressor, serializer;
+ if (t1 == null)
+ compressor = "lzo2";
+ else {
+ compressor = t1.image;
+ }
+ if (t2 == null)
+ serializer = "pig";
+ else {
+ serializer = t2.image;
+ }
+ if (fs == null)
+ {
+ if (mDefaultCGIndex != -1)
+ {
+ String msg = "Default Storage Information defined more than once";
+ throw new ParseException(msg);
+ }
+ mDefaultCGIndex = mCGCount;
+ mCompressor = compressor;
+ mSerializer = serializer;
+ } else
+ cs = new CGSchema(fs, false, serializer, compressor);
+ return cs;
+ }
+}
+
+Schema.ColumnSchema ColumnSchema(int colIndex) throws ParseException:
+{
+ Token t1;
+ String name = "";
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ LOOKAHEAD(SchemaRecord()) fs = SchemaRecord(mSchema, name, colIndex)
+| LOOKAHEAD(SchemaMap()) fs = SchemaMap(mSchema, name, colIndex)
+| LOOKAHEAD(AtomSchema()) fs = AtomSchema(mSchema, name, colIndex)
+ )
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AtomSchema(Schema schema, String name, int colIndex) throws ParseException :
+{
+ Token t1 = null;
+ Schema.ColumnSchema fs;
+}
+{
+ ( ( t1 = <IDENTIFIER> )
+ {
+ Schema.ColumnSchema fs0 = schema.getColumn(t1.image);
+ name += t1.image;
+ if (fs0 == null)
+ {
+ String msg = "Column "+name+" not defined in schema";
+ throw new ParseException(msg);
+ }
+ /* create a new field schema using concatenated name */
+ if (!partition.getPartitionInfo().setCGIndex(fs0, mCGCount, colIndex, name))
+ {
+ throw new ParseException("Column "+name+" specified more than once!");
+ }
+ fs = new Schema.ColumnSchema(name, fs0.schema, fs0.type);
+ return fs;
+ }
+ )
+}
+
+Schema.ColumnSchema SchemaMap(Schema schema, String name, int colIndex) throws ParseException :
+{
+ Token t1 = null;
+ Schema.ColumnSchema fs;
+}
+{
+ t1 = <IDENTIFIER> "#" fs = AnonymousMapSchema(schema.getColumn(t1.image), t1.image, 0, colIndex)
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema SchemaRecord(Schema schema, String name, int colIndex) throws ParseException :
+{
+ Token t1 = null;
+ Schema.ColumnSchema fs;
+}
+{
+ t1 = <IDENTIFIER> "." fs = AnonymousRecordSchema(schema.getColumn(t1.image), t1.image, 0, colIndex)
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AnonymousColumnSchema(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException :
+{
+ Token t1;
+ Schema.ColumnSchema fs = null;
+}
+{
+ (
+ LOOKAHEAD(AnonymousSchemaRecord()) fs = AnonymousSchemaRecord(schema, name, cl, colIndex)
+| LOOKAHEAD(AnonymousSchemaMap()) fs = AnonymousSchemaMap(schema, name, cl, colIndex)
+ )
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AnonymousSchemaRecord(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException :
+{
+ Schema.ColumnSchema fs;
+}
+{
+ "." fs = AnonymousRecordSchema(schema, name, cl, colIndex)
+ {
+ return fs;
+ }
+}
+
+Schema.ColumnSchema AnonymousSchemaMap(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException:
+{
+ Schema.ColumnSchema fs;
+}
+{
+ "#" fs = AnonymousMapSchema(schema, name, cl, colIndex)
+ {
+ return fs;
+ }
+}
+
+
+Schema RecordSchema(Schema list) throws ParseException :
+{
+ if (list == null)
+ list = new Schema();
+ Schema.ColumnSchema fs = null;
+ int colIndex = 0;
+}
+{
+ (
+ (
+ fs = ColumnSchema(colIndex) {list.add(fs);}
+ ( "," fs = ColumnSchema(++colIndex) {list.add(fs);})*
+ )
+| {} {list = null;}
+ )
+ { return list; }
+}
+
+Schema.ColumnSchema AnonymousRecordSchema(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException :
+{
+ Schema.ColumnSchema fs = null, fs0 = null;
+ Token t;
+ if (schema == null)
+ {
+ String msg = "no matching column: " + name;
+ throw new ParseException(msg);
+ } else if (schema.type != ColumnType.RECORD && schema.type != ColumnType.COLLECTION) {
+ String msg = "Column " + name + " is not a record or a collection of records";
+ throw new ParseException(msg);
+ } else if (schema.type == ColumnType.COLLECTION) {
+ String msg = "Split of COLLECTION Column, " + name + ", is not supported in this release";
+ throw new ParseException(msg);
+ }
+ if (schema.type == ColumnType.COLLECTION)
+ {
+ cl += 1;
+ if (schema.schema.getNumColumns() == 1 && schema.schema.getColumn(0).type == ColumnType.RECORD && (schema.schema.getColumn(0).name == null || schema.schema.getColumn(0).name.isEmpty()))
+ // an anonymous record inside a collection: go one level lower
+ {
+ schema = schema.schema.getColumn(0);
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.RECORD, name, null, false);
+ }
+ }
+}
+{
+ (
+ LOOKAHEAD(2) t = <IDENTIFIER> fs = AnonymousSchemaRecord(schema.schema.getColumn(t.image), name + "." + t.image, cl, colIndex)
+ {
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.RECORD, name, t.image, true);
+ return fs;
+ }
+|
+ LOOKAHEAD(2) t = <IDENTIFIER> fs = AnonymousSchemaMap(schema.schema.getColumn(t.image), name + "." + t.image, cl, colIndex)
+ {
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.MAP, name, t.image, true);
+ return fs;
+ }
+|
+ t= <IDENTIFIER>
+ {
+ partition.setSplit(schema, Partition.SplitType.RECORD, Partition.SplitType.NONE, name, t.image, false);
+ name = name + "." + t.image;
+ fs0 = schema.schema.getColumn(t.image);
+ if (fs0 == null)
+ {
+ String msg = "no matching column: " + name;
+ throw new ParseException(msg);
+ }
+ if (!partition.setCGIndex(fs0, mCGCount, colIndex, name))
+ throw new ParseException("Column "+name+" specified more than once!");
+ fs = new Schema.ColumnSchema(name, fs0.schema, fs0.type);
+ }
+ )
+ { return fs; }
+}
+
+Schema.ColumnSchema AnonymousMapSchema(Schema.ColumnSchema schema, String name, int cl, int colIndex) throws ParseException :
+{
+ Schema.ColumnSchema fs = null;
+ HashSet<String> keys;
+ if (schema == null)
+ {
+ String msg = "no matching column: " + name;
+ throw new ParseException(msg);
+ } else if (schema.type != ColumnType.MAP) {
+ String msg = "Column " + name + " is not a map";
+ throw new ParseException(msg);
+ }
+ partition.setSplit(schema, Partition.SplitType.MAP, Partition.SplitType.RECORD, name, null, false);
+}
+{
+ (
+ /* will not support nested splits inside a MAP key. Might be enabled later along with other
+ necessary changes
+ */
+ /*
+ LOOKAHEAD(2)
+ t = <IDENTIFIER> fs = AnonymousColumnSchema(schema.schema.getColumn(0), name = name + "#" + t.image, cl, colIndex)
+ {
+ return fs;
+ }
+|
+*/
+ "{" keys = hashKeys() "}"
+ {
+ if(!partition.getPartitionInfo().setKeyCGIndex(schema, mCGCount, colIndex, name, keys))
+ throw new ParseException("Column "+name+" specified more than once!");
+ fs = new Schema.ColumnSchema(name, schema.schema, schema.type);
+ }
+ )
+ { return fs; }
+}
+
+HashSet<String> hashKeys() :
+{
+ Token t;
+ HashSet<String> result = new HashSet<String>();
+}
+{
+ t = <IDENTIFIER> { result.add(t.image); }
+ ("|" t = <IDENTIFIER> { result.add(t.image); })*
+ {
+ return result;
+ }
+}
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserConstants.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserConstants.java?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserConstants.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserConstants.java Tue Aug 11 22:27:44 2009
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. TableStorageParserConstants.java */
+package org.apache.hadoop.zebra.types;
+
+
+/**
+ * Token literal values and constants.
+ * Generated by org.javacc.parser.OtherFilesGen#start()
+ */
+public interface TableStorageParserConstants {
+
+ /** End of File. */
+ int EOF = 0;
+ /** RegularExpression Id. */
+ int COMPRESSOR = 7;
+ /** RegularExpression Id. */
+ int SERIALIZER = 8;
+ /** RegularExpression Id. */
+ int ORDER = 9;
+ /** RegularExpression Id. */
+ int COMPRESS = 10;
+ /** RegularExpression Id. */
+ int SERIALIZE = 11;
+ /** RegularExpression Id. */
+ int LETTER = 12;
+ /** RegularExpression Id. */
+ int DIGIT = 13;
+ /** RegularExpression Id. */
+ int SPECIALCHAR = 14;
+ /** RegularExpression Id. */
+ int FSSPECIALCHAR = 15;
+ /** RegularExpression Id. */
+ int IDENTIFIER = 16;
+
+ /** Lexical state. */
+ int DEFAULT = 0;
+
+ /** Literal token values. */
+ String[] tokenImage = {
+ "<EOF>",
+ "\" \"",
+ "\"\\r\"",
+ "\"\\t\"",
+ "\"\\n\"",
+ "<token of kind 5>",
+ "<token of kind 6>",
+ "<COMPRESSOR>",
+ "<SERIALIZER>",
+ "\"order by\"",
+ "\"compress by\"",
+ "\"serialize by\"",
+ "<LETTER>",
+ "<DIGIT>",
+ "<SPECIALCHAR>",
+ "<FSSPECIALCHAR>",
+ "<IDENTIFIER>",
+ "\";\"",
+ "\"[\"",
+ "\"]\"",
+ "\"#\"",
+ "\".\"",
+ "\",\"",
+ "\"{\"",
+ "\"}\"",
+ "\"|\"",
+ };
+
+}
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserTokenManager.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserTokenManager.java?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserTokenManager.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/TableStorageParserTokenManager.java Tue Aug 11 22:27:44 2009
@@ -0,0 +1,931 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. TableStorageParserTokenManager.java */
+package org.apache.hadoop.zebra.types;
+import java.io.*;
+import java.util.*;
+
+/** Token Manager. */
+public class TableStorageParserTokenManager implements TableStorageParserConstants
+{
+
+ /** Debug output. */
+ public java.io.PrintStream debugStream = System.out;
+ /** Set debug output. */
+ public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
+private final int jjStopStringLiteralDfa_0(int pos, long active0)
+{
+ switch (pos)
+ {
+ case 0:
+ if ((active0 & 0xe00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ return 27;
+ }
+ return -1;
+ case 1:
+ if ((active0 & 0xe00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 1;
+ return 27;
+ }
+ return -1;
+ case 2:
+ if ((active0 & 0xe00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 2;
+ return 27;
+ }
+ return -1;
+ case 3:
+ if ((active0 & 0xe00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 3;
+ return 27;
+ }
+ return -1;
+ case 4:
+ if ((active0 & 0xe00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 4;
+ return 27;
+ }
+ return -1;
+ case 5:
+ if ((active0 & 0x200L) != 0L)
+ {
+ if (jjmatchedPos < 4)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 4;
+ }
+ return -1;
+ }
+ if ((active0 & 0xc00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 5;
+ return 27;
+ }
+ return -1;
+ case 6:
+ if ((active0 & 0xc00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 6;
+ return 27;
+ }
+ if ((active0 & 0x200L) != 0L)
+ {
+ if (jjmatchedPos < 4)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 4;
+ }
+ return -1;
+ }
+ return -1;
+ case 7:
+ if ((active0 & 0xc00L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 7;
+ return 27;
+ }
+ if ((active0 & 0x200L) != 0L)
+ {
+ if (jjmatchedPos < 4)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 4;
+ }
+ return -1;
+ }
+ return -1;
+ case 8:
+ if ((active0 & 0x400L) != 0L)
+ {
+ if (jjmatchedPos < 7)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 7;
+ }
+ return -1;
+ }
+ if ((active0 & 0x800L) != 0L)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 8;
+ return 27;
+ }
+ return -1;
+ case 9:
+ if ((active0 & 0x800L) != 0L)
+ {
+ if (jjmatchedPos < 8)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 8;
+ }
+ return -1;
+ }
+ if ((active0 & 0x400L) != 0L)
+ {
+ if (jjmatchedPos < 7)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 7;
+ }
+ return -1;
+ }
+ return -1;
+ case 10:
+ if ((active0 & 0x800L) != 0L)
+ {
+ if (jjmatchedPos < 8)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 8;
+ }
+ return -1;
+ }
+ if ((active0 & 0x400L) != 0L)
+ {
+ if (jjmatchedPos < 7)
+ {
+ jjmatchedKind = 16;
+ jjmatchedPos = 7;
+ }
+ return -1;
+ }
+ return -1;
+ default :
+ return -1;
+ }
+}
+private final int jjStartNfa_0(int pos, long active0)
+{
+ return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1);
+}
+private int jjStopAtPos(int pos, int kind)
+{
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ return pos + 1;
+}
+private int jjMoveStringLiteralDfa0_0()
+{
+ switch(curChar)
+ {
+ case 35:
+ return jjStopAtPos(0, 20);
+ case 44:
+ return jjStopAtPos(0, 22);
+ case 46:
+ return jjStopAtPos(0, 21);
+ case 59:
+ return jjStopAtPos(0, 17);
+ case 91:
+ return jjStopAtPos(0, 18);
+ case 93:
+ return jjStopAtPos(0, 19);
+ case 67:
+ case 99:
+ return jjMoveStringLiteralDfa1_0(0x400L);
+ case 79:
+ case 111:
+ return jjMoveStringLiteralDfa1_0(0x200L);
+ case 83:
+ case 115:
+ return jjMoveStringLiteralDfa1_0(0x800L);
+ case 123:
+ return jjStopAtPos(0, 23);
+ case 124:
+ return jjStopAtPos(0, 25);
+ case 125:
+ return jjStopAtPos(0, 24);
+ default :
+ return jjMoveNfa_0(2, 0);
+ }
+}
+private int jjMoveStringLiteralDfa1_0(long active0)
+{
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(0, active0);
+ return 1;
+ }
+ switch(curChar)
+ {
+ case 69:
+ case 101:
+ return jjMoveStringLiteralDfa2_0(active0, 0x800L);
+ case 79:
+ case 111:
+ return jjMoveStringLiteralDfa2_0(active0, 0x400L);
+ case 82:
+ case 114:
+ return jjMoveStringLiteralDfa2_0(active0, 0x200L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(0, active0);
+}
+private int jjMoveStringLiteralDfa2_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(0, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(1, active0);
+ return 2;
+ }
+ switch(curChar)
+ {
+ case 68:
+ case 100:
+ return jjMoveStringLiteralDfa3_0(active0, 0x200L);
+ case 77:
+ case 109:
+ return jjMoveStringLiteralDfa3_0(active0, 0x400L);
+ case 82:
+ case 114:
+ return jjMoveStringLiteralDfa3_0(active0, 0x800L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(1, active0);
+}
+private int jjMoveStringLiteralDfa3_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(1, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(2, active0);
+ return 3;
+ }
+ switch(curChar)
+ {
+ case 69:
+ case 101:
+ return jjMoveStringLiteralDfa4_0(active0, 0x200L);
+ case 73:
+ case 105:
+ return jjMoveStringLiteralDfa4_0(active0, 0x800L);
+ case 80:
+ case 112:
+ return jjMoveStringLiteralDfa4_0(active0, 0x400L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(2, active0);
+}
+private int jjMoveStringLiteralDfa4_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(2, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(3, active0);
+ return 4;
+ }
+ switch(curChar)
+ {
+ case 65:
+ case 97:
+ return jjMoveStringLiteralDfa5_0(active0, 0x800L);
+ case 82:
+ case 114:
+ return jjMoveStringLiteralDfa5_0(active0, 0x600L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(3, active0);
+}
+private int jjMoveStringLiteralDfa5_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(3, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(4, active0);
+ return 5;
+ }
+ switch(curChar)
+ {
+ case 32:
+ return jjMoveStringLiteralDfa6_0(active0, 0x200L);
+ case 69:
+ case 101:
+ return jjMoveStringLiteralDfa6_0(active0, 0x400L);
+ case 76:
+ case 108:
+ return jjMoveStringLiteralDfa6_0(active0, 0x800L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(4, active0);
+}
+private int jjMoveStringLiteralDfa6_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(4, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(5, active0);
+ return 6;
+ }
+ switch(curChar)
+ {
+ case 66:
+ case 98:
+ return jjMoveStringLiteralDfa7_0(active0, 0x200L);
+ case 73:
+ case 105:
+ return jjMoveStringLiteralDfa7_0(active0, 0x800L);
+ case 83:
+ case 115:
+ return jjMoveStringLiteralDfa7_0(active0, 0x400L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(5, active0);
+}
+private int jjMoveStringLiteralDfa7_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(5, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(6, active0);
+ return 7;
+ }
+ switch(curChar)
+ {
+ case 83:
+ case 115:
+ return jjMoveStringLiteralDfa8_0(active0, 0x400L);
+ case 89:
+ case 121:
+ if ((active0 & 0x200L) != 0L)
+ return jjStopAtPos(7, 9);
+ break;
+ case 90:
+ case 122:
+ return jjMoveStringLiteralDfa8_0(active0, 0x800L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(6, active0);
+}
+private int jjMoveStringLiteralDfa8_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(6, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(7, active0);
+ return 8;
+ }
+ switch(curChar)
+ {
+ case 32:
+ return jjMoveStringLiteralDfa9_0(active0, 0x400L);
+ case 69:
+ case 101:
+ return jjMoveStringLiteralDfa9_0(active0, 0x800L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(7, active0);
+}
+private int jjMoveStringLiteralDfa9_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(7, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(8, active0);
+ return 9;
+ }
+ switch(curChar)
+ {
+ case 32:
+ return jjMoveStringLiteralDfa10_0(active0, 0x800L);
+ case 66:
+ case 98:
+ return jjMoveStringLiteralDfa10_0(active0, 0x400L);
+ default :
+ break;
+ }
+ return jjStartNfa_0(8, active0);
+}
+private int jjMoveStringLiteralDfa10_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(8, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(9, active0);
+ return 10;
+ }
+ switch(curChar)
+ {
+ case 66:
+ case 98:
+ return jjMoveStringLiteralDfa11_0(active0, 0x800L);
+ case 89:
+ case 121:
+ if ((active0 & 0x400L) != 0L)
+ return jjStopAtPos(10, 10);
+ break;
+ default :
+ break;
+ }
+ return jjStartNfa_0(9, active0);
+}
+private int jjMoveStringLiteralDfa11_0(long old0, long active0)
+{
+ if (((active0 &= old0)) == 0L)
+ return jjStartNfa_0(9, old0);
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) {
+ jjStopStringLiteralDfa_0(10, active0);
+ return 11;
+ }
+ switch(curChar)
+ {
+ case 89:
+ case 121:
+ if ((active0 & 0x800L) != 0L)
+ return jjStopAtPos(11, 11);
+ break;
+ default :
+ break;
+ }
+ return jjStartNfa_0(10, active0);
+}
+static final long[] jjbitVec0 = {
+ 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+private int jjMoveNfa_0(int startState, int curPos)
+{
+ int startsAt = 0;
+ jjnewStateCnt = 27;
+ int i = 1;
+ jjstateSet[0] = startState;
+ int kind = 0x7fffffff;
+ for (;;)
+ {
+ if (++jjround == 0x7fffffff)
+ ReInitRounds();
+ if (curChar < 64)
+ {
+ long l = 1L << curChar;
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 2:
+ if (curChar == 47)
+ jjstateSet[jjnewStateCnt++] = 3;
+ else if (curChar == 45)
+ jjstateSet[jjnewStateCnt++] = 0;
+ break;
+ case 27:
+ case 26:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 16)
+ kind = 16;
+ jjCheckNAdd(26);
+ break;
+ case 0:
+ if (curChar != 45)
+ break;
+ if (kind > 5)
+ kind = 5;
+ jjCheckNAdd(1);
+ break;
+ case 1:
+ if ((0xffffffffffffdbffL & l) == 0L)
+ break;
+ if (kind > 5)
+ kind = 5;
+ jjCheckNAdd(1);
+ break;
+ case 3:
+ if (curChar == 42)
+ jjCheckNAddTwoStates(4, 5);
+ break;
+ case 4:
+ if ((0xfffffbffffffffffL & l) != 0L)
+ jjCheckNAddTwoStates(4, 5);
+ break;
+ case 5:
+ if (curChar == 42)
+ jjCheckNAddStates(0, 2);
+ break;
+ case 6:
+ if ((0xffff7bffffffffffL & l) != 0L)
+ jjCheckNAddTwoStates(7, 5);
+ break;
+ case 7:
+ if ((0xfffffbffffffffffL & l) != 0L)
+ jjCheckNAddTwoStates(7, 5);
+ break;
+ case 8:
+ if (curChar == 47 && kind > 6)
+ kind = 6;
+ break;
+ case 9:
+ if (curChar == 47)
+ jjstateSet[jjnewStateCnt++] = 3;
+ break;
+ case 10:
+ if (curChar == 50 && kind > 7)
+ kind = 7;
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ else if (curChar < 128)
+ {
+ long l = 1L << (curChar & 077);
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 2:
+ if ((0x7fffffe07fffffeL & l) != 0L)
+ {
+ if (kind > 16)
+ kind = 16;
+ jjCheckNAddTwoStates(25, 26);
+ }
+ if ((0x200000002L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 23;
+ else if ((0x1000000010000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 19;
+ else if ((0x8000000080L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 16;
+ else if ((0x100000001000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 12;
+ break;
+ case 27:
+ if ((0x7fffffe87fffffeL & l) != 0L)
+ {
+ if (kind > 16)
+ kind = 16;
+ jjCheckNAdd(26);
+ }
+ if ((0x7fffffe07fffffeL & l) != 0L)
+ {
+ if (kind > 16)
+ kind = 16;
+ jjCheckNAddTwoStates(25, 26);
+ }
+ break;
+ case 1:
+ if (kind > 5)
+ kind = 5;
+ jjstateSet[jjnewStateCnt++] = 1;
+ break;
+ case 4:
+ jjCheckNAddTwoStates(4, 5);
+ break;
+ case 6:
+ case 7:
+ jjCheckNAddTwoStates(7, 5);
+ break;
+ case 11:
+ if ((0x800000008000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 10;
+ break;
+ case 12:
+ if ((0x400000004000000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 11;
+ break;
+ case 13:
+ if ((0x100000001000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 12;
+ break;
+ case 14:
+ if ((0x1000000010000L & l) != 0L && kind > 7)
+ kind = 7;
+ break;
+ case 15:
+ if ((0x20000000200L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 14;
+ break;
+ case 16:
+ if ((0x400000004000000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 15;
+ break;
+ case 17:
+ if ((0x8000000080L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 16;
+ break;
+ case 18:
+ if ((0x8000000080L & l) != 0L && kind > 8)
+ kind = 8;
+ break;
+ case 19:
+ if ((0x20000000200L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 18;
+ break;
+ case 20:
+ if ((0x1000000010000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 19;
+ break;
+ case 21:
+ if ((0x800000008000L & l) != 0L && kind > 8)
+ kind = 8;
+ break;
+ case 22:
+ if ((0x4000000040000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 21;
+ break;
+ case 23:
+ if ((0x40000000400000L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 22;
+ break;
+ case 24:
+ if ((0x200000002L & l) != 0L)
+ jjstateSet[jjnewStateCnt++] = 23;
+ break;
+ case 25:
+ if ((0x7fffffe07fffffeL & l) == 0L)
+ break;
+ if (kind > 16)
+ kind = 16;
+ jjCheckNAddTwoStates(25, 26);
+ break;
+ case 26:
+ if ((0x7fffffe87fffffeL & l) == 0L)
+ break;
+ if (kind > 16)
+ kind = 16;
+ jjCheckNAdd(26);
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ else
+ {
+ int i2 = (curChar & 0xff) >> 6;
+ long l2 = 1L << (curChar & 077);
+ do
+ {
+ switch(jjstateSet[--i])
+ {
+ case 1:
+ if ((jjbitVec0[i2] & l2) == 0L)
+ break;
+ if (kind > 5)
+ kind = 5;
+ jjstateSet[jjnewStateCnt++] = 1;
+ break;
+ case 4:
+ if ((jjbitVec0[i2] & l2) != 0L)
+ jjCheckNAddTwoStates(4, 5);
+ break;
+ case 6:
+ case 7:
+ if ((jjbitVec0[i2] & l2) != 0L)
+ jjCheckNAddTwoStates(7, 5);
+ break;
+ default : break;
+ }
+ } while(i != startsAt);
+ }
+ if (kind != 0x7fffffff)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = curPos;
+ kind = 0x7fffffff;
+ }
+ ++curPos;
+ if ((i = jjnewStateCnt) == (startsAt = 27 - (jjnewStateCnt = startsAt)))
+ return curPos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return curPos; }
+ }
+}
+static final int[] jjnextStates = {
+ 5, 6, 8,
+};
+
+/** Token literal values. */
+public static final String[] jjstrLiteralImages = {
+"", null, null, null, null, null, null, null, null, null, null, null, null,
+null, null, null, null, "\73", "\133", "\135", "\43", "\56", "\54", "\173", "\175",
+"\174", };
+
+/** Lexer state names. */
+public static final String[] lexStateNames = {
+ "DEFAULT",
+};
+static final long[] jjtoToken = {
+ 0x3ff0f81L,
+};
+static final long[] jjtoSkip = {
+ 0x7eL,
+};
+protected SimpleCharStream input_stream;
+private final int[] jjrounds = new int[27];
+private final int[] jjstateSet = new int[54];
+protected char curChar;
+/** Constructor. */
+public TableStorageParserTokenManager(SimpleCharStream stream){
+ if (SimpleCharStream.staticFlag)
+ throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
+ input_stream = stream;
+}
+
+/** Constructor. */
+public TableStorageParserTokenManager(SimpleCharStream stream, int lexState){
+ this(stream);
+ SwitchTo(lexState);
+}
+
+/** Reinitialise parser. */
+public void ReInit(SimpleCharStream stream)
+{
+ jjmatchedPos = jjnewStateCnt = 0;
+ curLexState = defaultLexState;
+ input_stream = stream;
+ ReInitRounds();
+}
+private void ReInitRounds()
+{
+ int i;
+ jjround = 0x80000001;
+ for (i = 27; i-- > 0;)
+ jjrounds[i] = 0x80000000;
+}
+
+/** Reinitialise parser. */
+public void ReInit(SimpleCharStream stream, int lexState)
+{
+ ReInit(stream);
+ SwitchTo(lexState);
+}
+
+/** Switch to specified lex state. */
+public void SwitchTo(int lexState)
+{
+ if (lexState >= 1 || lexState < 0)
+ throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
+ else
+ curLexState = lexState;
+}
+
+protected Token jjFillToken()
+{
+ final Token t;
+ final String curTokenImage;
+ final int beginLine;
+ final int endLine;
+ final int beginColumn;
+ final int endColumn;
+ String im = jjstrLiteralImages[jjmatchedKind];
+ curTokenImage = (im == null) ? input_stream.GetImage() : im;
+ beginLine = input_stream.getBeginLine();
+ beginColumn = input_stream.getBeginColumn();
+ endLine = input_stream.getEndLine();
+ endColumn = input_stream.getEndColumn();
+ t = Token.newToken(jjmatchedKind, curTokenImage);
+
+ t.beginLine = beginLine;
+ t.endLine = endLine;
+ t.beginColumn = beginColumn;
+ t.endColumn = endColumn;
+
+ return t;
+}
+
+int curLexState = 0;
+int defaultLexState = 0;
+int jjnewStateCnt;
+int jjround;
+int jjmatchedPos;
+int jjmatchedKind;
+
+/** Get the next Token. */
+public Token getNextToken()
+{
+ Token matchedToken;
+ int curPos = 0;
+
+ EOFLoop :
+ for (;;)
+ {
+ try
+ {
+ curChar = input_stream.BeginToken();
+ }
+ catch(java.io.IOException e)
+ {
+ jjmatchedKind = 0;
+ matchedToken = jjFillToken();
+ return matchedToken;
+ }
+
+ try { input_stream.backup(0);
+ while (curChar <= 32 && (0x100002600L & (1L << curChar)) != 0L)
+ curChar = input_stream.BeginToken();
+ }
+ catch (java.io.IOException e1) { continue EOFLoop; }
+ jjmatchedKind = 0x7fffffff;
+ jjmatchedPos = 0;
+ curPos = jjMoveStringLiteralDfa0_0();
+ if (jjmatchedKind != 0x7fffffff)
+ {
+ if (jjmatchedPos + 1 < curPos)
+ input_stream.backup(curPos - jjmatchedPos - 1);
+ if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
+ {
+ matchedToken = jjFillToken();
+ return matchedToken;
+ }
+ else
+ {
+ continue EOFLoop;
+ }
+ }
+ int error_line = input_stream.getEndLine();
+ int error_column = input_stream.getEndColumn();
+ String error_after = null;
+ boolean EOFSeen = false;
+ try { input_stream.readChar(); input_stream.backup(1); }
+ catch (java.io.IOException e1) {
+ EOFSeen = true;
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ if (curChar == '\n' || curChar == '\r') {
+ error_line++;
+ error_column = 0;
+ }
+ else
+ error_column++;
+ }
+ if (!EOFSeen) {
+ input_stream.backup(1);
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ }
+ throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
+ }
+}
+
+private void jjCheckNAdd(int state)
+{
+ if (jjrounds[state] != jjround)
+ {
+ jjstateSet[jjnewStateCnt++] = state;
+ jjrounds[state] = jjround;
+ }
+}
+private void jjAddStates(int start, int end)
+{
+ do {
+ jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+ } while (start++ != end);
+}
+private void jjCheckNAddTwoStates(int state1, int state2)
+{
+ jjCheckNAdd(state1);
+ jjCheckNAdd(state2);
+}
+
+private void jjCheckNAddStates(int start, int end)
+{
+ do {
+ jjCheckNAdd(jjnextStates[start]);
+ } while (start++ != end);
+}
+
+}
Added: hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Token.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Token.java?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Token.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/java/org/apache/hadoop/zebra/types/Token.java Tue Aug 11 22:27:44 2009
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 4.1 */
+/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package org.apache.hadoop.zebra.types;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token implements java.io.Serializable {
+
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * An integer that describes the kind of this token. This numbering
+ * system is determined by JavaCCParser, and a table of these numbers is
+ * stored in the file ...Constants.java.
+ */
+ public int kind;
+
+ /** The line number of the first character of this Token. */
+ public int beginLine;
+ /** The column number of the first character of this Token. */
+ public int beginColumn;
+ /** The line number of the last character of this Token. */
+ public int endLine;
+ /** The column number of the last character of this Token. */
+ public int endColumn;
+
+ /**
+ * The string image of the token.
+ */
+ public String image;
+
+ /**
+ * A reference to the next regular (non-special) token from the input
+ * stream. If this is the last token from the input stream, or if the
+ * token manager has not read tokens beyond this one, this field is
+ * set to null. This is true only if this token is also a regular
+ * token. Otherwise, see below for a description of the contents of
+ * this field.
+ */
+ public Token next;
+
+ /**
+ * This field is used to access special tokens that occur prior to this
+ * token, but after the immediately preceding regular (non-special) token.
+ * If there are no such special tokens, this field is set to null.
+ * When there are more than one such special token, this field refers
+ * to the last of these special tokens, which in turn refers to the next
+ * previous special token through its specialToken field, and so on
+ * until the first special token (whose specialToken field is null).
+ * The next fields of special tokens refer to other special tokens that
+ * immediately follow it (without an intervening regular token). If there
+ * is no such token, this field is null.
+ */
+ public Token specialToken;
+
+ /**
+ * An optional attribute value of the Token.
+ * Tokens which are not used as syntactic sugar will often contain
+ * meaningful values that will be used later on by the compiler or
+ * interpreter. This attribute value is often different from the image.
+ * Any subclass of Token that actually wants to return a non-null value can
+ * override this method as appropriate.
+ */
+ public Object getValue() {
+ return null;
+ }
+
+ /**
+ * No-argument constructor
+ */
+ public Token() {}
+
+ /**
+ * Constructs a new token for the specified Image.
+ */
+ public Token(int kind)
+ {
+ this(kind, null);
+ }
+
+ /**
+ * Constructs a new token for the specified Image and Kind.
+ */
+ public Token(int kind, String image)
+ {
+ this.kind = kind;
+ this.image = image;
+ }
+
+ /**
+ * Returns the image.
+ */
+ public String toString()
+ {
+ return image;
+ }
+
+ /**
+ * Returns a new Token object, by default. However, if you want, you
+ * can create and return subclass objects based on the value of ofKind.
+ * Simply add the cases to the switch for all those special cases.
+ * For example, if you have a subclass of Token called IDToken that
+ * you want to create if ofKind is ID, simply add something like :
+ *
+ * case MyParserConstants.ID : return new IDToken(ofKind, image);
+ *
+ * to the following switch statement. Then you can cast matchedToken
+ * variable to the appropriate type and use sit in your lexical actions.
+ */
+ public static Token newToken(int ofKind, String image)
+ {
+ switch(ofKind)
+ {
+ default : return new Token(ofKind, image);
+ }
+ }
+
+ public static Token newToken(int ofKind)
+ {
+ return newToken(ofKind, null);
+ }
+
+}
+/* JavaCC - OriginalChecksum=01eb2fb32f13b4873357ad1b13269d27 (do not edit this line) */