You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@velocity.apache.org by jv...@locus.apache.org on 2000/09/03 02:31:41 UTC
cvs commit: jakarta-velocity/src/java/org/apache/velocity/parser/antlr VelocityRecognizer.java VelocityRecognizerTokenTypes.java VelocityRecognizerTokenTypes.txt test.vm velocity.g
jvanzyl 00/09/02 17:31:41
Added: src/java/org/apache/velocity/parser/antlr
VelocityRecognizer.java
VelocityRecognizerTokenTypes.java
VelocityRecognizerTokenTypes.txt test.vm velocity.g
Log:
- Terrence Parr has contributed a little proof of concept
grammar. Terrence Parr is the creator of ANTLR!
Revision Changes Path
1.1 jakarta-velocity/src/java/org/apache/velocity/parser/antlr/VelocityRecognizer.java
Index: VelocityRecognizer.java
===================================================================
// $ANTLR 2.7.0: "velocity.g" -> "VelocityRecognizer.java"$
import java.io.InputStream;
import antlr.TokenStreamException;
import antlr.TokenStreamIOException;
import antlr.TokenStreamRecognitionException;
import antlr.CharStreamException;
import antlr.CharStreamIOException;
import antlr.ANTLRException;
import java.io.Reader;
import java.util.Hashtable;
import antlr.CharScanner;
import antlr.InputBuffer;
import antlr.ByteBuffer;
import antlr.CharBuffer;
import antlr.Token;
import antlr.CommonToken;
import antlr.RecognitionException;
import antlr.NoViableAltForCharException;
import antlr.MismatchedCharException;
import antlr.TokenStream;
import antlr.ANTLRHashString;
import antlr.LexerSharedInputState;
import antlr.collections.impl.BitSet;
import antlr.SemanticException;
/** A simple proof of concept for proposed
* syntax changes to Velocity. This is not meant to be
* a specification--it merely demonstrates the feasibility
* of having a single lexer (no parser and w/o lexer states)
* handle Velocity input. Presumably, you would build an
* intermediate form (tree or whatever) from this lexer
* and then interpret it.
*
* Terence Parr, jGuru.com, antlr.org
* No rights reserved, no copyright.
* September 2, 2000.
*/
public class VelocityRecognizer extends antlr.CharScanner implements VelocityRecognizerTokenTypes, TokenStream
{
public static void main(String[] args) throws Exception {
VelocityRecognizer v = new VelocityRecognizer(System.in);
v.nextToken(); // ask for one token and it parses everything!
}
public VelocityRecognizer(InputStream in) {
this(new ByteBuffer(in));
}
public VelocityRecognizer(Reader in) {
this(new CharBuffer(in));
}
public VelocityRecognizer(InputBuffer ib) {
this(new LexerSharedInputState(ib));
}
public VelocityRecognizer(LexerSharedInputState state) {
super(state);
literals = new Hashtable();
caseSensitiveLiterals = true;
setCaseSensitive(true);
}
public Token nextToken() throws TokenStreamException {
Token theRetToken=null;
tryAgain:
for (;;) {
Token _token = null;
int _ttype = Token.INVALID_TYPE;
resetText();
try { // for char stream error handling
try { // for lexical error handling
if (((LA(1) >= '\3' && LA(1) <= '\377'))) {
mCONTENT(true);
theRetToken=_returnToken;
}
else {
if (LA(1)==EOF_CHAR) {uponEOF(); _returnToken = makeToken(Token.EOF_TYPE);}
else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());}
}
if ( _returnToken==null ) continue tryAgain; // found SKIP token
_ttype = _returnToken.getType();
_ttype = testLiteralsTable(_ttype);
_returnToken.setType(_ttype);
return _returnToken;
}
catch (RecognitionException e) {
throw new TokenStreamRecognitionException(e);
}
}
catch (CharStreamException cse) {
if ( cse instanceof CharStreamIOException ) {
throw new TokenStreamIOException(((CharStreamIOException)cse).io);
}
else {
throw new TokenStreamException(cse.getMessage());
}
}
}
}
public final void mCONTENT(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = CONTENT;
int _saveIndex;
{
int _cnt3=0;
_loop3:
do {
if (((LA(1) >= '\3' && LA(1) <= '\377'))) {
mCONTENT_ITEM(false);
}
else {
if ( _cnt3>=1 ) { break _loop3; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());}
}
_cnt3++;
} while (true);
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mCONTENT_ITEM(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = CONTENT_ITEM;
int _saveIndex;
if ((LA(1)=='$') && (LA(2)=='i') && (LA(3)=='f') && (_tokenSet_0.member(LA(4)))) {
mIF(false);
}
else if ((LA(1)=='$') && (_tokenSet_1.member(LA(2))) && (true) && (true)) {
mVARIABLE(false);
}
else if ((_tokenSet_2.member(LA(1)))) {
mTEXT(false);
}
else {
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mIF(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = IF;
int _saveIndex;
Token v=null;
Token c=null;
Token c2=null;
match("$if");
{
switch ( LA(1)) {
case '\t': case '\n': case '\r': case ' ':
{
mWS(false);
break;
}
case '(':
{
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
}
}
match('(');
{
switch ( LA(1)) {
case '\t': case '\n': case '\r': case ' ':
{
mWS(false);
break;
}
case '$':
{
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
}
}
mVARIABLE(true);
v=_returnToken;
{
switch ( LA(1)) {
case '\t': case '\n': case '\r': case ' ':
{
mWS(false);
break;
}
case ')':
{
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
}
}
match(')');
System.out.println("If var: "+v.getText());
mNESTED_CONTENT(true);
c=_returnToken;
System.out.println("If body: "+c.getText());
{
if ((LA(1)=='$') && (LA(2)=='e') && (LA(3)=='l')) {
match("$else");
mNESTED_CONTENT(true);
c2=_returnToken;
System.out.println("Else body: "+c2.getText());
}
else if ((LA(1)=='$') && (LA(2)=='e') && (LA(3)=='n')) {
}
else {
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
}
match("$end");
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mVARIABLE(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = VARIABLE;
int _saveIndex;
match('$');
mID(false);
{
_loop11:
do {
if ((LA(1)=='.') && (_tokenSet_1.member(LA(2))) && (true) && (true)) {
match('.');
mID(false);
{
if ((LA(1)=='(') && (_tokenSet_3.member(LA(2))) && (true) && (true)) {
match('(');
{
switch ( LA(1)) {
case '\t': case '\n': case '\r': case ' ':
case '"':
{
{
switch ( LA(1)) {
case '\t': case '\n': case '\r': case ' ':
{
mWS(false);
break;
}
case '"':
{
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
}
}
mLITERAL(false);
{
switch ( LA(1)) {
case '\t': case '\n': case '\r': case ' ':
{
mWS(false);
break;
}
case ')':
{
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
}
}
break;
}
case ')':
{
break;
}
default:
{
throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());
}
}
}
match(')');
}
else {
}
}
}
else {
break _loop11;
}
} while (true);
}
System.out.println("Variable: "+new String(text.getBuffer(),_begin,text.length()-_begin));
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mTEXT(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = TEXT;
int _saveIndex;
{
int _cnt25=0;
_loop25:
do {
if ((_tokenSet_2.member(LA(1))) && (true) && (true) && (true)) {
matchNot('$');
}
else {
if ( _cnt25>=1 ) { break _loop25; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());}
}
_cnt25++;
} while (true);
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mID(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = ID;
int _saveIndex;
{
int _cnt28=0;
_loop28:
do {
if (((LA(1) >= 'a' && LA(1) <= 'z')) && (true) && (true) && (true)) {
matchRange('a','z');
}
else if (((LA(1) >= 'A' && LA(1) <= 'Z')) && (true) && (true) && (true)) {
matchRange('A','Z');
}
else if (((LA(1) >= '0' && LA(1) <= '9')) && (true) && (true) && (true)) {
matchRange('0','9');
}
else {
if ( _cnt28>=1 ) { break _loop28; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());}
}
_cnt28++;
} while (true);
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mWS(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = WS;
int _saveIndex;
{
int _cnt31=0;
_loop31:
do {
switch ( LA(1)) {
case ' ':
{
match(' ');
break;
}
case '\t':
{
match('\t');
break;
}
case '\n':
{
match('\n');
break;
}
case '\r':
{
match('\r');
break;
}
default:
{
if ( _cnt31>=1 ) { break _loop31; } else {throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine());}
}
}
_cnt31++;
} while (true);
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mLITERAL(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = LITERAL;
int _saveIndex;
match('"');
{
_loop22:
do {
if ((_tokenSet_4.member(LA(1)))) {
matchNot('"');
}
else {
break _loop22;
}
} while (true);
}
match('"');
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
protected final void mNESTED_CONTENT(boolean _createToken) throws RecognitionException, CharStreamException, TokenStreamException {
int _ttype; Token _token=null; int _begin=text.length();
_ttype = NESTED_CONTENT;
int _saveIndex;
{
_loop19:
do {
// nongreedy exit test
if ((LA(1)=='$') && (LA(2)=='e') && (LA(3)=='l'||LA(3)=='n') && (LA(4)=='d'||LA(4)=='s')) break _loop19;
if (((LA(1) >= '\3' && LA(1) <= '\377')) && ((LA(2) >= '\3' && LA(2) <= '\377')) && ((LA(3) >= '\3' && LA(3) <= '\377')) && ((LA(4) >= '\3' && LA(4) <= '\377'))) {
mCONTENT_ITEM(false);
}
else {
break _loop19;
}
} while (true);
}
if ( _createToken && _token==null && _ttype!=Token.SKIP ) {
_token = makeToken(_ttype);
_token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));
}
_returnToken = _token;
}
private static final long _tokenSet_0_data_[] = { 1103806604800L, 0L, 0L, 0L, 0L };
public static final BitSet _tokenSet_0 = new BitSet(_tokenSet_0_data_);
private static final long _tokenSet_1_data_[] = { 287948901175001088L, 576460743847706622L, 0L, 0L, 0L };
public static final BitSet _tokenSet_1 = new BitSet(_tokenSet_1_data_);
private static final long _tokenSet_2_data_[] = { -68719476744L, -1L, -1L, -1L, 0L, 0L, 0L, 0L };
public static final BitSet _tokenSet_2 = new BitSet(_tokenSet_2_data_);
private static final long _tokenSet_3_data_[] = { 2220498101760L, 0L, 0L, 0L, 0L };
public static final BitSet _tokenSet_3 = new BitSet(_tokenSet_3_data_);
private static final long _tokenSet_4_data_[] = { -17179869192L, -1L, -1L, -1L, 0L, 0L, 0L, 0L };
public static final BitSet _tokenSet_4 = new BitSet(_tokenSet_4_data_);
}
1.1 jakarta-velocity/src/java/org/apache/velocity/parser/antlr/VelocityRecognizerTokenTypes.java
Index: VelocityRecognizerTokenTypes.java
===================================================================
// $ANTLR 2.7.0: "velocity.g" -> "VelocityRecognizer.java"$
public interface VelocityRecognizerTokenTypes {
int EOF = 1;
int NULL_TREE_LOOKAHEAD = 3;
int CONTENT = 4;
int CONTENT_ITEM = 5;
int VARIABLE = 6;
int IF = 7;
int NESTED_CONTENT = 8;
int LITERAL = 9;
int TEXT = 10;
int ID = 11;
int WS = 12;
}
1.1 jakarta-velocity/src/java/org/apache/velocity/parser/antlr/VelocityRecognizerTokenTypes.txt
Index: VelocityRecognizerTokenTypes.txt
===================================================================
// $ANTLR 2.7.0: velocity.g -> VelocityRecognizerTokenTypes.txt$
VelocityRecognizer // output token vocab name
CONTENT=4
CONTENT_ITEM=5
VARIABLE=6
IF=7
NESTED_CONTENT=8
LITERAL=9
TEXT=10
ID=11
WS=12
1.1 jakarta-velocity/src/java/org/apache/velocity/parser/antlr/test.vm
Index: test.vm
===================================================================
This is a test of
$if ($foo)
Velocity
$else
WebMacro
$end.
<b>More testing of $bob.method( " arg " ).call()</b>
more ick.
$if ($bar)
<b>$if ($xxx) Nested $end</b>
$end
1.1 jakarta-velocity/src/java/org/apache/velocity/parser/antlr/velocity.g
Index: velocity.g
===================================================================
/** A simple proof of concept for proposed
* syntax changes to Velocity. This is not meant to be
* a specification--it merely demonstrates the feasibility
* of having a single lexer (no parser and w/o lexer states)
* handle Velocity input. Presumably, you would build an
* intermediate form (tree or whatever) from this lexer
* and then interpret it.
*
* Terence Parr, jGuru.com, antlr.org
* No rights reserved, no copyright.
* September 2, 2000.
*/
class VelocityRecognizer extends Lexer;
options {
charVocabulary = '\3'..'\377'; // assume ascii for now
k=4;
}
{
public static void main(String[] args) throws Exception {
VelocityRecognizer v = new VelocityRecognizer(System.in);
v.nextToken(); // ask for one token and it parses everything!
}
}
CONTENT
: (CONTENT_ITEM)+
;
protected
CONTENT_ITEM
: IF
| VARIABLE
| TEXT
;
protected
VARIABLE
: '$' ID
( options {greedy=true;}
: '.' ID (options {greedy=true;} : '(' ((WS)? LITERAL (WS)?)? ')' )?
)*
{System.out.println("Variable: "+$getText);}
;
protected
IF : "$if" (WS)? '(' (WS)? v:VARIABLE (WS)? ')'
{System.out.println("If var: "+v.getText());}
c:NESTED_CONTENT
{System.out.println("If body: "+c.getText());}
( "$else" c2:NESTED_CONTENT
{System.out.println("Else body: "+c2.getText());}
)?
"$end"
;
protected
NESTED_CONTENT
: ( options {greedy=false;} : CONTENT_ITEM )* // fall out upon $end
;
protected
LITERAL
: '"' (~'"')* '"'
;
protected
TEXT: (options {greedy=true;} : ~'$')+
// {System.out.println("Text: "+$getText);}
;
protected
ID : (options {greedy=true;} : 'a'..'z'|'A'..'Z'|'0'..'9')+
;
protected
WS : (' '|'\t'|'\n'|'\r')+
;
Re: Terrence Parr and Antlr grammar
Posted by Jon Stevens <jo...@latchkey.com>.
on 9/2/2000 5:31 PM, "Jason van Zyl" <jv...@periapt.com> wrote:
> Anyway I would like to vote to give Terrence CVS
> write. I hope that his effort continues as it
> would amazing to have him help with Velocity!
>
> +1 For Terrence!
+1
-jon
--
http://scarab.tigris.org/ | http://noodle.tigris.org/
http://java.apache.org/ | http://java.apache.org/turbine/
http://www.working-dogs.com/ | http://jakarta.apache.org/velocity/
http://www.collab.net/ | http://www.sourcexchange.com/
Terrence Parr and Antlr grammar
Posted by Jason van Zyl <jv...@periapt.com>.
Hey,
Good news! Terrence Parr has donated a proof of concept
grammar, and some of his experience. We are currently
discussing options for the future grammar of Velocity.
He is an incredible resource as the creator of Antlr,
and someone who has years of experience writing
parsers for all sorts of languages.
Anyway I would like to vote to give Terrence CVS
write. I hope that his effort continues as it
would amazing to have him help with Velocity!
+1 For Terrence!
jvz.
--
Jason van Zyl
jvanzyl@periapt.com