You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/08/14 22:21:23 UTC
svn commit: r1157646 - in /incubator/lcf/trunk/framework: ./ scriptengine/
scriptengine/src/ scriptengine/src/main/ scriptengine/src/main/java/
scriptengine/src/main/java/org/ scriptengine/src/main/java/org/apache/
scriptengine/src/main/java/org/apache...
Author: kwright
Date: Sun Aug 14 20:21:23 2011
New Revision: 1157646
URL: http://svn.apache.org/viewvc?rev=1157646&view=rev
Log:
Begin work on script engine. This commit represents the lexical analysis part.
Added:
incubator/lcf/trunk/framework/scriptengine/
incubator/lcf/trunk/framework/scriptengine/src/
incubator/lcf/trunk/framework/scriptengine/src/main/
incubator/lcf/trunk/framework/scriptengine/src/main/java/
incubator/lcf/trunk/framework/scriptengine/src/main/java/org/
incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/
incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/
incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/
incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java (with props)
incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java (with props)
incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java (with props)
Modified:
incubator/lcf/trunk/framework/build.xml
Modified: incubator/lcf/trunk/framework/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/build.xml?rev=1157646&r1=1157645&r2=1157646&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/build.xml (original)
+++ incubator/lcf/trunk/framework/build.xml Sun Aug 14 20:21:23 2011
@@ -102,6 +102,18 @@
</javac>
</target>
+ <target name="compile-script-engine" depends="compile-core">
+ <mkdir dir="build/script-engine/classes"/>
+ <javac srcdir="scriptengine/src/main/java" destdir="build/script-engine/classes" target="1.5" source="1.5" debug="true" debuglevel="lines,vars,source">
+ <classpath>
+ <fileset dir="lib">
+ <include name="*.jar"/>
+ </fileset>
+ <pathelement location="build/core/classes"/>
+ </classpath>
+ </javac>
+ </target>
+
<target name="compile-authority-servlet" depends="compile-core,compile-agents,compile-pull-agent">
<mkdir dir="build/authority-servlet/classes"/>
<javac srcdir="authority-servlet/src/main/java" destdir="build/authority-servlet/classes" target="1.5" source="1.5" debug="true" debuglevel="lines,vars,source">
@@ -193,6 +205,11 @@
<jar destfile="build/jar/mcf-jetty-runner.jar" basedir="build/jetty-runner/classes"/>
</target>
+ <target name="jar-script-engine" depends="compile-script-engine">
+ <mkdir dir="build/jar"/>
+ <jar destfile="build/jar/mcf-script-engine.jar" basedir="build/script-engine/classes"/>
+ </target>
+
<target name="webapp-authority-service" depends="jar-core,jar-agents,jar-pull-agent,compile-authority-servlet">
<mkdir dir="build/webapp/authority-service/WEB-INF/lib"/>
<copy todir="build/webapp/authority-service/WEB-INF/lib">
@@ -367,6 +384,24 @@
</copy>
</target>
+ <target name="script-engine" depends="jar-script-engine,jar-core">
+ <mkdir dir="dist/script-engine/lib"/>
+ <copy todir="dist/script-engine/lib">
+ <fileset dir="lib">
+ <include name="commons-codec*.jar"/>
+ <include name="commons-collections*.jar"/>
+ <include name="commons-httpclient-mcf.jar"/>
+ <include name="commons-io*.jar"/>
+ <include name="commons-logging*.jar"/>
+ </fileset>
+ </copy>
+ <copy todir="dist/example/lib">
+ <fileset dir="build/jar">
+ <include name="mcf-core.jar"/>
+ </fileset>
+ </copy>
+ </target>
+
<target name="example" depends="war-crawler-ui,war-api-service,war-authority-service,jar-jetty-runner,jar-core,jar-agents,jar-pull-agent">
<mkdir dir="dist/example/lib"/>
<copy todir="dist/example/lib">
@@ -555,7 +590,7 @@
</junit>
</target>
- <target name="build" depends="processes,war-authority-service,war-crawler-ui,war-api-service,example"/>
+ <target name="build" depends="processes,war-authority-service,war-crawler-ui,war-api-service,example,script-engine"/>
<target name="all" depends="build,doc,build-tests,run-tests,run-tests-postgresql"/>
Added: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java?rev=1157646&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java (added)
+++ incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java Sun Aug 14 20:21:23 2011
@@ -0,0 +1,44 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/** This class tracks the position of a script execution, and allows
+* access to the characters in a nested file setup.
+*/
+public class Position
+{
+ protected TokenStream tokenStream;
+ protected int characterPosition;
+
+ public Position(TokenStream s, int characterPosition)
+ {
+ this.tokenStream = s;
+ this.characterPosition = characterPosition;
+ }
+
+ public TokenStream getTokenStream()
+ {
+ return tokenStream;
+ }
+
+ public int getCharacterPosition()
+ {
+ return characterPosition;
+ }
+
+}
\ No newline at end of file
Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java?rev=1157646&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java (added)
+++ incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java Sun Aug 14 20:21:23 2011
@@ -0,0 +1,48 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+public class Token
+{
+ public static final int TOKEN_PUNCTUATION = 0;
+ public static final int TOKEN_STRING = 1;
+
+ protected int tokenType;
+ protected String tokenValue;
+
+ public Token(int type, String value)
+ {
+ this.tokenType = type;
+ this.tokenValue = tokenValue;
+ }
+
+ public String getPunctuation()
+ {
+ if (tokenType == TOKEN_PUNCTUATION)
+ return tokenValue;
+ return null;
+ }
+
+ public String getString()
+ {
+ if (tokenType == TOKEN_STRING)
+ return tokenValue;
+ return null;
+ }
+
+}
\ No newline at end of file
Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java?rev=1157646&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java (added)
+++ incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java Sun Aug 14 20:21:23 2011
@@ -0,0 +1,170 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/** Convert a script into a stream of tokens.
+*/
+public class TokenStream
+{
+ protected String fileName;
+ protected String code;
+ protected int characterPosition = 0;
+ protected Token currentToken = null;
+
+ public TokenStream(String fileName, String code)
+ {
+ this.fileName = fileName;
+ this.code = code;
+ }
+
+ public int getCharacterPosition()
+ {
+ return characterPosition;
+ }
+
+ public void setCharacterPosition(int characterPosition)
+ {
+ this.characterPosition = characterPosition;
+ currentToken = null;
+ }
+
+ public Token peek()
+ {
+ if (currentToken == null)
+ currentToken = parseNextToken();
+ return currentToken;
+ }
+
+ public void skip()
+ {
+ currentToken = null;
+ }
+
+ public String getFileName()
+ {
+ return fileName;
+ }
+
+ // Protected methods
+
+ protected Token parseNextToken()
+ {
+ // Skip to the start of the next token (or exit if no token start can be found)
+ while (true)
+ {
+ // Skip white space
+ while (characterPosition < code.length())
+ {
+ char x = code.charAt(characterPosition);
+ if (x > ' ')
+ break;
+ characterPosition++;
+ }
+
+ // End of text?
+ if (characterPosition == code.length())
+ return null;
+
+ // Is it a comment?
+ char y = code.charAt(characterPosition);
+ if (y == '#')
+ {
+ // Skip to the end of the line, or the end of the text.
+ while (characterPosition < code.length())
+ {
+ char x = code.charAt(characterPosition++);
+ if (x == '\n')
+ break;
+ }
+ continue;
+ }
+ break;
+ }
+
+ // At the start of the next token.
+ // Figure out what kind of token it is. Currently we only care about punctuation
+ // and strings. Furthermore, no distinction is made between quoted and unquoted
+ // strings.
+ char z = code.charAt(characterPosition);
+ if (isLegalTokenCharacter(z))
+ {
+ // Legal token character
+ StringBuilder tokenBuffer = new StringBuilder();
+ int quoteMark = -1;
+ while (characterPosition < code.length())
+ {
+ char x = code.charAt(characterPosition);
+ if (quoteMark == -1)
+ {
+ // Not yet in a quote
+ if (x <= ' ' || x == '#')
+ break;
+ if (x == '\'' || x == '"')
+ {
+ characterPosition++;
+ quoteMark = (int)x;
+ }
+ else if (isLegalTokenCharacter(x))
+ {
+ characterPosition++;
+ tokenBuffer.append(x);
+ }
+ else
+ break;
+ }
+ else
+ {
+ characterPosition++;
+ // In a quote.
+ if (((int)x) == quoteMark)
+ // End of the quotation
+ break;
+ if (x == '\\')
+ {
+ // Escape character
+ if (characterPosition < code.length())
+ tokenBuffer.append(code.charAt(characterPosition++));
+ }
+ else
+ tokenBuffer.append(x);
+ }
+ }
+ return new Token(Token.TOKEN_STRING,tokenBuffer.toString());
+ }
+ else
+ {
+ // Set a punctuation token
+ characterPosition++;
+ return new Token(Token.TOKEN_PUNCTUATION,new StringBuilder(z).toString());
+ }
+ }
+
+ protected static boolean isLegalTokenCharacter(char x)
+ {
+ if (x >= '0' && x <= '9')
+ return true;
+ if (x >= 'a' && x <= 'z')
+ return true;
+ if (x >= 'A' && x <='Z')
+ return true;
+ if (x == '_' || x == '$' || x == '@')
+ return true;
+ return false;
+ }
+
+}
Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
------------------------------------------------------------------------------
svn:keywords = Id