You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by mb...@apache.org on 2007/11/22 11:39:33 UTC
svn commit: r597367 - in /incubator/uima/sandbox/trunk/WhitespaceTokenizer:
./ src/main/java/org/apache/uima/annotator/ src/main/resources/
src/main/resources/org/ src/main/resources/org/apache/
src/main/resources/org/apache/uima/ src/main/resources/or...
Author: mbaessler
Date: Thu Nov 22 02:39:28 2007
New Revision: 597367
URL: http://svn.apache.org/viewvc?rev=597367&view=rev
Log:
UIMA-645
externalize log messages
https://issues.apache.org/jira/browse/UIMA-645
Added:
incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/
incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/
incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/
incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/uima/
incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/uima/annotator/
incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/uima/annotator/whitespaceTokenizerMessages.properties
Modified:
incubator/uima/sandbox/trunk/WhitespaceTokenizer/pom.xml
incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/java/org/apache/uima/annotator/WhitespaceTokenizer.java
Modified: incubator/uima/sandbox/trunk/WhitespaceTokenizer/pom.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/WhitespaceTokenizer/pom.xml?rev=597367&r1=597366&r2=597367&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/WhitespaceTokenizer/pom.xml (original)
+++ incubator/uima/sandbox/trunk/WhitespaceTokenizer/pom.xml Thu Nov 22 02:39:28 2007
@@ -54,6 +54,9 @@
<resource>
<directory>desc</directory>
</resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
</resources>
</build>
<repositories>
Modified: incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/java/org/apache/uima/annotator/WhitespaceTokenizer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/java/org/apache/uima/annotator/WhitespaceTokenizer.java?rev=597367&r1=597366&r2=597367&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/java/org/apache/uima/annotator/WhitespaceTokenizer.java (original)
+++ incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/java/org/apache/uima/annotator/WhitespaceTokenizer.java Thu Nov 22 02:39:28 2007
@@ -38,183 +38,196 @@
public class WhitespaceTokenizer extends JTextAnnotator_ImplBase {
- private static final int CH_SPECIAL = 0;
+ private static final int CH_SPECIAL = 0;
- private static final int CH_NUMBER = 1;
+ private static final int CH_NUMBER = 1;
- private static final int CH_LETTER = 2;
+ private static final int CH_LETTER = 2;
- private static final int CH_WHITESPACE = 4;
+ private static final int CH_WHITESPACE = 4;
- private static final int CH_PUNCTUATION = 5;
-
- private static final int CH_NEWLINE = 6;
-
- private static final int UNDEFINED = -1;
-
- private static final int INVALID_CHAR = 0;
-
- public static final String TOKEN_ANNOTATION_NAME = "org.apache.uima.TokenAnnotation";
-
- public static final String SENTENCE_ANNOTATION_NAME = "org.apache.uima.SentenceAnnotation";
-
- public static final String TOKEN_TYPE_FEATURE_NAME = "tokenType";
-
- private Type tokenType;
-
- private Type sentenceType;
-
- private CAS cas = null;
-
- private Logger logger;
-
- private static List<String> punctuations = Arrays.asList(new String[] { ".", "!", "?" });
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.uima.analysis_engine.annotator.JTextAnnotator#process(org.apache.uima.jcas.impl.JCas,
- * org.apache.uima.analysis_engine.ResultSpecification)
- */
- public void process(JCas aJCas, ResultSpecification aResultSpec) throws AnnotatorProcessException {
- this.logger.log(Level.INFO, "Whitespace tokenizer starts processing");
-
- // get text content from the CAS
- char[] textContent = aJCas.getCas().getDocumentText().toCharArray();
- this.cas = aJCas.getCas();
- int tokenStart = UNDEFINED;
- int currentCharPos = 0;
- int sentenceStart = 0;
- int nextCharType = UNDEFINED;
- char nextChar = INVALID_CHAR;
-
- while (currentCharPos < textContent.length) {
- char currentChar = textContent[currentCharPos];
- int currentCharType = getCharacterType(currentChar);
-
- //get character class for current and next character
- if ((currentCharPos + 1) < textContent.length) {
- nextChar = textContent[currentCharPos + 1];
- nextCharType = getCharacterType(nextChar);
- } else {
- nextCharType = UNDEFINED;
- nextChar = INVALID_CHAR;
+ private static final int CH_PUNCTUATION = 5;
+
+ private static final int CH_NEWLINE = 6;
+
+ private static final int UNDEFINED = -1;
+
+ private static final int INVALID_CHAR = 0;
+
+ public static final String TOKEN_ANNOTATION_NAME = "org.apache.uima.TokenAnnotation";
+
+ public static final String SENTENCE_ANNOTATION_NAME = "org.apache.uima.SentenceAnnotation";
+
+ public static final String TOKEN_TYPE_FEATURE_NAME = "tokenType";
+
+ private Type tokenType;
+
+ private Type sentenceType;
+
+ private CAS cas = null;
+
+ private Logger logger;
+
+ private static List<String> punctuations = Arrays.asList(new String[] { ".",
+ "!", "?" });
+
+ public static final String MESSAGE_BUNDLE = "org.apache.uima.annotator.whitespaceTokenizerMessages";
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.uima.analysis_engine.annotator.JTextAnnotator#process(org.apache.uima.jcas.impl.JCas,
+ * org.apache.uima.analysis_engine.ResultSpecification)
+ */
+ public void process(JCas aJCas, ResultSpecification aResultSpec)
+ throws AnnotatorProcessException {
+
+ this.logger.logrb(Level.INFO, "WhitespaceTokenizer", "process",
+ MESSAGE_BUNDLE, "whitespace_tokenizer_info_start_processing");
+
+ // get text content from the CAS
+ char[] textContent = aJCas.getCas().getDocumentText().toCharArray();
+ this.cas = aJCas.getCas();
+ int tokenStart = UNDEFINED;
+ int currentCharPos = 0;
+ int sentenceStart = 0;
+ int nextCharType = UNDEFINED;
+ char nextChar = INVALID_CHAR;
+
+ while (currentCharPos < textContent.length) {
+ char currentChar = textContent[currentCharPos];
+ int currentCharType = getCharacterType(currentChar);
+
+ // get character class for current and next character
+ if ((currentCharPos + 1) < textContent.length) {
+ nextChar = textContent[currentCharPos + 1];
+ nextCharType = getCharacterType(nextChar);
+ } else {
+ nextCharType = UNDEFINED;
+ nextChar = INVALID_CHAR;
+ }
+
+ // check if current character is a letter or number
+ if (currentCharType == CH_LETTER || currentCharType == CH_NUMBER) {
+
+ // check if it is the first letter of a token
+ if (tokenStart == UNDEFINED) {
+ // start new token here
+ tokenStart = currentCharPos;
+ }
+ }
+
+ // check if current character is a whitespace character
+ else if (currentCharType == CH_WHITESPACE) {
+
+ // terminate current token
+ if (tokenStart != UNDEFINED) {
+ // end of current word
+ createAnnotation(this.tokenType, tokenStart, currentCharPos);
+ tokenStart = UNDEFINED;
+ }
+ }
+
+ // check if current character is a special character
+ else if (currentCharType == CH_SPECIAL) {
+
+ // terminate current token
+ if (tokenStart != UNDEFINED) {
+ // end of current word
+ createAnnotation(this.tokenType, tokenStart, currentCharPos);
+ tokenStart = UNDEFINED;
+ }
+
+ // create token for special character
+ createAnnotation(this.tokenType, currentCharPos, currentCharPos + 1);
+ }
+
+ // check if current character is new line character
+ else if (currentCharType == CH_NEWLINE) {
+ // terminate current token
+ if (tokenStart != UNDEFINED) {
+ // end of current word
+ createAnnotation(this.tokenType, tokenStart, currentCharPos);
+ tokenStart = UNDEFINED;
+ }
+ }
+
+ // check if current character is new punctuation character
+ else if (currentCharType == CH_PUNCTUATION) {
+
+ // terminates the current token
+ if (tokenStart != UNDEFINED) {
+ createAnnotation(this.tokenType, tokenStart, currentCharPos);
+ tokenStart = UNDEFINED;
+ }
+
+ // check next token type so see if we have a sentence end
+ if (((nextCharType == CH_WHITESPACE) || (nextCharType == CH_NEWLINE))
+ && (punctuations.contains(new String(
+ new char[] { currentChar })))) {
+ // terminate sentence
+ createAnnotation(this.sentenceType, sentenceStart,
+ currentCharPos + 1);
+ sentenceStart = currentCharPos + 1;
+ }
+ // create token for punctuation character
+ createAnnotation(this.tokenType, currentCharPos, currentCharPos + 1);
+ }
+ // go to the next token
+ currentCharPos++;
+ } // end of character loop
+
+ // we are at the end of the text terminate open token annotations
+ if (tokenStart != UNDEFINED) {
+ // end of current word
+ createAnnotation(this.tokenType, tokenStart, currentCharPos);
+ tokenStart = UNDEFINED;
}
- // check if current character is a letter or number
- if (currentCharType == CH_LETTER || currentCharType == CH_NUMBER) {
-
- // check if it is the first letter of a token
- if (tokenStart == UNDEFINED) {
- // start new token here
- tokenStart = currentCharPos;
- }
- }
-
- // check if current character is a whitespace character
- else if (currentCharType == CH_WHITESPACE) {
-
- // terminate current token
- if (tokenStart != UNDEFINED) {
- // end of current word
- createAnnotation(this.tokenType, tokenStart, currentCharPos);
- tokenStart = UNDEFINED;
- }
- }
-
- // check if current character is a special character
- else if (currentCharType == CH_SPECIAL) {
-
- // terminate current token
- if (tokenStart != UNDEFINED) {
- // end of current word
- createAnnotation(this.tokenType, tokenStart, currentCharPos);
- tokenStart = UNDEFINED;
- }
-
- // create token for special character
- createAnnotation(this.tokenType, currentCharPos, currentCharPos + 1);
- }
-
- // check if current character is new line character
- else if (currentCharType == CH_NEWLINE) {
- // terminate current token
- if (tokenStart != UNDEFINED) {
- // end of current word
- createAnnotation(this.tokenType, tokenStart, currentCharPos);
- tokenStart = UNDEFINED;
- }
+ // we are at the end of the text terminate open sentence annotations
+ if (sentenceStart != UNDEFINED) {
+ // end of current word
+ createAnnotation(this.sentenceType, sentenceStart, currentCharPos);
+ sentenceStart = UNDEFINED;
}
- // check if current character is new punctuation character
- else if (currentCharType == CH_PUNCTUATION) {
+ this.logger.logrb(Level.INFO, "WhitespaceTokenizer", "process",
+ MESSAGE_BUNDLE, "whitespace_tokenizer_info_stop_processing");
+ }
+
+ /**
+ * create an annotation of the given type in the CAS using startPos and
+ * endPos.
+ *
+ * @param annotationType
+ * annotation type
+ * @param startPos
+ * annotation start position
+ * @param endPos
+ * annotation end position
+ */
+ private void createAnnotation(Type annotationType, int startPos, int endPos) {
+
+ AnnotationFS annot = this.cas.createAnnotation(annotationType, startPos,
+ endPos);
+ this.cas.addFsToIndexes(annot);
+ }
+
+ /**
+ * returns the character type of the given character. Possible character
+ * classes are: CH_LETTER for all letters CH_NUMBER for all numbers
+ * CH_WHITESPACE for all whitespace characters CH_PUNCTUATUATION for all
+ * punctuation characters CH_NEWLINE for all new line characters CH_SPECIAL
+ * for all other characters that are not in any of the groups above
+ *
+ * @param character
+ * aCharacter
+ *
+ * @return returns the character type of the given character
+ */
+ private static int getCharacterType(char character) {
- // terminates the current token
- if (tokenStart != UNDEFINED) {
- createAnnotation(this.tokenType, tokenStart, currentCharPos);
- tokenStart = UNDEFINED;
- }
-
- // check next token type so see if we have a sentence end
- if (((nextCharType == CH_WHITESPACE) || (nextCharType == CH_NEWLINE))
- && (punctuations.contains(new String(new char[] { currentChar })))) {
- // terminate sentence
- createAnnotation(this.sentenceType, sentenceStart, currentCharPos + 1);
- sentenceStart = currentCharPos + 1;
- }
- // create token for punctuation character
- createAnnotation(this.tokenType, currentCharPos, currentCharPos + 1);
- }
- // go to the next token
- currentCharPos++;
- } //end of character loop
-
- // we are at the end of the text terminate open token annotations
- if (tokenStart != UNDEFINED) {
- // end of current word
- createAnnotation(this.tokenType, tokenStart, currentCharPos);
- tokenStart = UNDEFINED;
- }
-
- // we are at the end of the text terminate open sentence annotations
- if (sentenceStart != UNDEFINED) {
- // end of current word
- createAnnotation(this.sentenceType, sentenceStart, currentCharPos);
- sentenceStart = UNDEFINED;
- }
-
- this.logger.log(Level.INFO, "Whitespace tokenizer finished processing");
- }
-
- /**
- * create an annotation of the given type in the CAS using startPos and endPos.
- *
- * @param annotationType annotation type
- * @param startPos annotation start position
- * @param endPos annotation end position
- */
- private void createAnnotation(Type annotationType, int startPos, int endPos) {
-
- AnnotationFS annot = this.cas.createAnnotation(annotationType, startPos, endPos);
- this.cas.addFsToIndexes(annot);
- }
-
- /**
- * returns the character type of the given character. Possible character classes are:
- * CH_LETTER for all letters
- * CH_NUMBER for all numbers
- * CH_WHITESPACE for all whitespace characters
- * CH_PUNCTUATUATION for all punctuation characters
- * CH_NEWLINE for all new line characters
- * CH_SPECIAL for all other characters that are not in any of the groups above
- *
- * @param character aCharacter
- *
- * @return returns the character type of the given character
- */
- private static int getCharacterType(char character) {
-
- switch (Character.getType(character)) {
+ switch (Character.getType(character)) {
// letter characters
case Character.UPPERCASE_LETTER:
@@ -228,70 +241,77 @@
case Character.PRIVATE_USE:
case Character.SURROGATE:
case Character.MODIFIER_SYMBOL:
- return CH_LETTER;
+ return CH_LETTER;
- // number characters
+ // number characters
case Character.DECIMAL_DIGIT_NUMBER:
case Character.LETTER_NUMBER:
case Character.OTHER_NUMBER:
- return CH_NUMBER;
+ return CH_NUMBER;
- // whitespace characters
+ // whitespace characters
case Character.SPACE_SEPARATOR:
case Character.CONNECTOR_PUNCTUATION:
- return CH_WHITESPACE;
+ return CH_WHITESPACE;
case Character.DASH_PUNCTUATION:
case Character.START_PUNCTUATION:
case Character.END_PUNCTUATION:
case Character.OTHER_PUNCTUATION:
- return CH_PUNCTUATION;
+ return CH_PUNCTUATION;
case Character.LINE_SEPARATOR:
case Character.PARAGRAPH_SEPARATOR:
- return CH_NEWLINE;
+ return CH_NEWLINE;
case Character.CONTROL:
- if (character == '\n' || character == '\r') {
- return CH_NEWLINE;
- } else {
- return CH_SPECIAL;
- }
+ if (character == '\n' || character == '\r') {
+ return CH_NEWLINE;
+ } else {
+ return CH_SPECIAL;
+ }
default:
- return CH_SPECIAL;
- }
+ return CH_SPECIAL;
+ }
- }
+ }
- /* (non-Javadoc)
- * @see org.apache.uima.analysis_engine.annotator.Annotator_ImplBase#initialize(org.apache.uima.analysis_engine.annotator.AnnotatorContext)
- */
- public void initialize(AnnotatorContext aContext) throws AnnotatorInitializationException,
- AnnotatorConfigurationException {
-
- // initialize logger
- try {
- this.logger = aContext.getLogger();
- } catch (AnnotatorContextException ex) {
- throw new AnnotatorInitializationException(ex);
- }
-
- this.logger.log(Level.INFO, "Whitespace tokenizer successfully initialized");
- }
-
- /* (non-Javadoc)
- * @see org.apache.uima.analysis_engine.annotator.Annotator_ImplBase#typeSystemInit(org.apache.uima.cas.TypeSystem)
- */
- public void typeSystemInit(TypeSystem aTypeSystem) throws AnnotatorInitializationException,
- AnnotatorConfigurationException {
-
- // initialize cas token type
- this.tokenType = aTypeSystem.getType(TOKEN_ANNOTATION_NAME);
-
- this.sentenceType = aTypeSystem.getType(SENTENCE_ANNOTATION_NAME);
-
- this.logger.log(Level.INFO, "Whitespace tokenizer typesystem initialized");
- }
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.uima.analysis_engine.annotator.Annotator_ImplBase#initialize(org.apache.uima.analysis_engine.annotator.AnnotatorContext)
+ */
+ public void initialize(AnnotatorContext aContext)
+ throws AnnotatorInitializationException,
+ AnnotatorConfigurationException {
+
+ // initialize logger
+ try {
+ this.logger = aContext.getLogger();
+ } catch (AnnotatorContextException ex) {
+ throw new AnnotatorInitializationException(ex);
+ }
+ this.logger.logrb(Level.INFO, "WhitespaceTokenizer", "initialize",
+ MESSAGE_BUNDLE, "whitespace_tokenizer_info_initialized");
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.uima.analysis_engine.annotator.Annotator_ImplBase#typeSystemInit(org.apache.uima.cas.TypeSystem)
+ */
+ public void typeSystemInit(TypeSystem aTypeSystem)
+ throws AnnotatorInitializationException,
+ AnnotatorConfigurationException {
+
+ // initialize cas token type
+ this.tokenType = aTypeSystem.getType(TOKEN_ANNOTATION_NAME);
+
+ this.sentenceType = aTypeSystem.getType(SENTENCE_ANNOTATION_NAME);
+
+ this.logger.logrb(Level.INFO, "WhitespaceTokenizer", "typeSystemInit",
+ MESSAGE_BUNDLE, "whitespace_tokenizer_info_typesystem_initialized");
+ }
}
Added: incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/uima/annotator/whitespaceTokenizerMessages.properties
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/uima/annotator/whitespaceTokenizerMessages.properties?rev=597367&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/uima/annotator/whitespaceTokenizerMessages.properties (added)
+++ incubator/uima/sandbox/trunk/WhitespaceTokenizer/src/main/resources/org/apache/uima/annotator/whitespaceTokenizerMessages.properties Thu Nov 22 02:39:28 2007
@@ -0,0 +1,23 @@
+# ***************************************************************
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# ***************************************************************
+
+whitespace_tokenizer_info_start_processing = "Whitespace tokenizer starts processing"
+whitespace_tokenizer_info_stop_processing = "Whitespace tokenizer finished processing"
+whitespace_tokenizer_info_initialized = "Whitespace tokenizer successfully initialized"
+whitespace_tokenizer_info_typesystem_initialized = "Whitespace tokenizer typesystem initialized"