You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by pk...@apache.org on 2012/12/19 14:32:23 UTC
svn commit: r1423857 - in /uima/sandbox/TextMarker/trunk:
uima-docbook-textmarker/src/docbook/
uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/
uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/
uimaj-textmarker/src...
Author: pkluegl
Date: Wed Dec 19 13:32:21 2012
New Revision: 1423857
URL: http://svn.apache.org/viewvc?rev=1423857&view=rev
Log:
UIMA-2524
- refactored the HTMLAnnotator
- removed functionality for stripping html tags
- updated documentation
- finished test
Added:
uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlAnnotator.java
- copied, changed from r1423467, uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlAnnotator.java
uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlVisitor.java
- copied, changed from r1423467, uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlVisitor.java
Removed:
uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/
Modified:
uima/sandbox/TextMarker/trunk/uima-docbook-textmarker/src/docbook/tools.textmarker.overview.xml
uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/resources/org/apache/uima/textmarker/engine/HtmlAnnotator.xml
uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/test/java/org/apache/uima/textmarker/engine/HtmlAnnotatorTest.java
Modified: uima/sandbox/TextMarker/trunk/uima-docbook-textmarker/src/docbook/tools.textmarker.overview.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/TextMarker/trunk/uima-docbook-textmarker/src/docbook/tools.textmarker.overview.xml?rev=1423857&r1=1423856&r2=1423857&view=diff
==============================================================================
--- uima/sandbox/TextMarker/trunk/uima-docbook-textmarker/src/docbook/tools.textmarker.overview.xml (original)
+++ uima/sandbox/TextMarker/trunk/uima-docbook-textmarker/src/docbook/tools.textmarker.overview.xml Wed Dec 19 13:32:21 2012
@@ -1058,26 +1058,13 @@ ae.process(cas);]]></programlisting>
for each HTML element spanning the content of the element, whereas the most common elements are represented by own types.
The document <quote><![CDATA[This text is <b>bold</b>.]]></quote>, for example, would be annotated with an annotation of the type
<quote>org.apache.uima.textmarker.type.html.B</quote> for the word <quote>bold</quote>. The HTML annotator can be configured
- in order to include the start and end element in the created annotations. Additionally, the Analysis Engine is also able to strip the HTML element,
- but retraining the HTML annotations. Thereby, an HTML document can be converted to a plain text document, which contains the annotations about the HTML layout.
+ in order to include the start and end element in the created annotations.
A descriptor file for this Analysis Engine is located in the folder <quote>descriptor/utils</quote> of a TextMarker project.
</para>
<section id="ugr.tools.tm.ae.html.parameter">
<title>Configuration Parameters</title>
<para>
</para>
- <section id="ugr.tools.tm.ae.html.parameter.plainTextOutput">
- <title>plainTextOutput</title>
- <para>
- This parameter specifies whether a new document without the HTML elements should be created. The default value is <quote>false</quote>.
- </para>
- </section>
- <section id="ugr.tools.tm.ae.html.parameter.outputViewName">
- <title>outputViewName</title>
- <para>
- This parameter specifies in which view the optional new document without HTML element should be stored.
- </para>
- </section>
<section id="ugr.tools.tm.ae.html.parameter.onlyContent">
<title>onlyContent</title>
<para>
Copied: uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlAnnotator.java (from r1423467, uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlAnnotator.java)
URL: http://svn.apache.org/viewvc/uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlAnnotator.java?p2=uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlAnnotator.java&p1=uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlAnnotator.java&r1=1423467&r2=1423857&rev=1423857&view=diff
==============================================================================
--- uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlAnnotator.java (original)
+++ uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlAnnotator.java Wed Dec 19 13:32:21 2012
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.uima.textmarker.engine.html;
+package org.apache.uima.textmarker.engine;
import java.util.ArrayList;
import java.util.List;
@@ -25,12 +25,8 @@ import java.util.List;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_component.JCasAnnotator_ImplBase;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.CASException;
-import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.jcas.JCas;
-import org.apache.uima.jcas.cas.StringArray;
import org.apache.uima.resource.ResourceInitializationException;
import org.htmlparser.Parser;
import org.htmlparser.util.NodeList;
@@ -39,100 +35,41 @@ import org.htmlparser.util.ParserExcepti
public class HtmlAnnotator extends JCasAnnotator_ImplBase {
public static final String NAMESPACE = "org.apache.uima.textmarker.type.html.";
- public static final String PLAIN_TEXT_OUTPUT = "plainTextOutput";
-
- public static final String OUTPUT_VIEW_NAME = "outputViewName";
-
public static final String ONLY_CONTENT = "onlyContent";
- private Boolean plainTextOutput;
-
- private String outputViewName;
-
private Boolean onlyContent;
@Override
public void initialize(UimaContext aContext) throws ResourceInitializationException {
super.initialize(aContext);
- plainTextOutput = (Boolean) aContext.getConfigParameterValue(PLAIN_TEXT_OUTPUT);
- outputViewName = (String) aContext.getConfigParameterValue(OUTPUT_VIEW_NAME);
onlyContent = (Boolean) aContext.getConfigParameterValue(ONLY_CONTENT);
- plainTextOutput = plainTextOutput == null ? false : plainTextOutput;
onlyContent = onlyContent == null ? true : onlyContent;
}
@Override
public void process(JCas jcas) throws AnalysisEngineProcessException {
- CAS cas = jcas.getCas();
- JCas outJCas = jcas;
- CAS outCas = cas;
-
- String documentText = cas.getDocumentText();
- String plainTextDocument = documentText;
- List<HtmlAnnotation> annotationList = new ArrayList<HtmlAnnotation>();
- List<HtmlRemark> remarks = new ArrayList<HtmlRemark>();
-
+ String documentText = jcas.getDocumentText();
+ List<AnnotationFS> annotations = new ArrayList<AnnotationFS>();
+ List<AnnotationFS> annotationStack = new ArrayList<AnnotationFS>();
try {
Parser parser = new Parser(documentText);
NodeList list = parser.parse(null);
- HtmlVisitor visitor = new HtmlVisitor(jcas, plainTextOutput, onlyContent);
+ HtmlVisitor visitor = new HtmlVisitor(jcas, onlyContent);
list.visitAllNodesWith(visitor);
- plainTextDocument = visitor.getPlainTextDocument();
- remarks = visitor.getRemarks();
- annotationList = visitor.getAnnotationList();
+ annotations = visitor.getAnnotations();
+ annotationStack = visitor.getAnnotationStack();
} catch (ParserException e) {
throw new AnalysisEngineProcessException(e);
}
-
- if (plainTextOutput) {
- outCas = cas.createView(outputViewName);
- try {
- outJCas = outCas.getJCas();
- } catch (CASException e) {
- throw new AnalysisEngineProcessException(e);
+ for (AnnotationFS each : annotations) {
+ if (each.getBegin() < each.getEnd()) {
+ jcas.addFsToIndexes(each);
}
- outCas.setDocumentText(plainTextDocument);
}
-
- for (int k = 0; k < remarks.size(); k++) {
- HtmlRemark remark = remarks.get(k);
- AnnotationFS annotation;
- Type type = outCas.getTypeSystem().getType(NAMESPACE + "REMARK");
- int begin = remark.getAnnotation().getBegin();
- int end = remark.getAnnotation().getEnd() - remark.getDelta();
- annotation = outCas.createAnnotation(type, begin, end);
- annotation.setFeatureValueFromString(annotation.getType().getFeatureByBaseName("comment"),
- remark.getComment());
- outCas.addFsToIndexes(annotation);
- }
- for (int i = 0; i < annotationList.size(); i++) {
- AnnotationFS annotation;
- HtmlAnnotation annoDelta = annotationList.get(i);
- int begin = annoDelta.getAnnotation().getBegin();
- int end = annoDelta.getAnnotation().getEnd() - annoDelta.getDelta();
-
- StringArray attributeName = new StringArray(outJCas, annoDelta.getAttributeName().size());
- for (int j = 0; j < attributeName.size(); j++) {
- attributeName.set(j, annoDelta.getAttributeName().get(j));
- }
- StringArray attributeValue = new StringArray(outJCas, annoDelta.getAttributeValue().size());
- for (int k = 0; k < attributeName.size(); k++) {
- attributeValue.set(k, annoDelta.getAttributeValue().get(k));
+ for (AnnotationFS each : annotationStack) {
+ if (each.getBegin() < each.getEnd()) {
+ jcas.addFsToIndexes(each);
}
- if(begin == 0 && end == 0) {
- begin = 0;
- end = outCas.getDocumentAnnotation().getEnd();
- }
- annotation = outCas.createAnnotation(annoDelta.getAnnotation().getType(), begin, end);
-
- annotation.setFeatureValue(annotation.getType().getFeatureByBaseName("attributeName"),
- attributeName);
- annotation.setFeatureValue(annotation.getType().getFeatureByBaseName("attributeValue"),
- attributeValue);
-
- annotation.setFeatureValueFromString(annotation.getType().getFeatureByBaseName("name"),
- annoDelta.getName());
- outCas.addFsToIndexes(annotation);
}
}
Copied: uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlVisitor.java (from r1423467, uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlVisitor.java)
URL: http://svn.apache.org/viewvc/uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlVisitor.java?p2=uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlVisitor.java&p1=uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlVisitor.java&r1=1423467&r2=1423857&rev=1423857&view=diff
==============================================================================
--- uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/html/HtmlVisitor.java (original)
+++ uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/java/org/apache/uima/textmarker/engine/HtmlVisitor.java Wed Dec 19 13:32:21 2012
@@ -17,229 +17,119 @@
* under the License.
*/
-package org.apache.uima.textmarker.engine.html;
+package org.apache.uima.textmarker.engine;
import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.CASException;
+import org.apache.uima.cas.Feature;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.cas.StringArray;
import org.htmlparser.Attribute;
+import org.htmlparser.Node;
import org.htmlparser.Remark;
import org.htmlparser.Tag;
import org.htmlparser.visitors.NodeVisitor;
public class HtmlVisitor extends NodeVisitor {
- private int delta;
-
private CAS cas;
- private boolean body = false;
-
- private List<HtmlAnnotation> annotationList = new ArrayList<HtmlAnnotation>();
-
- private List<HtmlAnnotation> annotations = new ArrayList<HtmlAnnotation>();
+ private List<AnnotationFS> annotations = new ArrayList<AnnotationFS>();
- private List<HtmlRemark> remarks = new ArrayList<HtmlRemark>();
-
- private String plainTextDocument;
+ private LinkedList<AnnotationFS> annotationStack = new LinkedList<AnnotationFS>();
private JCas jcas;
- private boolean stripHtml;
-
private boolean onlyContent;
- public HtmlVisitor(JCas jcas, boolean stripHtml, boolean onlyContent) {
+ public HtmlVisitor(JCas jcas, boolean onlyContent) {
super();
this.jcas = jcas;
this.cas = jcas.getCas();
- this.stripHtml = stripHtml;
this.onlyContent = onlyContent;
- plainTextDocument = cas.getDocumentText();
}
public void visitTag(Tag tag) {
- String name = tag.getRawTagName().toUpperCase();
- if (body == true) {
- tag2annotation(tag, name, cas);
- } else if (name.equals("BODY")) {
- body = true;
- setDelta(tag, name, cas);
- } else if (name.equals("HTML")) {
- tag.setStartPosition(0);
- tag2annotation(tag, name, cas);
+ String name = getName(tag);
+ Type type = getType(name, cas);
+ boolean tagClosed = isTagClosed(tag);
+ boolean tagStillOpen = false;
+ int begin = getBeginOffset(tag);
+ int end = begin;
+ if (tagClosed) {
+ end = getEndOffset(tag);
} else {
- head2annotation(tag, name, cas);
+ Tag endTag = tag.getEndTag();
+ if (endTag != null) {
+ end = getEndOffset(endTag);
+ } else {
+ end = getEndOffset(tag);
+ tagStillOpen = true;
+ }
}
- }
-
- public void visitRemarkNode(Remark node) {
- if (body == true) {
- node2annotation(node);
+ AnnotationFS annotation = cas.createAnnotation(type, begin, end);
+ processAttributes(annotation, tag);
+ Feature nameFeature = annotation.getType().getFeatureByBaseName("name");
+ annotation.setStringValue(nameFeature, name);
+ if (tagStillOpen) {
+ annotationStack.add(annotation);
} else {
- headNode2annotation(node);
+ annotations.add(annotation);
}
}
- public void visitEndTag(Tag tag) {
- if (body == true) {
- endTag(tag);
- }
+ public void visitRemarkNode(Remark node) {
+ Type type = getType("REMARK", cas);
+ int begin = getBeginOffset(node);
+ int end = getEndOffset(node);
+ AnnotationFS annotation = cas.createAnnotation(type, begin, end);
+ Feature feature = type.getFeatureByBaseName("comment");
+ annotation.setStringValue(feature, node.getText());
+ annotations.add(annotation);
}
- public void tag2annotation(Tag tag, String name, CAS cas) {
- Type type = getType(name, cas);
- if(name.endsWith("/")) {
- name = name.substring(0, name.length()-1);
- }
- if (name.equals("STYLE")) {
- tag.setEndPosition(tag.getEndTag().getStartPosition());
- }
- int begin = getOffset(tag.getStartPosition());
-
- delta += tag.getEndPosition() - tag.getStartPosition();
- int i;
- int end = tag.getEndPosition();
- if (tag.getEndTag() != null) {
- i = tag.getEndTag().getStartPosition();
- end = tag.getEndTag().getEndPosition();
- } else {
- // TODO fix this
- i = tag.getEndPosition();
- tag.toString();
- String startTag = "<" + tag.getRawTagName();
- String endTag = "</" + tag.getRawTagName() + ">";
- int j = i - 1;
- int k = i - 2;
- while (k < j && k > -1) {
- k = cas.getDocumentText().indexOf(startTag, k + 1);
- j = cas.getDocumentText().indexOf(endTag, j + 1);
- }
- if (i < j) {
- i = j;
- }
-
- }
-
- if (onlyContent) {
- begin = getOffset(tag.getEndPosition());
- end = i;
- }
- AnnotationFS annotation = cas.createAnnotation(type, begin, end);
- for (int k = 0; k < tag.getAttributesEx().size(); k++) {
- String test = tag.getAttributesEx().elementAt(k).toString();
- if (test.startsWith(" ") | test.startsWith("/")) {
- tag.getAttributesEx().remove(k);
- k--;
+ public void visitEndTag(Tag tag) {
+ String name = getName(tag);
+ Iterator<AnnotationFS> descendingIterator = annotationStack.descendingIterator();
+ AnnotationFS found = null;
+ while (descendingIterator.hasNext()) {
+ AnnotationFS each = (AnnotationFS) descendingIterator.next();
+ Feature nameFeature = each.getType().getFeatureByBaseName("name");
+ String eachName = each.getStringValue(nameFeature);
+ if (name.equals(eachName)) {
+ int endOffset = getEndOffset(tag);
+ Feature endFeature = each.getType().getFeatureByBaseName("end");
+ each.setIntValue(endFeature, endOffset);
+ found = each;
+ break;
}
}
- if (name.equals("STYLE") || name.equals("TITLE") || name.equals("SCRIPT")) {
- annotation.setFeatureValueFromString(annotation.getType().getFeatureByBaseName("name"),
- tag.getText());
- } else {
- annotation.setFeatureValueFromString(annotation.getType().getFeatureByBaseName("name"), name);
- }
-
- StringArray attributeName;
- StringArray attributeValue;
- int size = tag.getAttributesEx().size() - 1;
- attributeName = new StringArray(jcas, size);
- attributeValue = new StringArray(jcas, size);
- for (int j = 0; j < size; j++) {
- Attribute attribute = (Attribute) tag.getAttributesEx().elementAt(j + 1);
- attributeName.set(j, attribute.getName());
- attributeValue.set(j, attribute.getValue());
- }
- if (i <= tag.getEndPosition()) {
- HtmlAnnotation annotationDelta = new HtmlAnnotation(annotation, delta, attributeName,
- attributeValue, name, stripHtml);
- annotationList.add(annotationDelta);
- } else {
- HtmlAnnotation annotationDelta = new HtmlAnnotation(annotation, delta, attributeName,
- attributeValue, name, stripHtml);
- annotations.add(annotationDelta);
+ if (found != null) {
+ annotationStack.remove(found);
+ annotations.add(found);
}
- }
- private int getOffset(int offset) {
- if (stripHtml) {
- return offset - delta;
- }
- return offset;
}
- public void setDelta(Tag tag, String name, CAS cas) {
- delta = tag.getStartPosition();
- plainTextDocument = plainTextDocument.substring(delta);
- tag2annotation(tag, name, cas);
+ private boolean isTagClosed(Tag tag) {
+ return tag.getRawTagName().endsWith("/");
}
- public void head2annotation(Tag tag, String name, CAS cas) {
- Type type = getType(name, cas);
- if (name.equals("STYLE")) {
- tag.setEndPosition(tag.getEndTag().getStartPosition());
- }
- int begin = getOffset(tag.getStartPosition());
-
- delta += tag.getEndPosition() - tag.getStartPosition();
- int i;
- int end = tag.getEndPosition();
- if (tag.getEndTag() != null) {
- i = tag.getEndTag().getStartPosition();
- end = tag.getEndTag().getEndPosition();
- } else {
- // TODO fix this
- i = tag.getEndPosition();
- String startTag = "<" + tag.getRawTagName();
- String endTag = "</" + tag.getRawTagName() + ">";
- int j = i - 1;
- int k = i - 2;
- while (k < j && k > -1) {
- k = cas.getDocumentText().indexOf(startTag, k + 1);
- j = cas.getDocumentText().indexOf(endTag, j + 1);
- }
- if (i < j) {
- i = j;
- }
-
+ private String getName(Tag tag) {
+ String result = tag.getRawTagName().toUpperCase();
+ if (result.endsWith("/")) {
+ result = result.substring(0, result.length() - 1);
}
-
- if (onlyContent) {
- begin = getOffset(tag.getEndPosition());
- end = i;
- }
-
- AnnotationFS annotation = cas.createAnnotation(type, begin, end);
- for (int k = 0; k < tag.getAttributesEx().size(); k++) {
- String test = tag.getAttributesEx().elementAt(k).toString();
- if (test.startsWith(" ") | test.startsWith("/")) {
- tag.getAttributesEx().remove(k);
- k--;
- }
+ if (result.startsWith("/")) {
+ result = result.substring(1);
}
- if (tag.getEndTag() != null && !name.equals("HEAD")) {
- name = plainTextDocument.substring(tag.getEndPosition(), tag.getEndTag().getStartPosition());
- }
- StringArray attributeName;
- StringArray attributeValue;
- int size = tag.getAttributesEx().size() - 1;
- attributeName = new StringArray(jcas, size);
- attributeValue = new StringArray(jcas, size);
- for (int j = 0; j < size; j++) {
- Attribute attribute = (Attribute) tag.getAttributesEx().elementAt(j + 1);
- attributeName.set(j, attribute.getName());
- attributeValue.set(j, attribute.getValue());
- }
- HtmlAnnotation annotationDelta = new HtmlAnnotation(annotation, 0, attributeName,
- attributeValue, name, stripHtml);
- annotationList.add(annotationDelta);
-
+ return result;
}
private Type getType(String name, CAS cas) {
@@ -251,70 +141,261 @@ public class HtmlVisitor extends NodeVis
return type;
}
- public void node2annotation(Remark tag) {
- Type type = getType("REMARK", cas);
- int begin = getOffset(tag.getStartPosition());
-
- delta += tag.getEndPosition() - tag.getStartPosition();
- int end = tag.getEndPosition();
-
+ private int getBeginOffset(Node tag) {
if (onlyContent) {
- begin = getOffset(tag.getEndPosition());
+ return tag.getEndPosition();
+ } else {
+ return tag.getStartPosition();
}
- AnnotationFS annotation = cas.createAnnotation(type, begin, end);
- HtmlRemark remark = new HtmlRemark(annotation, 0, stripHtml);
- String comment = tag.getText();
- remark.setComment(comment);
- remarks.add(remark);
- }
- public void headNode2annotation(Remark tag) {
- Type type = getType("REMARK", cas);
- int begin = getOffset(tag.getStartPosition());
+ }
- delta += tag.getEndPosition() - tag.getStartPosition();
- int end = tag.getEndPosition();
+ private int getEndOffset(Node tag) {
if (onlyContent) {
- begin = getOffset(tag.getEndPosition());
- }
-
- AnnotationFS annotation = cas.createAnnotation(type, begin, end);
-
- HtmlRemark remark = new HtmlRemark(annotation, 0, stripHtml);
- String comment = tag.getText();
- remark.setComment(comment);
- remarks.add(remark);
- }
-
- public void endTag(Tag endtag) {
- int size = annotations.size();
- if (size > 0) {
- HtmlAnnotation annotationDelta = annotations.get(size - 1);
- String test = endtag.getRawTagName().toUpperCase().substring(1);
-
- if (annotationDelta
- .getAnnotation()
- .getFeatureValueAsString(
- annotationDelta.getAnnotation().getType().getFeatureByBaseName("name"))
- .equals(test)) {
- annotationDelta.changeDelta(delta);
- annotationList.add(annotationDelta);
- annotations.remove(size - 1);
- }
+ return tag.getStartPosition();
+ } else {
+ return tag.getEndPosition();
}
- delta += endtag.getEndPosition() - endtag.getStartPosition();
- }
-
- public String getPlainTextDocument() {
- return plainTextDocument;
}
- public List<HtmlAnnotation> getAnnotationList() {
- return annotationList;
- }
-
- public List<HtmlRemark> getRemarks() {
- return remarks;
- }
+ private void processAttributes(AnnotationFS annotation, Tag tag) {
+ int size = tag.getAttributesEx().size() - 1;
+ StringArray attributeName = new StringArray(jcas, size);
+ StringArray attributeValue = new StringArray(jcas, size);
+ for (int i = 0; i < size; i++) {
+ Attribute attribute = (Attribute) tag.getAttributesEx().elementAt(i + 1);
+ attributeName.set(i, attribute.getName());
+ attributeValue.set(i, attribute.getValue());
+ }
+ Feature feature1 = annotation.getType().getFeatureByBaseName("attributeName");
+ annotation.setFeatureValue(feature1, attributeName);
+ Feature feature2 = annotation.getType().getFeatureByBaseName("attributeValue");
+ annotation.setFeatureValue(feature2, attributeValue);
+ }
+
+ public List<AnnotationFS> getAnnotations() {
+ return annotations;
+ }
+
+ public LinkedList<AnnotationFS> getAnnotationStack() {
+ return annotationStack;
+ }
+
+ //
+ //
+ // public void tag2annotation(Tag tag, String name, CAS cas) {
+ // Type type = getType(name, cas);
+ // if (name.endsWith("/")) {
+ // name = name.substring(0, name.length() - 1);
+ // }
+ // if (name.equals("STYLE")) {
+ // tag.setEndPosition(tag.getEndTag().getStartPosition());
+ // }
+ // int begin = getOffset(tag.getStartPosition());
+ //
+ // delta += tag.getEndPosition() - tag.getStartPosition();
+ // int i;
+ // int end = tag.getEndPosition();
+ // if (tag.getEndTag() != null) {
+ // i = tag.getEndTag().getStartPosition();
+ // end = tag.getEndTag().getEndPosition();
+ // } else {
+ // // TODO fix this
+ // i = tag.getEndPosition();
+ // tag.toString();
+ // String startTag = "<" + tag.getRawTagName();
+ // String endTag = "</" + tag.getRawTagName() + ">";
+ // int j = i - 1;
+ // int k = i - 2;
+ // while (k < j && k > -1) {
+ // k = cas.getDocumentText().indexOf(startTag, k + 1);
+ // j = cas.getDocumentText().indexOf(endTag, j + 1);
+ // }
+ // if (i < j) {
+ // i = j;
+ // }
+ //
+ // }
+ //
+ // if (onlyContent) {
+ // begin = getOffset(tag.getEndPosition());
+ // end = i;
+ // }
+ // AnnotationFS annotation = cas.createAnnotation(type, begin, end);
+ // for (int k = 0; k < tag.getAttributesEx().size(); k++) {
+ // String test = tag.getAttributesEx().elementAt(k).toString();
+ // if (test.startsWith(" ") | test.startsWith("/")) {
+ // tag.getAttributesEx().remove(k);
+ // k--;
+ // }
+ // }
+ // if (name.equals("STYLE") || name.equals("TITLE") || name.equals("SCRIPT")) {
+ // annotation.setFeatureValueFromString(annotation.getType().getFeatureByBaseName("name"),
+ // tag.getText());
+ // } else {
+ // annotation.setFeatureValueFromString(annotation.getType().getFeatureByBaseName("name"), name);
+ // }
+ //
+ // StringArray attributeName;
+ // StringArray attributeValue;
+ // int size = tag.getAttributesEx().size() - 1;
+ // attributeName = new StringArray(jcas, size);
+ // attributeValue = new StringArray(jcas, size);
+ // for (int j = 0; j < size; j++) {
+ // Attribute attribute = (Attribute) tag.getAttributesEx().elementAt(j + 1);
+ // attributeName.set(j, attribute.getName());
+ // attributeValue.set(j, attribute.getValue());
+ // }
+ // if (i <= tag.getEndPosition()) {
+ // HtmlAnnotation annotationDelta = new HtmlAnnotation(annotation, delta, attributeName,
+ // attributeValue, name, stripHtml);
+ // annotationList.add(annotationDelta);
+ // } else {
+ // HtmlAnnotation annotationDelta = new HtmlAnnotation(annotation, delta, attributeName,
+ // attributeValue, name, stripHtml);
+ // annotations.add(annotationDelta);
+ // }
+ // }
+ //
+ // private int getOffset(int offset) {
+ // if (stripHtml) {
+ // return offset - delta;
+ // }
+ // return offset;
+ // }
+ //
+ // public void setDelta(Tag tag, String name, CAS cas) {
+ // delta = tag.getStartPosition();
+ // plainTextDocument = plainTextDocument.substring(delta);
+ // tag2annotation(tag, name, cas);
+ // }
+ //
+ // public void head2annotation(Tag tag, String name, CAS cas) {
+ // Type type = getType(name, cas);
+ // if (name.equals("STYLE")) {
+ // tag.setEndPosition(tag.getEndTag().getStartPosition());
+ // }
+ // int begin = getOffset(tag.getStartPosition());
+ //
+ // delta += tag.getEndPosition() - tag.getStartPosition();
+ // int i;
+ // int end = tag.getEndPosition();
+ // if (tag.getEndTag() != null) {
+ // i = tag.getEndTag().getStartPosition();
+ // end = tag.getEndTag().getEndPosition();
+ // } else {
+ // // TODO fix this
+ // i = tag.getEndPosition();
+ // String startTag = "<" + tag.getRawTagName();
+ // String endTag = "</" + tag.getRawTagName() + ">";
+ // int j = i - 1;
+ // int k = i - 2;
+ // while (k < j && k > -1) {
+ // k = cas.getDocumentText().indexOf(startTag, k + 1);
+ // j = cas.getDocumentText().indexOf(endTag, j + 1);
+ // }
+ // if (i < j) {
+ // i = j;
+ // }
+ //
+ // }
+ //
+ // if (onlyContent) {
+ // begin = getOffset(tag.getEndPosition());
+ // end = i;
+ // }
+ //
+ // AnnotationFS annotation = cas.createAnnotation(type, begin, end);
+ // for (int k = 0; k < tag.getAttributesEx().size(); k++) {
+ // String test = tag.getAttributesEx().elementAt(k).toString();
+ // if (test.startsWith(" ") | test.startsWith("/")) {
+ // tag.getAttributesEx().remove(k);
+ // k--;
+ // }
+ // }
+ // if (tag.getEndTag() != null && !name.equals("HEAD")) {
+ // name = plainTextDocument.substring(tag.getEndPosition(), tag.getEndTag().getStartPosition());
+ // }
+ // StringArray attributeName;
+ // StringArray attributeValue;
+ // int size = tag.getAttributesEx().size() - 1;
+ // attributeName = new StringArray(jcas, size);
+ // attributeValue = new StringArray(jcas, size);
+ // for (int j = 0; j < size; j++) {
+ // Attribute attribute = (Attribute) tag.getAttributesEx().elementAt(j + 1);
+ // attributeName.set(j, attribute.getName());
+ // attributeValue.set(j, attribute.getValue());
+ // }
+ // HtmlAnnotation annotationDelta = new HtmlAnnotation(annotation, 0, attributeName,
+ // attributeValue, name, stripHtml);
+ // annotationList.add(annotationDelta);
+ //
+ // }
+ //
+ // public void node2annotation(Remark tag) {
+ // Type type = getType("REMARK", cas);
+ // int begin = getOffset(tag.getStartPosition());
+ //
+ // delta += tag.getEndPosition() - tag.getStartPosition();
+ // int end = tag.getEndPosition();
+ //
+ // if (onlyContent) {
+ // begin = getOffset(tag.getEndPosition());
+ // }
+ // AnnotationFS annotation = cas.createAnnotation(type, begin, end);
+ // HtmlRemark remark = new HtmlRemark(annotation, 0, stripHtml);
+ // String comment = tag.getText();
+ // remark.setComment(comment);
+ // remarks.add(remark);
+ // }
+ //
+ // public void headNode2annotation(Remark tag) {
+ // Type type = getType("REMARK", cas);
+ // int begin = getOffset(tag.getStartPosition());
+ //
+ // delta += tag.getEndPosition() - tag.getStartPosition();
+ // int end = tag.getEndPosition();
+ // if (onlyContent) {
+ // begin = getOffset(tag.getEndPosition());
+ // }
+ //
+ // AnnotationFS annotation = cas.createAnnotation(type, begin, end);
+ //
+ // HtmlRemark remark = new HtmlRemark(annotation, 0, stripHtml);
+ // String comment = tag.getText();
+ // remark.setComment(comment);
+ // remarks.add(remark);
+ // }
+ //
+ // public void endTag(Tag endtag) {
+ // int size = annotations.size();
+ // if (size > 0) {
+ // HtmlAnnotation annotationDelta = annotations.get(size - 1);
+ // String test = endtag.getRawTagName().toUpperCase().substring(1);
+ //
+ // if (annotationDelta
+ // .getAnnotation()
+ // .getFeatureValueAsString(
+ // annotationDelta.getAnnotation().getType().getFeatureByBaseName("name"))
+ // .equals(test)) {
+ // annotationDelta.changeDelta(delta);
+ // annotationList.add(annotationDelta);
+ // annotations.remove(size - 1);
+ // }
+ // }
+ // delta += endtag.getEndPosition() - endtag.getStartPosition();
+ // }
+ //
+ // public String getPlainTextDocument() {
+ // return plainTextDocument;
+ // }
+ //
+ // public List<HtmlAnnotation> getAnnotationList() {
+ // return annotationList;
+ // }
+ //
+ // public List<HtmlRemark> getRemarks() {
+ // return remarks;
+ // }
}
Modified: uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/resources/org/apache/uima/textmarker/engine/HtmlAnnotator.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/resources/org/apache/uima/textmarker/engine/HtmlAnnotator.xml?rev=1423857&r1=1423856&r2=1423857&view=diff
==============================================================================
--- uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/resources/org/apache/uima/textmarker/engine/HtmlAnnotator.xml (original)
+++ uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/main/resources/org/apache/uima/textmarker/engine/HtmlAnnotator.xml Wed Dec 19 13:32:21 2012
@@ -1,27 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
<analysisEngineDescription xmlns="http://uima.apache.org/resourceSpecifier">
<frameworkImplementation>org.apache.uima.java</frameworkImplementation>
<primitive>true</primitive>
- <annotatorImplementationName>org.apache.uima.textmarker.engine.html.HtmlAnnotator</annotatorImplementationName>
+ <annotatorImplementationName>org.apache.uima.textmarker.engine.HtmlAnnotator</annotatorImplementationName>
<analysisEngineMetaData>
<name>PlainTextAnnotator</name>
<description/>
@@ -29,18 +10,6 @@
<vendor/>
<configurationParameters>
<configurationParameter>
- <name>plainTextOutput</name>
- <type>Boolean</type>
- <multiValued>false</multiValued>
- <mandatory>false</mandatory>
- </configurationParameter>
- <configurationParameter>
- <name>outputViewName</name>
- <type>String</type>
- <multiValued>false</multiValued>
- <mandatory>false</mandatory>
- </configurationParameter>
- <configurationParameter>
<name>onlyContent</name>
<type>Boolean</type>
<multiValued>false</multiValued>
@@ -49,12 +18,6 @@
</configurationParameters>
<configurationParameterSettings>
<nameValuePair>
- <name>plainTextOutput</name>
- <value>
- <boolean>false</boolean>
- </value>
- </nameValuePair>
- <nameValuePair>
<name>onlyContent</name>
<value>
<boolean>true</boolean>
Modified: uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/test/java/org/apache/uima/textmarker/engine/HtmlAnnotatorTest.java
URL: http://svn.apache.org/viewvc/uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/test/java/org/apache/uima/textmarker/engine/HtmlAnnotatorTest.java?rev=1423857&r1=1423856&r2=1423857&view=diff
==============================================================================
--- uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/test/java/org/apache/uima/textmarker/engine/HtmlAnnotatorTest.java (original)
+++ uima/sandbox/TextMarker/trunk/uimaj-textmarker/src/test/java/org/apache/uima/textmarker/engine/HtmlAnnotatorTest.java Wed Dec 19 13:32:21 2012
@@ -22,20 +22,16 @@ package org.apache.uima.textmarker.engin
import static org.junit.Assert.assertEquals;
import java.io.File;
-import java.net.URISyntaxException;
import java.net.URL;
import org.apache.uima.UIMAFramework;
import org.apache.uima.analysis_engine.AnalysisEngine;
-import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.cas.text.AnnotationIndex;
import org.apache.uima.resource.ResourceSpecifier;
-import org.apache.uima.textmarker.TextMarkerTestUtils;
-import org.apache.uima.textmarker.engine.html.HtmlAnnotator;
import org.apache.uima.util.FileUtils;
import org.apache.uima.util.XMLInputSource;
import org.junit.Test;
@@ -62,55 +58,44 @@ public class HtmlAnnotatorTest {
AnnotationIndex<AnnotationFS> ai = null;
FSIterator<AnnotationFS> iterator = null;
- ae.setConfigParameterValue("plainTextOutput", false);
- ae.setConfigParameterValue("outputViewName", "");
ae.setConfigParameterValue("onlyContent", false);
ae.reconfigure();
cas.reset();
cas.setDocumentText(html);
ae.process(cas);
-
-
-
ai = cas.getAnnotationIndex(tagType);
-// for (AnnotationFS each : ai) {
-// System.out.println(each.getCoveredText().replaceAll("[\n\r]", ""));
-// }
-
-
iterator = ai.iterator();
-// assertEquals(7, ai.size());
-// AnnotationFS a = iterator.next();
-// assertEquals("HTML", a.getType().getShortName());
-// assertEquals(0, a.getBegin());
-// assertEquals(127, a.getEnd());
-// a = iterator.next();
-// assertEquals("HEAD", a.getType().getShortName());
-// assertEquals(8, a.getBegin());
-// assertEquals(39, a.getEnd());
-// a = iterator.next();
-// assertEquals("REMARK", a.getType().getShortName());
-// assertEquals(16, a.getBegin());
-// assertEquals(30, a.getEnd());
-// a = iterator.next();
-// assertEquals("BODY", a.getType().getShortName());
-// assertEquals(41, a.getBegin());
-// assertEquals(118, a.getEnd());
-// a = iterator.next();
-// assertEquals("TAG", a.getType().getShortName());
-// assertEquals(64, a.getBegin());
-// assertEquals(69, a.getEnd());
-// a = iterator.next();
-// assertEquals("B", a.getType().getShortName());
-// assertEquals(78, a.getBegin());
-// assertEquals(89, a.getEnd());
-// a = iterator.next();
-// assertEquals("TAG", a.getType().getShortName());
-// assertEquals(91, a.getBegin());
-// assertEquals(96, a.getEnd());
+ assertEquals(7, ai.size());
+ AnnotationFS a = iterator.next();
+ assertEquals("HTML", a.getType().getShortName());
+ assertEquals(0, a.getBegin());
+ assertEquals(127, a.getEnd());
+ a = iterator.next();
+ assertEquals("HEAD", a.getType().getShortName());
+ assertEquals(8, a.getBegin());
+ assertEquals(39, a.getEnd());
+ a = iterator.next();
+ assertEquals("REMARK", a.getType().getShortName());
+ assertEquals(16, a.getBegin());
+ assertEquals(30, a.getEnd());
+ a = iterator.next();
+ assertEquals("BODY", a.getType().getShortName());
+ assertEquals(41, a.getBegin());
+ assertEquals(118, a.getEnd());
+ a = iterator.next();
+ assertEquals("BR", a.getType().getShortName());
+ assertEquals(64, a.getBegin());
+ assertEquals(69, a.getEnd());
+ a = iterator.next();
+ assertEquals("B", a.getType().getShortName());
+ assertEquals(78, a.getBegin());
+ assertEquals(89, a.getEnd());
+ a = iterator.next();
+ assertEquals("BR", a.getType().getShortName());
+ assertEquals(91, a.getBegin());
+ assertEquals(96, a.getEnd());
- ae.setConfigParameterValue("plainTextOutput", false);
- ae.setConfigParameterValue("outputViewName", "");
+
ae.setConfigParameterValue("onlyContent", true);
ae.reconfigure();
cas.reset();
@@ -118,25 +103,22 @@ public class HtmlAnnotatorTest {
ae.process(cas);
ai = cas.getAnnotationIndex(tagType);
iterator = ai.iterator();
-// assertEquals(7, ai.size());
-// a = iterator.next();
-
-
- ae.setConfigParameterValue("plainTextOutput", true);
- ae.setConfigParameterValue("outputViewName", "plain");
- ae.setConfigParameterValue("onlyContent", false);
- ae.reconfigure();
- cas.reset();
- cas.setDocumentText(html);
- ae.process(cas);
-
- ae.setConfigParameterValue("plainTextOutput", false);
- ae.setConfigParameterValue("outputViewName", "plain");
- ae.setConfigParameterValue("onlyContent", true);
- ae.reconfigure();
- cas.reset();
- cas.setDocumentText(html);
- ae.process(cas);
-
+ assertEquals(4, ai.size());
+ a = iterator.next();
+ assertEquals("HTML", a.getType().getShortName());
+ assertEquals(6, a.getBegin());
+ assertEquals(120, a.getEnd());
+ a = iterator.next();
+ assertEquals("HEAD", a.getType().getShortName());
+ assertEquals(14, a.getBegin());
+ assertEquals(32, a.getEnd());
+ a = iterator.next();
+ assertEquals("BODY", a.getType().getShortName());
+ assertEquals(47, a.getBegin());
+ assertEquals(111, a.getEnd());
+ a = iterator.next();
+ assertEquals("B", a.getType().getShortName());
+ assertEquals(81, a.getBegin());
+ assertEquals(85, a.getEnd());
}
}