You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2012/02/04 03:28:03 UTC
svn commit: r1240442 - in
/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad:
ADChunkSampleStream.java ADChunkSampleStreamFactory.java
Author: colen
Date: Sat Feb 4 02:28:03 2012
New Revision: 1240442
URL: http://svn.apache.org/viewvc?rev=1240442&view=rev
Log:
OPENNLP-423: Improved AD Chunker formatter.
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java (contents, props changed)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java?rev=1240442&r1=1240441&r2=1240442&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java Sat Feb 4 02:28:03 2012
@@ -73,7 +73,7 @@ public class ADChunkSampleStream impleme
* a stream of lines as {@link String}
*/
public ADChunkSampleStream(ObjectStream<String> lineStream) {
- this.adSentenceStream = new ADSentenceStream(lineStream);
+ this.adSentenceStream = new ADSentenceStream(lineStream);
}
/**
@@ -135,15 +135,21 @@ public class ADChunkSampleStream impleme
if (elements[i].isLeaf()) {
processLeaf((Leaf) elements[i], false, "O", sentence, tags, target);
} else {
- processNode((Node) elements[i], sentence, tags, target);
+ processNode((Node) elements[i], sentence, tags, target, null);
}
}
}
}
private void processNode(Node node, List<String> sentence, List<String> tags,
- List<String> target) {
+ List<String> target, String inheritedTag) {
String phraseTag = getChunkTag(node.getSyntacticTag());
+
+ boolean inherited = false;
+ if(phraseTag.equals("O") && inheritedTag != null) {
+ phraseTag = inheritedTag;
+ inherited = true;
+ }
TreeElement[] elements = node.getElements();
for (int i = 0; i < elements.length; i++) {
@@ -152,10 +158,13 @@ public class ADChunkSampleStream impleme
if ( i > 0 && elements[i - 1].isLeaf() && phraseTag != null && !phraseTag.equals("O")) {
isIntermediate = true;
}
+ if(inherited && target.size() > 0 && target.get(target.size() - 1).endsWith(phraseTag)) {
+ isIntermediate = true;
+ }
processLeaf((Leaf) elements[i], isIntermediate, phraseTag, sentence,
tags, target);
} else {
- processNode((Node) elements[i], sentence, tags, target);
+ processNode((Node) elements[i], sentence, tags, target, phraseTag);
}
}
}
@@ -166,11 +175,11 @@ public class ADChunkSampleStream impleme
- if (leaf.getSyntacticTag() != null
+ if (leaf.getFunctionalTag() != null
&& phraseTag.equals("O")) {
- if(leaf.getSyntacticTag().endsWith("v-fin")) {
+ if(leaf.getFunctionalTag().equals("v-fin")) {
phraseTag = "VP";
- } else if(leaf.getSyntacticTag().endsWith(":n")) {
+ } else if(leaf.getFunctionalTag().equals("n")) {
phraseTag = "NP";
}
}
@@ -189,16 +198,28 @@ public class ADChunkSampleStream impleme
if (leaf.getSyntacticTag() == null) {
tags.add(leaf.getLexeme());
} else {
- tags.add(getMorphologicalTag(leaf.getSyntacticTag()));
+ tags.add(ADChunkSampleStream.convertFuncTag(leaf.getFunctionalTag(), false));
}
target.add(chunkTag);
}
- private String getMorphologicalTag(String tag) {
- return tag.substring(tag.lastIndexOf(":") + 1);
- }
+ public static String convertPhraseTag(String phraseTag) {
+ if ("NP".equals(phraseTag) || "VP".equals(phraseTag)) {
+ return phraseTag;
+ }
+ return "O";
+ }
+
+ public static String convertFuncTag(String t, boolean useCGTags) {
+ if (useCGTags) {
+ if ("art".equals(t) || "pron-det".equals(t) || "pron-indef".equals(t)) {
+ t = "det";
+ }
+ }
+ return t;
+ }
- private String getChunkTag(String tag) {
+ private String getChunkTag(String tag) {
String phraseTag = tag.substring(tag.lastIndexOf(":") + 1);
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
('svn:eol-style' removed)
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java?rev=1240442&r1=1240441&r2=1240442&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStreamFactory.java Sat Feb 4 02:28:03 2012
@@ -18,6 +18,7 @@
package opennlp.tools.formats.ad;
import java.io.File;
+import java.io.FileInputStream;
import java.nio.charset.Charset;
import opennlp.tools.chunker.ChunkSample;
@@ -28,6 +29,7 @@ import opennlp.tools.cmdline.CmdLineUtil
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.formats.LanguageSampleStreamFactory;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.PlainTextByLineStream;
/**
* A Factory to create a Arvores Deitadas ChunkStream from the command line
@@ -74,10 +76,12 @@ public class ADChunkSampleStreamFactory
language = params.getLang();
- Charset encoding = params.getEncoding();
+ FileInputStream sampleDataIn = CmdLineUtil.openInFile(params.getData());
+
+ ObjectStream<String> lineStream = new PlainTextByLineStream(sampleDataIn.getChannel(),
+ params.getEncoding());
- ADChunkSampleStream sampleStream =
- new ADChunkSampleStream(CmdLineUtil.openInFile(params.getData()), encoding.name());
+ ADChunkSampleStream sampleStream = new ADChunkSampleStream(lineStream);
if(params.getStart() != null && params.getStart() > -1) {
sampleStream.setStart(params.getStart());