You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by bg...@apache.org on 2012/04/06 01:43:59 UTC
svn commit: r1310132 [6/6] - in /opennlp/sandbox/opennlp-similarity/src:
main/java/opennlp/tools/similarity/apps/
main/java/opennlp/tools/similarity/apps/taxo_builder/
main/java/opennlp/tools/similarity/apps/utils/
main/java/opennlp/tools/textsimilarit...
Modified: opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/WordNode.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/WordNode.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/WordNode.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/WordNode.java Thu Apr 5 23:43:57 2012
@@ -21,71 +21,73 @@ import java.util.ArrayList;
import java.util.List;
public class WordNode extends SyntacticTreeNode {
- // the word in the sentence
- private String word;
- private String lemma;
-
- public WordNode(String type, String word) {
- super(type);
-
- setWord(word);
- }
-
- public String getWord() {
- return word;
- }
-
- public void setWord(String word) {
- this.word = word;
-
- // update lemma accordingly
- this.lemma = null;
- /*WordDictionary.getInstance().getLemmaOrWord(word,
- getType()); */
- }
-
- @Override
- public String getLemma(boolean removeStopWord) {
- if (removeStopWord) // && Feature.isStopWord(lemma, getType()))
- return null;
-
- return lemma;
- }
-
- @Override
- public List<SyntacticTreeNode> getChildren() {
- // a word node is a leaf and has no children
- return null;
- }
-
- @Override
- public String getText() {
- return word;
- }
-
- @Override
- public String toStringIndented(int numTabs) {
- String indent = SyntacticTreeNode.getIndent(numTabs);
- StringBuilder builder = new StringBuilder();
- builder.append(indent).append("type = ").append(getType())
- .append(", word = ").append(word);
-
- return builder.toString();
- }
-
- public static void main(String[] args) {
- }
-
- @Override
- public List<String> getOrderedPOSList() {
- List<String> types = new ArrayList<String>();
- types.add(getType());
- return types;
- }
- @Override
- public List<String> getOrderedLemmaList() {
- List<String> types = new ArrayList<String>();
- types.add(this.getWord());
- return types;
- }
+ // the word in the sentence
+ private String word;
+ private String lemma;
+
+ public WordNode(String type, String word) {
+ super(type);
+
+ setWord(word);
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+
+ // update lemma accordingly
+ this.lemma = null;
+ /*
+ * WordDictionary.getInstance().getLemmaOrWord(word, getType());
+ */
+ }
+
+ @Override
+ public String getLemma(boolean removeStopWord) {
+ if (removeStopWord) // && Feature.isStopWord(lemma, getType()))
+ return null;
+
+ return lemma;
+ }
+
+ @Override
+ public List<SyntacticTreeNode> getChildren() {
+ // a word node is a leaf and has no children
+ return null;
+ }
+
+ @Override
+ public String getText() {
+ return word;
+ }
+
+ @Override
+ public String toStringIndented(int numTabs) {
+ String indent = SyntacticTreeNode.getIndent(numTabs);
+ StringBuilder builder = new StringBuilder();
+ builder.append(indent).append("type = ").append(getType())
+ .append(", word = ").append(word);
+
+ return builder.toString();
+ }
+
+ public static void main(String[] args) {
+ }
+
+ @Override
+ public List<String> getOrderedPOSList() {
+ List<String> types = new ArrayList<String>();
+ types.add(getType());
+ return types;
+ }
+
+ @Override
+ public List<String> getOrderedLemmaList() {
+ List<String> types = new ArrayList<String>();
+ types.add(this.getWord());
+ return types;
+ }
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SpeechRecognitionResultsProcessorTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SpeechRecognitionResultsProcessorTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SpeechRecognitionResultsProcessorTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SpeechRecognitionResultsProcessorTest.java Thu Apr 5 23:43:57 2012
@@ -26,24 +26,26 @@ import opennlp.tools.similarity.apps.Spe
public class SpeechRecognitionResultsProcessorTest extends TestCase {
- public void testRestaurantEntityInSpeechRecognitionResults(){
- SpeechRecognitionResultsProcessor proc = new SpeechRecognitionResultsProcessor();
- List<SentenceMeaningfullnessScore> res = proc.runSearchAndScoreMeaningfulness( Arrays.asList(new String[]{
- "remember to buy milk tomorrow for details",
- "remember to buy milk tomorrow from trader joes",
- "remember to buy milk tomorrow from 3 to jones",
- "remember to buy milk tomorrow for for details",
- "remember to buy milk tomorrow from third to joes",
- "remember to buy milk tomorrow from third to jones",
- "remember to buy milk tomorrow from for d jones"
- }));
-
- assertTrue(res.get(1).getScore()> res.get(0).getScore() && res.get(1).getScore()> res.get(2).getScore() &&
- res.get(1).getScore()> res.get(3).getScore() && res.get(1).getScore()> res.get(4).getScore() &&
- res.get(1).getScore()> res.get(5).getScore() && res.get(1).getScore()> res.get(6).getScore()
- );
- proc.close();
-
- }
+ public void testRestaurantEntityInSpeechRecognitionResults() {
+ SpeechRecognitionResultsProcessor proc = new SpeechRecognitionResultsProcessor();
+ List<SentenceMeaningfullnessScore> res = proc
+ .runSearchAndScoreMeaningfulness(Arrays.asList(new String[] {
+ "remember to buy milk tomorrow for details",
+ "remember to buy milk tomorrow from trader joes",
+ "remember to buy milk tomorrow from 3 to jones",
+ "remember to buy milk tomorrow for for details",
+ "remember to buy milk tomorrow from third to joes",
+ "remember to buy milk tomorrow from third to jones",
+ "remember to buy milk tomorrow from for d jones" }));
+
+ assertTrue(res.get(1).getScore() > res.get(0).getScore()
+ && res.get(1).getScore() > res.get(2).getScore()
+ && res.get(1).getScore() > res.get(3).getScore()
+ && res.get(1).getScore() > res.get(4).getScore()
+ && res.get(1).getScore() > res.get(5).getScore()
+ && res.get(1).getScore() > res.get(6).getScore());
+ proc.close();
+
+ }
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyBuildMatchTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyBuildMatchTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyBuildMatchTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyBuildMatchTest.java Thu Apr 5 23:43:57 2012
@@ -38,12 +38,15 @@ public class TaxonomyBuildMatchTest exte
}
public void testTaxonomyMatch() {
- TaxoQuerySnapshotMatcher matcher = new TaxoQuerySnapshotMatcher("src/test/resources/taxonomies/irs_domTaxo.dat");
- int score = matcher.getTaxoScore("Can Form 1040 EZ be used to claim the earned income credit.",
- "Can Form 1040EZ be used to claim the earned income credit? . Must I be entitled to claim a child as a dependent to claim the earned income credit based on the child being ");
+ TaxoQuerySnapshotMatcher matcher = new TaxoQuerySnapshotMatcher(
+ "src/test/resources/taxonomies/irs_domTaxo.dat");
+ int score = matcher
+ .getTaxoScore(
+ "Can Form 1040 EZ be used to claim the earned income credit.",
+ "Can Form 1040EZ be used to claim the earned income credit? . Must I be entitled to claim a child as a dependent to claim the earned income credit based on the child being ");
- System.out.println("The score is: "+ score);
- assertTrue(score>3);
+ System.out.println("The score is: " + score);
+ assertTrue(score > 3);
matcher.close();
}
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/GeneralizationListReducerTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/GeneralizationListReducerTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/GeneralizationListReducerTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/GeneralizationListReducerTest.java Thu Apr 5 23:43:57 2012
@@ -22,8 +22,8 @@ import java.util.List;
import junit.framework.TestCase;
-public class GeneralizationListReducerTest extends TestCase{
- private GeneralizationListReducer generalizationListReducer = new GeneralizationListReducer();
+public class GeneralizationListReducerTest extends TestCase {
+ private GeneralizationListReducer generalizationListReducer = new GeneralizationListReducer();
public void notNull() {
assertNotNull(generalizationListReducer);
@@ -61,16 +61,18 @@ public class GeneralizationListReducerTe
assertFalse(ch2.isASubChunk(ch1));
assertFalse(ch5.isASubChunk(ch4));
assertTrue(ch4.isASubChunk(ch5));
-
+
assertFalse(ch2.isASubChunk(ch3));
assertFalse(ch3.isASubChunk(ch2));
-
+
assertFalse(ch5.isASubChunk(ch3));
assertFalse(ch3.isASubChunk(ch5));
List<ParseTreeChunk> res = generalizationListReducer
.applyFilteringBySubsumption(inp);
- assertEquals(res.toString(), "[VP [VB-run IN-around NP-tigers NP-zoo ], NP [DT-the NP-tigers ], NP [DT-the NN-* VBG-flying NN-car ]]");
+ assertEquals(
+ res.toString(),
+ "[VP [VB-run IN-around NP-tigers NP-zoo ], NP [DT-the NP-tigers ], NP [DT-the NN-* VBG-flying NN-car ]]");
System.out.println(res);
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/LemmaFormManagerTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/LemmaFormManagerTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/LemmaFormManagerTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/LemmaFormManagerTest.java Thu Apr 5 23:43:57 2012
@@ -23,23 +23,27 @@ import junit.framework.TestCase;
import org.junit.Test;
import org.junit.runner.RunWith;
-public class LemmaFormManagerTest extends TestCase{
+public class LemmaFormManagerTest extends TestCase {
private LemmaFormManager lemmaFormManager = new LemmaFormManager();
-
public void testNotNull() {
assertNotNull(lemmaFormManager);
}
-
public void testMatches() {
- assertEquals(lemmaFormManager.matchLemmas(null, "loud", "loudness", "NN"), "loud" );
+ assertEquals(lemmaFormManager.matchLemmas(null, "loud", "loudness", "NN"),
+ "loud");
assertEquals(lemmaFormManager.matchLemmas(null, "24", "12", "CD"), null);
- assertEquals(lemmaFormManager.matchLemmas(null, "loud", "loudly", "NN"), "loud" );
- assertEquals(lemmaFormManager.matchLemmas(null, "!upgrade", "upgrade", "NN"), "!upgrade" );
- assertEquals(lemmaFormManager.matchLemmas(null, "!upgrade", "upgrades", "NN"), null );
- assertEquals(lemmaFormManager.matchLemmas(null, "!upgrade", "get", "NN"), null);
+ assertEquals(lemmaFormManager.matchLemmas(null, "loud", "loudly", "NN"),
+ "loud");
+ assertEquals(
+ lemmaFormManager.matchLemmas(null, "!upgrade", "upgrade", "NN"),
+ "!upgrade");
+ assertEquals(
+ lemmaFormManager.matchLemmas(null, "!upgrade", "upgrades", "NN"), null);
+ assertEquals(lemmaFormManager.matchLemmas(null, "!upgrade", "get", "NN"),
+ null);
}
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkListScorerTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkListScorerTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkListScorerTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkListScorerTest.java Thu Apr 5 23:43:57 2012
@@ -25,7 +25,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import junit.framework.TestCase;
-public class ParseTreeChunkListScorerTest extends TestCase{
+public class ParseTreeChunkListScorerTest extends TestCase {
private ParseTreeChunkListScorer parseTreeChunkListScorer = new ParseTreeChunkListScorer();
private ParseTreeChunk parseTreeChunk = new ParseTreeChunk();
@@ -35,7 +35,7 @@ public class ParseTreeChunkListScorerTes
+ " [ [VB-get NN-visa IN-* NN-* IN-in .-* ], [VBD-* IN-* NN-* NN-* .-* ], [VB-* NP-* ]]]");
double sc = parseTreeChunkListScorer.getParseTreeChunkListScore(chs);
- assertTrue(sc>1.90);
-
+ assertTrue(sc > 1.90);
+
}
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/ParseTreeChunkTest.java Thu Apr 5 23:43:57 2012
@@ -26,7 +26,7 @@ import junit.framework.TestCase;
import org.junit.Test;
import org.junit.runner.RunWith;
-public class ParseTreeChunkTest extends TestCase{
+public class ParseTreeChunkTest extends TestCase {
private ParseTreeMatcherDeterministic parseTreeMatcher = new ParseTreeMatcherDeterministic();
private ParseTreeChunk parseTreeChunk = new ParseTreeChunk();
private ParseTreeChunkListScorer parseTreeChunkListScorer = new ParseTreeChunkListScorer();
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/SyntMatcherTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/SyntMatcherTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/SyntMatcherTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/SyntMatcherTest.java Thu Apr 5 23:43:57 2012
@@ -32,88 +32,97 @@ import org.junit.runner.RunWith;
public class SyntMatcherTest extends TestCase {
- private ParserChunker2MatcherProcessor parserChunker2Matcher;
+ private ParserChunker2MatcherProcessor parserChunker2Matcher;
- private ParseTreeChunk parseTreeChunk = new ParseTreeChunk();
+ private ParseTreeChunk parseTreeChunk = new ParseTreeChunk();
- public void notNullTest() {
- parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
- assertNotNull(parserChunker2Matcher);
- }
-
- public void testMatch() {
- parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
- List<List<ParseTreeChunk>> matchResult = parserChunker2Matcher
- .assessRelevance(
- // "Can I get auto focus lens for digital camera",
- // "How can I get short focus zoom lens for digital camera"
- "Pulitzer Prize-Winning Reporter is an Illegal Immigrant",
- "Gay Pulitzer Prize-Winning Reporter Jose Antonio Vargas Comes Out as Undocumented " +
- "Immigrant Jose Antonio Vargas, a gay journalist who won a Pulitzer Prize " +
- "for his coverage of the Virginia Tech shootings in the Washington Post")
- .getMatchResult();
-
- System.out.println(matchResult);
- assertEquals( "[[ [NNP-pulitzer NNP-prize NNP-winning NNP-reporter ], [NN-immigrant ]], []]",
- matchResult.toString());
- System.out.println(parseTreeChunk.listToString(matchResult));
- assertEquals(" np [ [NNP-pulitzer NNP-prize NNP-winning NNP-reporter ], [NN-immigrant ]]",
- parseTreeChunk.listToString(matchResult));
-
- matchResult = parserChunker2Matcher
- .assessRelevance(
- "Sounds too good to be true but it actually is, the world's first flying car is finally here. ",
- "While it may seem like something straight out of a sci-fi " +
- "movie, the flying car might soon become a reality. ").getMatchResult();
-
- // TODO: possibly problem in new POS tagger from Parser
- System.out.println(matchResult);
- // was "[[ [DT-the NN-* VBG-flying NN-car ]], []]"
- assertEquals("[[ [PRP-it ], [DT-the NN-* NNS-* ]], [ [DT-the NN-* NNS-* ]]]",
- matchResult.toString()
- );
- System.out.println(parseTreeChunk.listToString(matchResult));
- assertEquals( " np [ [PRP-it ], [DT-the NN-* NNS-* ]] vp [ [DT-the NN-* NNS-* ]]",
- parseTreeChunk.listToString(matchResult));
-
- parserChunker2Matcher.close();
-
- }
-
-
- public void testMatchDigitalCamera() {
- parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
- List<List<ParseTreeChunk>> matchResult = parserChunker2Matcher.assessRelevance(
- "I am curious how to use the digital zoom of this camera for filming insects",
- "How can I get short focus zoom lens for digital camera").getMatchResult();
-
- System.out.println(matchResult);
- assertEquals("[[ [PRP-i ], [NN-zoom NN-camera ], [JJ-digital NN-* ], [NN-* IN-for ], [NN-camera ]], [ [JJ-digital NN-* ], [NN-zoom NN-camera ], [NN-* IN-for ]]]",
- matchResult.toString());
- System.out.println(parseTreeChunk.listToString(matchResult));
- assertEquals(" np [ [PRP-i ], [NN-zoom NN-camera ], [JJ-digital NN-* ], [NN-* IN-for ], [NN-camera ]] vp [ [JJ-digital NN-* ], [NN-zoom NN-camera ], [NN-* IN-for ]]",
- parseTreeChunk.listToString(matchResult));
- parserChunker2Matcher.close();
- }
-
-
- public void testHighSimilarity() {
- parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
- List<List<ParseTreeChunk>> matchResult = parserChunker2Matcher.assessRelevance(
- "Can I get auto focus lens for digital camera",
- "How can I get short focus zoom lens for digital camera").getMatchResult();
-
- System.out.println(matchResult);
- assertEquals( "[[ [PRP-i ], [NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ], [JJ-digital NN-camera ]], [ [VB-get NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ]]]",
- matchResult.toString());
- System.out.println(parseTreeChunk.listToString(matchResult));
- assertEquals(" np [ [PRP-i ], [NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ], [JJ-digital NN-camera ]] vp [ [VB-get NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ]]",
- parseTreeChunk.listToString(matchResult) );
- parserChunker2Matcher.close();
- }
-
- public void testZClose(){
- ParserChunker2MatcherProcessor.getInstance().close();
- }
+ public void notNullTest() {
+ parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
+ assertNotNull(parserChunker2Matcher);
+ }
+
+ public void testMatch() {
+ parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
+ List<List<ParseTreeChunk>> matchResult = parserChunker2Matcher
+ .assessRelevance(
+ // "Can I get auto focus lens for digital camera",
+ // "How can I get short focus zoom lens for digital camera"
+ "Pulitzer Prize-Winning Reporter is an Illegal Immigrant",
+ "Gay Pulitzer Prize-Winning Reporter Jose Antonio Vargas Comes Out as Undocumented "
+ + "Immigrant Jose Antonio Vargas, a gay journalist who won a Pulitzer Prize "
+ + "for his coverage of the Virginia Tech shootings in the Washington Post")
+ .getMatchResult();
+
+ System.out.println(matchResult);
+ assertEquals(
+ "[[ [NNP-pulitzer NNP-prize NNP-winning NNP-reporter ], [NN-immigrant ]], []]",
+ matchResult.toString());
+ System.out.println(parseTreeChunk.listToString(matchResult));
+ assertEquals(
+ " np [ [NNP-pulitzer NNP-prize NNP-winning NNP-reporter ], [NN-immigrant ]]",
+ parseTreeChunk.listToString(matchResult));
+
+ matchResult = parserChunker2Matcher
+ .assessRelevance(
+ "Sounds too good to be true but it actually is, the world's first flying car is finally here. ",
+ "While it may seem like something straight out of a sci-fi "
+ + "movie, the flying car might soon become a reality. ")
+ .getMatchResult();
+
+ // TODO: possibly problem in new POS tagger from Parser
+ System.out.println(matchResult);
+ // was "[[ [DT-the NN-* VBG-flying NN-car ]], []]"
+ assertEquals(
+ "[[ [PRP-it ], [DT-the NN-* NNS-* ]], [ [DT-the NN-* NNS-* ]]]",
+ matchResult.toString());
+ System.out.println(parseTreeChunk.listToString(matchResult));
+ assertEquals(
+ " np [ [PRP-it ], [DT-the NN-* NNS-* ]] vp [ [DT-the NN-* NNS-* ]]",
+ parseTreeChunk.listToString(matchResult));
+
+ parserChunker2Matcher.close();
+
+ }
+
+ public void testMatchDigitalCamera() {
+ parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
+ List<List<ParseTreeChunk>> matchResult = parserChunker2Matcher
+ .assessRelevance(
+ "I am curious how to use the digital zoom of this camera for filming insects",
+ "How can I get short focus zoom lens for digital camera")
+ .getMatchResult();
+
+ System.out.println(matchResult);
+ assertEquals(
+ "[[ [PRP-i ], [NN-zoom NN-camera ], [JJ-digital NN-* ], [NN-* IN-for ], [NN-camera ]], [ [JJ-digital NN-* ], [NN-zoom NN-camera ], [NN-* IN-for ]]]",
+ matchResult.toString());
+ System.out.println(parseTreeChunk.listToString(matchResult));
+ assertEquals(
+ " np [ [PRP-i ], [NN-zoom NN-camera ], [JJ-digital NN-* ], [NN-* IN-for ], [NN-camera ]] vp [ [JJ-digital NN-* ], [NN-zoom NN-camera ], [NN-* IN-for ]]",
+ parseTreeChunk.listToString(matchResult));
+ parserChunker2Matcher.close();
+ }
+
+ public void testHighSimilarity() {
+ parserChunker2Matcher = ParserChunker2MatcherProcessor.getInstance();
+ List<List<ParseTreeChunk>> matchResult = parserChunker2Matcher
+ .assessRelevance("Can I get auto focus lens for digital camera",
+ "How can I get short focus zoom lens for digital camera")
+ .getMatchResult();
+
+ System.out.println(matchResult);
+ assertEquals(
+ "[[ [PRP-i ], [NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ], [JJ-digital NN-camera ]], [ [VB-get NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ]]]",
+ matchResult.toString());
+ System.out.println(parseTreeChunk.listToString(matchResult));
+ assertEquals(
+ " np [ [PRP-i ], [NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ], [JJ-digital NN-camera ]] vp [ [VB-get NN-focus NNS-* NNS-lens IN-for JJ-digital NN-camera ]]",
+ parseTreeChunk.listToString(matchResult));
+ parserChunker2Matcher.close();
+ }
+
+ public void testZClose() {
+ ParserChunker2MatcherProcessor.getInstance().close();
+ }
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java Thu Apr 5 23:43:57 2012
@@ -1,3 +1,20 @@
+/*
+
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package opennlp.tools.textsimilarity.chunker2matcher;
import java.util.List;
@@ -8,95 +25,110 @@ import opennlp.tools.textsimilarity.Pars
import opennlp.tools.textsimilarity.ParseTreeChunkListScorer;
import opennlp.tools.textsimilarity.TextSimilarityBagOfWords;
-public class ParserChunker2MatcherProcessorTest extends TestCase{
- private ParserChunker2MatcherProcessor parser;
- private TextSimilarityBagOfWords parserBOW = new TextSimilarityBagOfWords ();
- private ParseTreeChunkListScorer parseTreeChunkListScorer = new ParseTreeChunkListScorer();
-
- public void testGroupedPhrasesFormer(){
- parser = ParserChunker2MatcherProcessor.getInstance();
- String text = "Where do I apply? Go to your town office or city hall. If your town doesn't have an office, ask the town clerk or a Selectman. Tell them that you need a 1040 tax form . I Can 't Pay the Taxes on my House: What Can I Do?. Pine Tree Legal";
-
-
-
- List<List<ParseTreeChunk>> res = parser.formGroupedPhrasesFromChunksForPara(text);
- System.out.println(res);
- assertEquals(
- "[[NP [PRP$-your NN-town NN-office CC-or NN-city NN-hall ], NP [PRP$-your NN-town NN-doesn NN-t ], NP [DT-an NN-office ], NP [DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], NP [DT-a NNP-Selectman ], NP [PRP-them IN-that PRP-you ], NP [PRP-you ], NP [DT-a CD-1040 NN-tax NN-form ], NP [PRP-I ], NP [DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [WP-What MD-Can PRP-I ], NP [PRP-I ], NP [NNP-Pine NNP-Tree NNP-Legal ]], [VP [VBP-do RB-I VB-apply ], VP [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], VP [VBP-have DT-an NN-office ], VP [VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], VP [VB-Tell PRP-them IN-that PRP-you ], VP [VBP-need DT-a CD-1040 NN-tax NN-form ], VP [MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], VP [VB-Do ]], [PP [TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], PP [IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ]],
[], [SENTENCE [WRB-Where VBP-do RB-I VB-apply ], SENTENCE [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], SENTENCE [IN-If PRP$-your NN-town NN-doesn NN-t VBP-have DT-an NN-office VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], SENTENCE [VB-Tell PRP-them IN-that PRP-you VBP-need DT-a CD-1040 NN-tax NN-form ], SENTENCE [PRP-I MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I VB-Do ], SENTENCE [NNP-Pine NNP-Tree NNP-Legal ]]]",
- // "[[NP [PRP$-your NN-town NN-office CC-or NN-city NN-hall ], NP [PRP$-your NN-town NN-doesn NN-t ], NP [DT-an NN-office ], NP [DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], NP [DT-a NNP-Selectman ], NP [PRP-them IN-that PRP-you ], NP [PRP-you ], NP [DT-a CD-1040 NN-tax NN-form ], NP [PRP-I ], NP [DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [WP-What MD-Can PRP-I ], NP [PRP-I ], NP [NNP-Pine NNP-Tree NNP-Legal ]], [VP [VBP-do RB-I VB-apply ], VP [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], VP [VBP-have DT-an NN-office ], VP [VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], VP [VB-Tell PRP-them IN-that PRP-you ], VP [VBP-need DT-a CD-1040 NN-tax NN-form ], VP [MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], VP [VB-Do NNP-Pine NNP-Tree NNP-Legal ]], [PP [TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], PP [IN-on PRP$-my NN
P-House WP-What MD-Can PRP-I ]], [], [SENTENCE [WRB-Where VBP-do RB-I VB-apply ], SENTENCE [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], SENTENCE [IN-If PRP$-your NN-town NN-doesn NN-t VBP-have DT-an NN-office VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], SENTENCE [VB-Tell PRP-them IN-that PRP-you VBP-need DT-a CD-1040 NN-tax NN-form ], SENTENCE [PRP-I MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I VB-Do NNP-Pine NNP-Tree NNP-Legal ]]]",
- res.toString());
-
- res = parser.formGroupedPhrasesFromChunksForSentence("How can I get short focus zoom lens for digital camera");
- assertEquals(
- "[[NP [PRP-I ], NP [JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ], NP [JJ-digital NN-camera ]], [VP [VB-get JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ]], [PP [IN-for JJ-digital NN-camera ]], [], [SENTENCE [WRB-How MD-can PRP-I VB-get JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ]]]",
- res.toString());
-
- res = parser.formGroupedPhrasesFromChunksForSentence("Its classy design and the Mercedes name make it a very cool vehicle to drive. ");
- assertEquals(
- "[[NP [PRP$-Its JJ-classy NN-design CC-and DT-the NNP-Mercedes NN-name ], NP [DT-the NNP-Mercedes NN-name ], NP [PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ], NP [DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ], NP [NN-drive ]], [VP [VBP-make PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ]], [PP [TO-to NN-drive ]], [], [SENTENCE [PRP$-Its JJ-classy NN-design CC-and DT-the NNP-Mercedes NN-name VBP-make PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ]]]",
- res.toString());
- res = parser.formGroupedPhrasesFromChunksForSentence("Sounds too good to be true but it actually is, the world's first flying car is finally here. ");
- assertEquals(
- "[[NP [PRP-it RB-actually ], NP [DT-the NN-world NNS-s JJ-first NN-flying NN-car ]], [VP [VBZ-Sounds RB-too JJ-good ], VP [TO-to VB-be JJ-true CC-but PRP-it RB-actually ], VP [VBZ-is DT-the NN-world NNS-s JJ-first NN-flying NN-car ], VP [VBZ-is RB-finally RB-here ]], [], [ADJP [RB-too JJ-good ], ADJP [JJ-true CC-but PRP-it RB-actually ]], [SENTENCE [VBZ-Sounds RB-too JJ-good TO-to VB-be JJ-true CC-but PRP-it RB-actually VBZ-is DT-the NN-world NNS-s JJ-first NN-flying NN-car VBZ-is RB-finally RB-here ]]]",
- res.toString());
- res = parser.formGroupedPhrasesFromChunksForSentence("UN Ambassador Ron Prosor repeated the Israeli position that the only way the Palestinians will get UN membership and statehood is through direct negotiations with the Israelis on a comprehensive peace agreement");
- assertEquals(
- "[[NP [NNP-UN NNP-Ambassador NNP-Ron NNP-Prosor ], NP [DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians ], NP [DT-the JJ-only NN-way DT-the NNPS-Palestinians ], NP [DT-the NNPS-Palestinians ], NP [NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [DT-a JJ-comprehensive NN-peace NN-agreement ]], [VP [VBD-repeated DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians ], VP [MD-will VB-get IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]], [PP [IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians
], PP [IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]], [], [SENTENCE [NNP-UN NNP-Ambassador NNP-Ron NNP-Prosor VBD-repeated DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians MD-will VB-get IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]]]",
- res.toString());
- parser.close();
- }
-
- public void testPrintParseTree(){
- parser = ParserChunker2MatcherProcessor.getInstance();
- try {
- parser.printParseTree("How can I get short focus zoom lens for digital camera");
- } catch (Exception e) {
- // when models does not read
- }
- parser.close();
- }
-
- public void testRelevanceAssessm(){
- parser = ParserChunker2MatcherProcessor.getInstance();
- String phrase1 = "Its classy design and the Mercedes name make it a very cool vehicle to drive. "
- + "The engine makes it a powerful car. "
- + "The strong engine gives it enough power. "
- + "The strong engine gives the car a lot of power.";
- String phrase2 = "This car has a great engine. "
- + "This car has an amazingly good engine. "
- + "This car provides you a very good mileage.";
-
- System.out.println(parser.assessRelevance(phrase1, phrase2).getMatchResult());
- parser.close();
-
- }
-
- public void testCompareRelevanceAssessmWithBagOfWords(){
- parser = ParserChunker2MatcherProcessor.getInstance();
- // we first demonstrate how similarity expression for DIFFERENT cases have too high score for bagOfWords
- String phrase1 = "How to deduct rental expense from income ";
- String phrase2 = "How to deduct repair expense from rental income.";
- List<List<ParseTreeChunk>> matchResult = parser.assessRelevance(phrase1, phrase2).getMatchResult();
- assertEquals(matchResult.toString(),
- "[[ [NN-expense IN-from NN-income ], [JJ-rental NN-* ], [NN-income ]], [ [TO-to VB-deduct JJ-rental NN-* ], [VB-deduct NN-expense IN-from NN-income ]]]");
- System.out.println(matchResult);
- double matchScore = parseTreeChunkListScorer.getParseTreeChunkListScore(matchResult);
- double bagOfWordsScore = parserBOW.assessRelevanceAndGetScore(phrase1, phrase2);
- assertTrue(matchScore+2 < bagOfWordsScore);
- System.out.println("MatchScore is adequate ( = "+matchScore + ") and bagOfWordsScore = "+bagOfWordsScore+" is too high");
-
- // we now demonstrate how similarity can be captured by POS and cannot be captured by bagOfWords
- phrase1 = "Way to minimize medical expense for my daughter";
- phrase2 = "Means to deduct educational expense for my son";
- matchResult = parser.assessRelevance(phrase1, phrase2).getMatchResult();
- assertEquals(matchResult.toString(),
- "[[ [JJ-* NN-expense IN-for PRP$-my NN-* ], [PRP$-my NN-* ]], [ [TO-to VB-* JJ-* NN-expense IN-for PRP$-my NN-* ]]]");
- System.out.println(matchResult);
- matchScore = parseTreeChunkListScorer.getParseTreeChunkListScore(matchResult);
- bagOfWordsScore = parserBOW.assessRelevanceAndGetScore(phrase1, phrase2);
- assertTrue(matchScore > 2*bagOfWordsScore);
- System.out.println("MatchScore is adequate ( = "+matchScore + ") and bagOfWordsScore = "+bagOfWordsScore+" is too low");
- parser.close();
+public class ParserChunker2MatcherProcessorTest extends TestCase {
+ private ParserChunker2MatcherProcessor parser;
+ private TextSimilarityBagOfWords parserBOW = new TextSimilarityBagOfWords();
+ private ParseTreeChunkListScorer parseTreeChunkListScorer = new ParseTreeChunkListScorer();
+
+ public void testGroupedPhrasesFormer() {
+ parser = ParserChunker2MatcherProcessor.getInstance();
+ String text = "Where do I apply? Go to your town office or city hall. If your town doesn't have an office, ask the town clerk or a Selectman. Tell them that you need a 1040 tax form . I Can 't Pay the Taxes on my House: What Can I Do?. Pine Tree Legal";
+
+ List<List<ParseTreeChunk>> res = parser
+ .formGroupedPhrasesFromChunksForPara(text);
+ System.out.println(res);
+ assertEquals(
+ "[[NP [PRP$-your NN-town NN-office CC-or NN-city NN-hall ], NP [PRP$-your NN-town NN-doesn NN-t ], NP [DT-an NN-office ], NP [DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], NP [DT-a NNP-Selectman ], NP [PRP-them IN-that PRP-you ], NP [PRP-you ], NP [DT-a CD-1040 NN-tax NN-form ], NP [PRP-I ], NP [DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [WP-What MD-Can PRP-I ], NP [PRP-I ], NP [NNP-Pine NNP-Tree NNP-Legal ]], [VP [VBP-do RB-I VB-apply ], VP [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], VP [VBP-have DT-an NN-office ], VP [VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], VP [VB-Tell PRP-them IN-that PRP-you ], VP [VBP-need DT-a CD-1040 NN-tax NN-form ], VP [MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], VP [VB-Do ]], [PP [TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], PP [IN-on PRP$-my NNP-House WP-What MD-Can PRP-
I ]], [], [SENTENCE [WRB-Where VBP-do RB-I VB-apply ], SENTENCE [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], SENTENCE [IN-If PRP$-your NN-town NN-doesn NN-t VBP-have DT-an NN-office VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], SENTENCE [VB-Tell PRP-them IN-that PRP-you VBP-need DT-a CD-1040 NN-tax NN-form ], SENTENCE [PRP-I MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I VB-Do ], SENTENCE [NNP-Pine NNP-Tree NNP-Legal ]]]",
+ // "[[NP [PRP$-your NN-town NN-office CC-or NN-city NN-hall ], NP [PRP$-your NN-town NN-doesn NN-t ], NP [DT-an NN-office ], NP [DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], NP [DT-a NNP-Selectman ], NP [PRP-them IN-that PRP-you ], NP [PRP-you ], NP [DT-a CD-1040 NN-tax NN-form ], NP [PRP-I ], NP [DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [WP-What MD-Can PRP-I ], NP [PRP-I ], NP [NNP-Pine NNP-Tree NNP-Legal ]], [VP [VBP-do RB-I VB-apply ], VP [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], VP [VBP-have DT-an NN-office ], VP [VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], VP [VB-Tell PRP-them IN-that PRP-you ], VP [VBP-need DT-a CD-1040 NN-tax NN-form ], VP [MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], VP [VB-Do NNP-Pine NNP-Tree NNP-Legal ]], [PP [TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], PP [IN-on PRP$-m
y NNP-House WP-What MD-Can PRP-I ]], [], [SENTENCE [WRB-Where VBP-do RB-I VB-apply ], SENTENCE [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], SENTENCE [IN-If PRP$-your NN-town NN-doesn NN-t VBP-have DT-an NN-office VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], SENTENCE [VB-Tell PRP-them IN-that PRP-you VBP-need DT-a CD-1040 NN-tax NN-form ], SENTENCE [PRP-I MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I VB-Do NNP-Pine NNP-Tree NNP-Legal ]]]",
+ res.toString());
+
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("How can I get short focus zoom lens for digital camera");
+ assertEquals(
+ "[[NP [PRP-I ], NP [JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ], NP [JJ-digital NN-camera ]], [VP [VB-get JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ]], [PP [IN-for JJ-digital NN-camera ]], [], [SENTENCE [WRB-How MD-can PRP-I VB-get JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ]]]",
+ res.toString());
+
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("Its classy design and the Mercedes name make it a very cool vehicle to drive. ");
+ assertEquals(
+ "[[NP [PRP$-Its JJ-classy NN-design CC-and DT-the NNP-Mercedes NN-name ], NP [DT-the NNP-Mercedes NN-name ], NP [PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ], NP [DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ], NP [NN-drive ]], [VP [VBP-make PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ]], [PP [TO-to NN-drive ]], [], [SENTENCE [PRP$-Its JJ-classy NN-design CC-and DT-the NNP-Mercedes NN-name VBP-make PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ]]]",
+ res.toString());
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("Sounds too good to be true but it actually is, the world's first flying car is finally here. ");
+ assertEquals(
+ "[[NP [PRP-it RB-actually ], NP [DT-the NN-world NNS-s JJ-first NN-flying NN-car ]], [VP [VBZ-Sounds RB-too JJ-good ], VP [TO-to VB-be JJ-true CC-but PRP-it RB-actually ], VP [VBZ-is DT-the NN-world NNS-s JJ-first NN-flying NN-car ], VP [VBZ-is RB-finally RB-here ]], [], [ADJP [RB-too JJ-good ], ADJP [JJ-true CC-but PRP-it RB-actually ]], [SENTENCE [VBZ-Sounds RB-too JJ-good TO-to VB-be JJ-true CC-but PRP-it RB-actually VBZ-is DT-the NN-world NNS-s JJ-first NN-flying NN-car VBZ-is RB-finally RB-here ]]]",
+ res.toString());
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("UN Ambassador Ron Prosor repeated the Israeli position that the only way the Palestinians will get UN membership and statehood is through direct negotiations with the Israelis on a comprehensive peace agreement");
+ assertEquals(
+ "[[NP [NNP-UN NNP-Ambassador NNP-Ron NNP-Prosor ], NP [DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians ], NP [DT-the JJ-only NN-way DT-the NNPS-Palestinians ], NP [DT-the NNPS-Palestinians ], NP [NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [DT-a JJ-comprehensive NN-peace NN-agreement ]], [VP [VBD-repeated DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians ], VP [MD-will VB-get IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]], [PP [IN-that DT-the JJ-only NN-way DT-the NNPS-Palestin
ians ], PP [IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]], [], [SENTENCE [NNP-UN NNP-Ambassador NNP-Ron NNP-Prosor VBD-repeated DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians MD-will VB-get IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]]]",
+ res.toString());
+ parser.close();
+ }
+
+ public void testPrintParseTree() {
+ parser = ParserChunker2MatcherProcessor.getInstance();
+ try {
+ parser
+ .printParseTree("How can I get short focus zoom lens for digital camera");
+ } catch (Exception e) {
+ // when models does not read
+ }
+ parser.close();
+ }
+
+ public void testRelevanceAssessm() {
+ parser = ParserChunker2MatcherProcessor.getInstance();
+ String phrase1 = "Its classy design and the Mercedes name make it a very cool vehicle to drive. "
+ + "The engine makes it a powerful car. "
+ + "The strong engine gives it enough power. "
+ + "The strong engine gives the car a lot of power.";
+ String phrase2 = "This car has a great engine. "
+ + "This car has an amazingly good engine. "
+ + "This car provides you a very good mileage.";
+
+ System.out.println(parser.assessRelevance(phrase1, phrase2)
+ .getMatchResult());
+ parser.close();
+
+ }
+
+ public void testCompareRelevanceAssessmWithBagOfWords() {
+ parser = ParserChunker2MatcherProcessor.getInstance();
+ // we first demonstrate how similarity expression for DIFFERENT cases have
+ // too high score for bagOfWords
+ String phrase1 = "How to deduct rental expense from income ";
+ String phrase2 = "How to deduct repair expense from rental income.";
+ List<List<ParseTreeChunk>> matchResult = parser.assessRelevance(phrase1,
+ phrase2).getMatchResult();
+ assertEquals(
+ matchResult.toString(),
+ "[[ [NN-expense IN-from NN-income ], [JJ-rental NN-* ], [NN-income ]], [ [TO-to VB-deduct JJ-rental NN-* ], [VB-deduct NN-expense IN-from NN-income ]]]");
+ System.out.println(matchResult);
+ double matchScore = parseTreeChunkListScorer
+ .getParseTreeChunkListScore(matchResult);
+ double bagOfWordsScore = parserBOW.assessRelevanceAndGetScore(phrase1,
+ phrase2);
+ assertTrue(matchScore + 2 < bagOfWordsScore);
+ System.out.println("MatchScore is adequate ( = " + matchScore
+ + ") and bagOfWordsScore = " + bagOfWordsScore + " is too high");
+
+ // we now demonstrate how similarity can be captured by POS and cannot be
+ // captured by bagOfWords
+ phrase1 = "Way to minimize medical expense for my daughter";
+ phrase2 = "Means to deduct educational expense for my son";
+ matchResult = parser.assessRelevance(phrase1, phrase2).getMatchResult();
+ assertEquals(
+ matchResult.toString(),
+ "[[ [JJ-* NN-expense IN-for PRP$-my NN-* ], [PRP$-my NN-* ]], [ [TO-to VB-* JJ-* NN-expense IN-for PRP$-my NN-* ]]]");
+ System.out.println(matchResult);
+ matchScore = parseTreeChunkListScorer
+ .getParseTreeChunkListScore(matchResult);
+ bagOfWordsScore = parserBOW.assessRelevanceAndGetScore(phrase1, phrase2);
+ assertTrue(matchScore > 2 * bagOfWordsScore);
+ System.out.println("MatchScore is adequate ( = " + matchScore
+ + ") and bagOfWordsScore = " + bagOfWordsScore + " is too low");
+ parser.close();
- }
+ }
}
Modified: opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java?rev=1310132&r1=1310131&r2=1310132&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java Thu Apr 5 23:43:57 2012
@@ -1,30 +1,49 @@
+/*
+
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package opennlp.tools.textsimilarity.chunker2matcher;
import java.util.List;
import junit.framework.TestCase;
-public class PhraseNodeTest extends TestCase{
- ParserChunker2MatcherProcessor proc = ParserChunker2MatcherProcessor.getInstance();
- public void testPOSTagsExtraction(){
-
- SentenceNode node = proc.parseSentenceNode("How can I get there");
-
- try {
- List<String> pOSlist = node.getOrderedPOSList();
- assertEquals("[WRB, MD, PRP, VB, RB]", pOSlist.toString());
-
- node = proc.parseSentenceNode("where do I apply");
- pOSlist = node.getOrderedPOSList();
- assertEquals("[WRB, VBP, PRP, RB]", pOSlist.toString());
-
- // should NOT start with upper case! last tag is missing
- node = proc.parseSentenceNode("Where do I apply");
- pOSlist = node.getOrderedPOSList();
- assertEquals("[WRB, VBP, PRP]", pOSlist.toString());
- } catch (Exception e) { // for run without models, where init fails
- assertEquals(node, null);
- }
+public class PhraseNodeTest extends TestCase {
+ ParserChunker2MatcherProcessor proc = ParserChunker2MatcherProcessor
+ .getInstance();
+
+ public void testPOSTagsExtraction() {
+
+ SentenceNode node = proc.parseSentenceNode("How can I get there");
+
+ try {
+ List<String> pOSlist = node.getOrderedPOSList();
+ assertEquals("[WRB, MD, PRP, VB, RB]", pOSlist.toString());
+
+ node = proc.parseSentenceNode("where do I apply");
+ pOSlist = node.getOrderedPOSList();
+ assertEquals("[WRB, VBP, PRP, RB]", pOSlist.toString());
+
+ // should NOT start with upper case! last tag is missing
+ node = proc.parseSentenceNode("Where do I apply");
+ pOSlist = node.getOrderedPOSList();
+ assertEquals("[WRB, VBP, PRP]", pOSlist.toString());
+ } catch (Exception e) { // for run without models, where init fails
+ assertEquals(node, null);
}
-
+ }
+
}