You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/11/18 10:48:42 UTC
svn commit: r1203557 - in /incubator/opennlp/sandbox/opennlp-similarity/src:
main/java/opennlp/tools/similarity/apps/
main/java/opennlp/tools/textsimilarity/chunker2matcher/
test/java/opennlp/tools/similarity/
test/java/opennlp/tools/similarity/apps/ t...
Author: joern
Date: Fri Nov 18 09:48:41 2011
New Revision: 1203557
URL: http://svn.apache.org/viewvc?rev=1203557&view=rev
Log:
OPENNLP-387 Demonstration on how similarity component improves search accuracy. Thanks to Boris Galitsky for providing a patch.
Added:
incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/HitBaseComparable.java (with props)
incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/SearchResultsProcessor.java (with props)
incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/
incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/
incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SearchResultsProcessorTest.java (with props)
incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/
incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java (with props)
incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java (with props)
Modified:
incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/BingWebQueryRunner.java
incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SentenceNode.java
incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SyntacticTreeNode.java
Modified: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/BingWebQueryRunner.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/BingWebQueryRunner.java?rev=1203557&r1=1203556&r2=1203557&view=diff
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/BingWebQueryRunner.java (original)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/BingWebQueryRunner.java Fri Nov 18 09:48:41 2011
@@ -75,16 +75,20 @@ public class BingWebQueryRunner {
}
if (resultSet != null) {
for (int i = 0; i < resultSet.length(); i++) {
- HitBase hit = new HitBase();
- JSONObject singleResult = resultSet.getJSONObject(i);
- hit.setAbstractText(singleResult.getString("Description"));
- hit.setDate(singleResult.getString("DateTime"));
- String title = StringUtils.replace(singleResult.getString("Title"),
- "î", " ");
- hit.setTitle(title);
- hit.setUrl(singleResult.getString("Url"));
-
- resp.appendHits(hit);
+ try {
+ HitBase hit = new HitBase();
+ JSONObject singleResult = resultSet.getJSONObject(i);
+ hit.setAbstractText(singleResult.getString("Description"));
+ hit.setDate(singleResult.getString("DateTime"));
+ String title = StringUtils.replace(singleResult.getString("Title"),
+ "î", " ");
+ hit.setTitle(title);
+ hit.setUrl(singleResult.getString("Url"));
+
+ resp.appendHits(hit);
+ } catch (Exception e) {
+ // incomplete search result: do not through exception
+ }
}
}
return resp;
Added: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/HitBaseComparable.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/HitBaseComparable.java?rev=1203557&view=auto
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/HitBaseComparable.java (added)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/HitBaseComparable.java Fri Nov 18 09:48:41 2011
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.similarity.apps;
+
+import java.util.Comparator;
+
+public class HitBaseComparable implements Comparator<HitBase> {
+
+ public int compare(HitBase o1, HitBase o2) {
+ return (o1.getGenerWithQueryScore() > o2.getGenerWithQueryScore() ? -1
+ : (o1 == o2 ? 0 : 1));
+ }
+}
\ No newline at end of file
Propchange: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/HitBaseComparable.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/SearchResultsProcessor.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/SearchResultsProcessor.java?rev=1203557&view=auto
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/SearchResultsProcessor.java (added)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/SearchResultsProcessor.java Fri Nov 18 09:48:41 2011
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.similarity.apps;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Logger;
+
+import opennlp.tools.textsimilarity.ParseTreeChunk;
+import opennlp.tools.textsimilarity.ParseTreeChunkListScorer;
+import opennlp.tools.textsimilarity.SentencePairMatchResult;
+import opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcessor;
+
+public class SearchResultsProcessor extends BingWebQueryRunner {
+ private static Logger LOG = Logger
+ .getLogger("opennlp.tools.similarity.apps.SearchResultsProcessor");
+ private ParseTreeChunkListScorer parseTreeChunkListScorer = new ParseTreeChunkListScorer();
+
+ /*
+ * Takes Bing API search results and calculates the parse tree similarity
+ * between the question and each snippet. Ranks those snippets with higher
+ * similarity score up
+ */
+ private BingResponse calculateMatchScoreResortHits(BingResponse resp,
+ String searchQuery) {
+ // TODO
+ /*
+ * if query is multi-sentence, special handling int indexDot =
+ * searchQuery.indexOf("."); if (indexDot>0 &&
+ * indexDot<searchQuery.length()-1){ MultipleSentenceQueryAnswerer ans = new
+ * MultipleSentenceQueryAnswerer(); return
+ * ans.calculateMatchScoreResortHits(resp, searchQuery); }
+ */
+ List<HitBase> newHitList = new ArrayList<HitBase>();
+ ParserChunker2MatcherProcessor sm = ParserChunker2MatcherProcessor
+ .getInstance();
+
+ for (HitBase hit : resp.getHits()) {
+ String snapshot = hit.getAbstractText().replace("<b>...</b>", ". ")
+ .replace("<b>", "").replace("</b>", "");
+ snapshot = snapshot.replace("</B>", "").replace("<B>", "")
+ .replace("<br>", "").replace("</br>", "").replace("...", ". ")
+ .replace("|", " ").replace(">", " ");
+ snapshot += " . " + hit.getTitle();
+ Double score = 0.0;
+ try {
+ SentencePairMatchResult matchRes = sm.assessRelevance(snapshot,
+ searchQuery);
+ List<List<ParseTreeChunk>> match = matchRes.getMatchResult();
+ score = parseTreeChunkListScorer.getParseTreeChunkListScore(match);
+ LOG.finest(score + " | " + snapshot);
+ } catch (Exception e) {
+ LOG.severe("Problem processing snapshot " + snapshot);
+ e.printStackTrace();
+ }
+ hit.setGenerWithQueryScore(score);
+ newHitList.add(hit);
+ }
+ Collections.sort(newHitList, new HitBaseComparable());
+ resp.setHits(newHitList);
+ LOG.info("\n\n ============= NEW ORDER ================= ");
+ for (HitBase hit : newHitList) {
+ LOG.info(hit.toString());
+ }
+
+ return resp;
+ }
+
+ public List<HitBase> runSearch(String query) {
+ BingResponse resp = null, // obtained from bing
+ newResp = null; // re-sorted based on similarity
+ try {
+ List<String> resultList = search(query, "", "", 30);
+ resp = populateBingHit(resultList.get(0));
+ // now we apply our own relevance filter
+ newResp = calculateMatchScoreResortHits(resp, query);
+
+ } catch (Exception e) {
+ // e.printStackTrace();
+ LOG.info("No search results for query '" + query);
+ e.printStackTrace();
+ return null;
+ }
+ // cast to super class
+ List<HitBase> hits = new ArrayList<HitBase>();
+ for (HitBase h : resp.getHits())
+ hits.add((HitBase) h);
+
+ hits = removeDuplicates(hits, 0.9);
+
+ return hits;
+ }
+
+}
Propchange: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/SearchResultsProcessor.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java?rev=1203557&r1=1203556&r2=1203557&view=diff
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java (original)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java Fri Nov 18 09:48:41 2011
@@ -83,7 +83,7 @@ public class ParserChunker2MatcherProces
private Parser parser;
private ChunkerME chunker;
private final int NUMBER_OF_SECTIONS_IN_SENTENCE_CHUNKS = 5;
- Logger logger = Logger.getLogger("opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcessor");
+ private static Logger LOG = Logger.getLogger("opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcessor");
static {
//TODO config
@@ -172,7 +172,7 @@ public class ParserChunker2MatcherProces
try {
parseArray = ParserTool.parseLine(sentence, parser, 1);
} catch (Throwable t) {
- logger.log(Level.WARNING, "failed to parse the sentence : '"+sentence, t);
+ LOG.log(Level.WARNING, "failed to parse the sentence : '"+sentence, t);
return null;
}
@@ -189,12 +189,14 @@ public class ParserChunker2MatcherProces
String[] sentences = splitSentences(para);
for(String sent: sentences){
List<List<ParseTreeChunk>> singleSentChunks = formGroupedPhrasesFromChunksForSentence(sent);
+ if (singleSentChunks==null)
+ continue;
if (listOfChunksAccum.size()<1 ){
listOfChunksAccum = new ArrayList<List<ParseTreeChunk>>(singleSentChunks);
} else
for(int i= 0; i<NUMBER_OF_SECTIONS_IN_SENTENCE_CHUNKS; i++){
//make sure not null
- if (singleSentChunks.size()!=NUMBER_OF_SECTIONS_IN_SENTENCE_CHUNKS)
+ if (singleSentChunks == null || singleSentChunks.size()!=NUMBER_OF_SECTIONS_IN_SENTENCE_CHUNKS)
break;
List<ParseTreeChunk> phraseI = singleSentChunks.get(i);
List<ParseTreeChunk> phraseIaccum = listOfChunksAccum.get(i);
@@ -202,7 +204,6 @@ public class ParserChunker2MatcherProces
listOfChunksAccum.set(i, phraseIaccum);
}
}
-
return listOfChunksAccum;
}
@@ -215,22 +216,41 @@ public class ParserChunker2MatcherProces
String[] toks = tokenizer.tokenize(sentence);
String[] tags = new String[toks.length]; //posTagger.tag(toks);
- int t=0;
SentenceNode node = parseSentenceNode(sentence);
+ if (node==null){
+ LOG.info("Problem parsing sentence '"+sentence);
+ return null;
+ }
List<String> POSlist = node.getOrderedPOSList();
tags = POSlist.toArray(new String[0]);
-
+ if (toks.length != tags.length){
+ LOG.info("disagreement between toks and tags; sent = '"+sentence + "'\n tags = "+tags +
+ "\n will now try this sentence in lower case" );
+ node = parseSentenceNode(sentence.toLowerCase());
+ if (node==null){
+ LOG.info("Problem parsing sentence '"+sentence);
+ return null;
+ }
+ POSlist = node.getOrderedPOSList();
+ tags = POSlist.toArray(new String[0]);
+ if (toks.length != tags.length){
+ LOG.info("AGAIN: disagreement between toks and tags for lower case! ");
+ if (toks.length>tags.length){
+ String[] newToks = new String[tags.length];
+ for(int i = 0; i<tags.length; i++ ){
+ newToks[i] = toks[i];
+ }
+ toks = newToks;
+
+ } else
+ return null;
+ }
+
+ }
String[] res = chunker.chunk(toks, tags);
Span[] span = chunker.chunkAsSpans(toks, tags);
Sequence[] seq = chunker.topKSequences(toks, tags);
- /* correction for chunking tags
- for(int i=0; i< toks.length; i++){
- if (toks[i].equalsIgnoreCase("is")){
- res[i] = "B-VP";
- }
- } */
-
List<List<ParseTreeChunk>> listOfChunks = new ArrayList<List<ParseTreeChunk>>();
List<ParseTreeChunk> nounPhr = new ArrayList<ParseTreeChunk>(),
prepPhr = new ArrayList<ParseTreeChunk>(), verbPhr = new ArrayList<ParseTreeChunk>(),
@@ -258,7 +278,7 @@ public class ParserChunker2MatcherProces
for(int j=i+1; j<res.length; j++){
if (res[j].startsWith("B-VP")){
nounPhr.add(new ParseTreeChunk("NP", lemmas, pOSs));
- logger.info(i + " => " +lemmas);
+ LOG.info(i + " => " +lemmas);
currPhraseClosed = true;
break;
} else {
@@ -268,7 +288,7 @@ public class ParserChunker2MatcherProces
}
if (!currPhraseClosed){
nounPhr.add(new ParseTreeChunk("NP", lemmas, pOSs));
- logger.info(i + " => " + lemmas);
+ LOG.fine(i + " => " + lemmas);
}
} else if (bi_POS.startsWith("B-PP")){// beginning of a phrase
@@ -279,7 +299,7 @@ public class ParserChunker2MatcherProces
for(int j=i+1; j<res.length; j++){
if (res[j].startsWith("B-VP")){
prepPhr.add(new ParseTreeChunk("PP", lemmas, pOSs));
- logger.info(i + " => " + lemmas);
+ LOG.fine(i + " => " + lemmas);
currPhraseClosed = true;
break;
} else {
@@ -289,7 +309,7 @@ public class ParserChunker2MatcherProces
}
if (!currPhraseClosed){
prepPhr.add(new ParseTreeChunk("PP", lemmas, pOSs));
- logger.info(i + " => " + lemmas);
+ LOG.fine(i + " => " + lemmas);
}
} else
if (bi_POS.startsWith("B-VP")){// beginning of a phrase
@@ -300,7 +320,7 @@ public class ParserChunker2MatcherProces
for(int j=i+1; j<res.length; j++){
if (res[j].startsWith("B-VP")){
verbPhr.add(new ParseTreeChunk("VP", lemmas, pOSs));
- logger.info(i + " => " +lemmas);
+ LOG.fine(i + " => " +lemmas);
currPhraseClosed = true;
break;
} else {
@@ -310,7 +330,7 @@ public class ParserChunker2MatcherProces
}
if (!currPhraseClosed){
verbPhr.add(new ParseTreeChunk("VP", lemmas, pOSs));
- logger.info(i + " => " + lemmas);
+ LOG.fine(i + " => " + lemmas);
}
} else
if (bi_POS.startsWith("B-ADJP") ){// beginning of a phrase
@@ -321,7 +341,7 @@ public class ParserChunker2MatcherProces
for(int j=i+1; j<res.length; j++){
if (res[j].startsWith("B-VP")){
adjPhr.add(new ParseTreeChunk("ADJP", lemmas, pOSs));
- logger.info(i + " => " +lemmas);
+ LOG.fine(i + " => " +lemmas);
currPhraseClosed = true;
break;
} else {
@@ -331,7 +351,7 @@ public class ParserChunker2MatcherProces
}
if (!currPhraseClosed){
adjPhr.add(new ParseTreeChunk("ADJP", lemmas, pOSs));
- logger.info(i + " => " + lemmas);
+ LOG.fine(i + " => " + lemmas);
}
}
}
@@ -378,7 +398,7 @@ public class ParserChunker2MatcherProces
sentenceNode = sentenceToSentenceNode(sentenceParse);
} catch (Exception e) {
// don't fail the whole paragraph when a single sentence fails
- System.err.println("Faile to convert sentence to node. error: " + e);
+ LOG.severe("Failed to convert sentence to node. error: " + e);
sentenceNode = null;
}
@@ -398,15 +418,14 @@ public class ParserChunker2MatcherProces
// convert the OpenNLP Parse to our own tree nodes
SyntacticTreeNode node = toSyntacticTreeNode(sentenceParse);
- if ((node == null) || !(node instanceof SentenceNode))
+ if ((node == null))
return null;
-
- SentenceNode sentenceNode = (SentenceNode) node;
-
- // fix the parsing tree
- fixParsingTree(sentenceNode);
-
- return sentenceNode;
+ if (node instanceof SentenceNode)
+ return (SentenceNode)node;
+ else if (node instanceof PhraseNode){
+ SentenceNode sn = new SentenceNode("sentence", node.getChildren()) ;
+ return sn;
+ } else return null;
}
public List<List<SentenceNode>> parseTextNode(String text) {
@@ -563,7 +582,7 @@ public class ParserChunker2MatcherProces
// check for junk types
String type = parse.getType();
- if (SyntacticTreeNode.isJunkType(type))
+ if (SyntacticTreeNode.isJunkType(type, parse) )
return null;
String text = parse.getText();
@@ -606,14 +625,7 @@ public class ParserChunker2MatcherProces
return childrenNodeList;
}
- private static void fixParsingTree(SentenceNode sentenceNode) {
- // logger.finest("before = " + sentenceNode);
- // for (ParsingTreeFixer fixer : FIXER_LIST) {
- // fixer.fix(sentenceNode);
- // }
- // logger.finest("after = " + sentenceNode);
- }
-
+
public SentencePairMatchResult assessRelevance(String para1, String para2)
{
ParserChunker2MatcherProcessor parser = ParserChunker2MatcherProcessor.getInstance();
@@ -630,8 +642,11 @@ public class ParserChunker2MatcherProces
}
private List<LemmaPair> listListParseTreeChunk2ListLemmaPairs(
List<List<ParseTreeChunk>> sent1GrpLst) {
- List<ParseTreeChunk> wholeSentence = sent1GrpLst.get(sent1GrpLst.size()-1); // whole sentence is last list in the list of lists
List<LemmaPair> results = new ArrayList<LemmaPair>();
+ if (sent1GrpLst==null || sent1GrpLst.size() <1)
+ return results;
+ List<ParseTreeChunk> wholeSentence = sent1GrpLst.get(sent1GrpLst.size()-1); // whole sentence is last list in the list of lists
+
List<String> pOSs = wholeSentence.get(0).getPOSs();
List<String> lemmas = wholeSentence.get(0).getLemmas();
for(int i= 0; i< lemmas.size(); i++){
@@ -653,7 +668,7 @@ public class ParserChunker2MatcherProces
public static void main(String[] args) throws Exception {
-
+ String text = "Where do I apply? Go to your town office or city hall. If your town doesn't have an office, ask the town clerk or a Selectman. Tell them that you need a 1040 tax form . I Can 't Pay the Taxes on my House: What Can I Do?. Pine Tree Legal";
/*
* String text =
@@ -674,8 +689,11 @@ public class ParserChunker2MatcherProces
// String sentence = "I love Fresh body styling";
// String phrase = "I captures way more detail in high contrast scenes";
ParserChunker2MatcherProcessor parser = ParserChunker2MatcherProcessor.getInstance();
- parser.printParseTree("How can I get short focus zoom lens for digital camera");
- parser.formGroupedPhrasesFromChunksForSentence("How can I get short focus zoom lens for digital camera");
+ List<List<ParseTreeChunk>> res = parser.formGroupedPhrasesFromChunksForPara(text);
+ System.out.println(res);
+
+ //parser.printParseTree("How can I get short focus zoom lens for digital camera");
+ //parser.formGroupedPhrasesFromChunksForSentence("How can I get short focus zoom lens for digital camera");
System.exit(0);
Modified: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SentenceNode.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SentenceNode.java?rev=1203557&r1=1203556&r2=1203557&view=diff
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SentenceNode.java (original)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SentenceNode.java Fri Nov 18 09:48:41 2011
@@ -68,4 +68,6 @@ public class SentenceNode extends Phrase
}
return types;
}
+
+
}
Modified: incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SyntacticTreeNode.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SyntacticTreeNode.java?rev=1203557&r1=1203556&r2=1203557&view=diff
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SyntacticTreeNode.java (original)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/SyntacticTreeNode.java Fri Nov 18 09:48:41 2011
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.List;
import opennlp.tools.parser.AbstractBottomUpParser;
+import opennlp.tools.parser.Parse;
public abstract class SyntacticTreeNode {
// the POS type
@@ -87,7 +88,7 @@ public abstract class SyntacticTreeNode
return builder.toString();
}
- public static boolean isJunkType(String type) {
+ public static boolean isJunkType(String type, Parse parse) {
if (type == null)
return true;
@@ -96,8 +97,11 @@ public abstract class SyntacticTreeNode
return true;
// the punctuation nodes are not useful, '.', '.', '?', '!', ';', etc
- if (type.equals(",") || type.equals(".") || type.equals("?")
- || type.equals("!") || type.equals(";"))
+ if ((type.equals(",") || type.equals(".") || type.equals("?")
+ || type.equals("!") || type.equals(";"))
+ // TODO : Parser gives type = '.' instead of VB
+ //&& ( parse.getHead().toString().length()<2
+ )
return true;
return false;
Added: incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SearchResultsProcessorTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SearchResultsProcessorTest.java?rev=1203557&view=auto
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SearchResultsProcessorTest.java (added)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SearchResultsProcessorTest.java Fri Nov 18 09:48:41 2011
@@ -0,0 +1,38 @@
+package opennlp.tools.similarity.apps;
+
+import java.util.List;
+
+import junit.framework.TestCase;
+
+public class SearchResultsProcessorTest extends TestCase{
+ SearchResultsProcessor proc = new SearchResultsProcessor();
+
+
+ public void testSearchOrder(){
+ List<HitBase> res = proc.runSearch("How can I pay tax on my income abroad");
+
+ // we verify that top answers have high similarity score
+ System.out.println(res);
+ HitBase first = res.get(0);
+ assertTrue( first.getGenerWithQueryScore()>3.0);
+ //assertTrue(first.getTitle().indexOf("Foreign")>-1 && first.getTitle().indexOf("earned")>-1);
+
+ HitBase second = res.get(1);
+ assertTrue( second.getGenerWithQueryScore()>1.9);
+ //assertTrue(second.getTitle().indexOf("living abroad")>-1);
+
+ }
+
+ public void testSearchOrder2(){
+ List<HitBase> res = proc.runSearch(
+ "Can I estimate what my income tax would be by using my last pay");
+
+ System.out.println(res);
+ HitBase first = res.get(0);
+ assertTrue( first.getGenerWithQueryScore()>1.9);
+
+ HitBase second = res.get(1);
+ assertTrue( second.getGenerWithQueryScore()>1.9);
+
+ }
+}
Propchange: incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/similarity/apps/SearchResultsProcessorTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java?rev=1203557&view=auto
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java (added)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java Fri Nov 18 09:48:41 2011
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.textsimilarity.chunker2matcher;
+
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import opennlp.tools.textsimilarity.ParseTreeChunk;
+
+public class ParserChunker2MatcherProcessorTest extends TestCase {
+ ParserChunker2MatcherProcessor parser = ParserChunker2MatcherProcessor
+ .getInstance();
+
+ public void testGroupedPhrasesFormer() {
+
+ String text = "Where do I apply? Go to your town office or city hall. If your town doesn't have an office, ask the town clerk or a Selectman. Tell them that you need a 1040 tax form . I Can 't Pay the Taxes on my House: What Can I Do?. Pine Tree Legal";
+
+ List<List<ParseTreeChunk>> res = parser
+ .formGroupedPhrasesFromChunksForPara(text);
+ System.out.println(res);
+ assertEquals(
+ "[[NP [PRP$-your NN-town NN-office CC-or NN-city NN-hall ], NP [PRP$-your NN-town NN-doesn NN-t ], NP [DT-an NN-office ], NP [DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], NP [DT-a NNP-Selectman ], NP [PRP-them IN-that PRP-you ], NP [PRP-you ], NP [DT-a CD-1040 NN-tax NN-form ], NP [PRP-I ], NP [DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [PRP$-my NNP-House WP-What MD-Can PRP-I ], NP [WP-What MD-Can PRP-I ], NP [PRP-I ], NP [NNP-Pine NNP-Tree NNP-Legal ]], [VP [VBP-do RB-I VB-apply ], VP [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], VP [VBP-have DT-an NN-office ], VP [VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], VP [VB-Tell PRP-them IN-that PRP-you ], VP [VBP-need DT-a CD-1040 NN-tax NN-form ], VP [MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I ], VP [VB-Do NNP-Pine NNP-Tree NNP-Legal ]], [PP [TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], PP [IN-on PRP$-my N
NP-House WP-What MD-Can PRP-I ]], [], [SENTENCE [WRB-Where VBP-do RB-I VB-apply ], SENTENCE [VB-Go TO-to PRP$-your NN-town NN-office CC-or NN-city NN-hall ], SENTENCE [IN-If PRP$-your NN-town NN-doesn NN-t VBP-have DT-an NN-office VB-ask DT-the NN-town NN-clerk CC-or DT-a NNP-Selectman ], SENTENCE [VB-Tell PRP-them IN-that PRP-you VBP-need DT-a CD-1040 NN-tax NN-form ], SENTENCE [PRP-I MD-Can VB-t VB-Pay DT-the NNS-Taxes IN-on PRP$-my NNP-House WP-What MD-Can PRP-I VB-Do NNP-Pine NNP-Tree NNP-Legal ]]]",
+ res.toString());
+
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("How can I get short focus zoom lens for digital camera");
+ assertEquals(
+ "[[NP [PRP-I ], NP [JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ], NP [JJ-digital NN-camera ]], [VP [VB-get JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ]], [PP [IN-for JJ-digital NN-camera ]], [], [SENTENCE [WRB-How MD-can PRP-I VB-get JJ-short NN-focus NN-zoom NN-lens IN-for JJ-digital NN-camera ]]]",
+ res.toString());
+
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("Its classy design and the Mercedes name make it a very cool vehicle to drive. ");
+ assertEquals(
+ "[[NP [PRP$-Its JJ-classy NN-design CC-and DT-the NNP-Mercedes NN-name ], NP [DT-the NNP-Mercedes NN-name ], NP [PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ], NP [DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ], NP [NN-drive ]], [VP [VBP-make PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ]], [PP [TO-to NN-drive ]], [], [SENTENCE [PRP$-Its JJ-classy NN-design CC-and DT-the NNP-Mercedes NN-name VBP-make PRP-it DT-a RB-very JJ-cool NN-vehicle TO-to NN-drive ]]]",
+ res.toString());
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("Sounds too good to be true but it actually is, the world's first flying car is finally here. ");
+ assertEquals(
+ "[[NP [PRP-it RB-actually ], NP [DT-the NN-world NNS-s JJ-first NN-flying NN-car ]], [VP [VBZ-Sounds RB-too JJ-good ], VP [TO-to VB-be JJ-true CC-but PRP-it RB-actually ], VP [VBZ-is DT-the NN-world NNS-s JJ-first NN-flying NN-car ], VP [VBZ-is RB-finally RB-here ]], [], [ADJP [RB-too JJ-good ], ADJP [JJ-true CC-but PRP-it RB-actually ]], [SENTENCE [VBZ-Sounds RB-too JJ-good TO-to VB-be JJ-true CC-but PRP-it RB-actually VBZ-is DT-the NN-world NNS-s JJ-first NN-flying NN-car VBZ-is RB-finally RB-here ]]]",
+ res.toString());
+ res = parser
+ .formGroupedPhrasesFromChunksForSentence("UN Ambassador Ron Prosor repeated the Israeli position that the only way the Palestinians will get UN membership and statehood is through direct negotiations with the Israelis on a comprehensive peace agreement");
+ assertEquals(
+ "[[NP [NNP-UN NNP-Ambassador NNP-Ron NNP-Prosor ], NP [DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians ], NP [DT-the JJ-only NN-way DT-the NNPS-Palestinians ], NP [DT-the NNPS-Palestinians ], NP [NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], NP [DT-a JJ-comprehensive NN-peace NN-agreement ]], [VP [VBD-repeated DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians ], VP [MD-will VB-get IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]], [PP [IN-that DT-the JJ-only NN-way DT-the NNPS-Palestin
ians ], PP [IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ], PP [IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]], [], [SENTENCE [NNP-UN NNP-Ambassador NNP-Ron NNP-Prosor VBD-repeated DT-the JJ-Israeli NN-position IN-that DT-the JJ-only NN-way DT-the NNPS-Palestinians MD-will VB-get IN-UN NN-membership CC-and NN-statehood VBZ-is IN-through JJ-direct NNS-negotiations IN-with DT-the NNP-Israelis IN-on DT-a JJ-comprehensive NN-peace NN-agreement ]]]",
+ res.toString());
+ }
+
+ public void testPrintParseTree() {
+ parser
+ .printParseTree("How can I get short focus zoom lens for digital camera");
+ }
+
+ public void testRelevanceAssessm() {
+ String phrase1 = "Its classy design and the Mercedes name make it a very cool vehicle to drive. "
+ + "The engine makes it a powerful car. "
+ + "The strong engine gives it enough power. "
+ + "The strong engine gives the car a lot of power.";
+ String phrase2 = "This car has a great engine. "
+ + "This car has an amazingly good engine. "
+ + "This car provides you a very good mileage.";
+ String sentence = "Not to worry with the 2cv.";
+
+ System.out.println(parser.assessRelevance(phrase1, phrase2));
+
+ }
+}
Propchange: incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessorTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java?rev=1203557&view=auto
==============================================================================
--- incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java (added)
+++ incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java Fri Nov 18 09:48:41 2011
@@ -0,0 +1,44 @@
+package opennlp.tools.textsimilarity.chunker2matcher;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.List;
+
+import junit.framework.TestCase;
+
+public class PhraseNodeTest extends TestCase {
+ ParserChunker2MatcherProcessor proc = ParserChunker2MatcherProcessor
+ .getInstance();
+
+ public void testPOSTagsExtraction() {
+
+ SentenceNode node = proc.parseSentenceNode("How can I get there");
+ List<String> pOSlist = node.getOrderedPOSList();
+ assertEquals("[WRB, MD, PRP, VB, RB]", pOSlist.toString());
+
+ node = proc.parseSentenceNode("where do I apply");
+ pOSlist = node.getOrderedPOSList();
+ assertEquals("[WRB, VBP, PRP, RB]", pOSlist.toString());
+
+ // should NOT start with upper case!
+ node = proc.parseSentenceNode("Where do I apply");
+ pOSlist = node.getOrderedPOSList();
+ assertEquals("[WRB, VBP, PRP]", pOSlist.toString());
+ }
+
+}
Propchange: incubator/opennlp/sandbox/opennlp-similarity/src/test/java/opennlp/tools/textsimilarity/chunker2matcher/PhraseNodeTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain