You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by bg...@apache.org on 2012/10/03 00:46:55 UTC
svn commit: r1393226 - in
/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools:
similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java
textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
Author: bgalitsky
Date: Tue Oct 2 22:46:54 2012
New Revision: 1393226
URL: http://svn.apache.org/viewvc?rev=1393226&view=rev
Log:
OPENNLP-497 Fixed maven script to build distributions
better handling of cases where models are unavailable
Modified:
opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java
opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
Modified: opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java?rev=1393226&r1=1393225&r2=1393226&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java Tue Oct 2 22:46:54 2012
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.logging.Logger;
import opennlp.tools.similarity.apps.utils.FileHandler;
+import opennlp.tools.textsimilarity.TextProcessor;
import opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcessor;
//import com.thoughtworks.xstream.XStream;
@@ -66,8 +67,14 @@ public class TaxoQuerySnapshotMatcher {
query = query.toLowerCase();
snapshot = snapshot.toLowerCase();
- String[] queryWords = sm.getTokenizer().tokenize(query);
- String[] snapshotWords = sm.getTokenizer().tokenize(snapshot);
+ String[] queryWords = null, snapshotWords = null;
+ try {
+ queryWords = sm.getTokenizer().tokenize(query);
+ snapshotWords = sm.getTokenizer().tokenize(snapshot);
+ } catch (Exception e) { // if OpenNLP model is unavailable, use different tokenizer
+ queryWords = TextProcessor.fastTokenize(query, false).toArray(new String[0]);
+ snapshotWords = TextProcessor.fastTokenize(snapshot, false).toArray(new String[0]);
+ }
List<String> queryList = Arrays.asList(queryWords);
List<String> snapshotList = Arrays.asList(snapshotWords);
Modified: opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java?rev=1393226&r1=1393225&r2=1393226&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java Tue Oct 2 22:46:54 2012
@@ -178,8 +178,6 @@ public class ParserChunker2MatcherProces
// be
// separated correctly
- // paragraph = TextNormalizer.normalizeText(paragraph);
-
// parse sentence by sentence
String[] sentences = splitSentences(paragraph);
List<Parse> parseList = new ArrayList<Parse>(sentences.length);
@@ -209,19 +207,13 @@ public class ParserChunker2MatcherProces
if (sentence == null || sentence.trim().length() < MIN_SENTENCE_LENGTH)
return null;
- // if (normalizeText)
- // sentence = TextNormalizer.normalizeText(sentence);
-
Parse[] parseArray = null;
try {
parseArray = ParserTool.parseLine(sentence, parser, 1);
} catch (Throwable t) {
- LOG.log(Level.WARNING, "failed to parse the sentence : '" + sentence, t);
+ LOG.log(Level.WARNING, "failed to parse the sentence : '" + sentence); //, t);
return null;
}
-
- // Parse[] chunks = ChunkSampleStream.getInitialChunks(parseArray[0]) ;
-
// there should be only one result parse
if (parseArray != null && parseArray.length > 0)
return parseArray[0];
@@ -586,13 +578,13 @@ public class ParserChunker2MatcherProces
SentenceModel model = new SentenceModel(is);
sentenceDetector = new SentenceDetectorME(model);
} catch (IOException e) {
- e.printStackTrace();
+ //e.printStackTrace();
} finally {
if (is != null) {
try {
is.close();
} catch (IOException e) {
- e.printStackTrace();
+ // we swallow exception to support the cached run
}
}
}
@@ -605,12 +597,12 @@ public class ParserChunker2MatcherProces
TokenizerModel model = new TokenizerModel(is);
tokenizer = new TokenizerME(model);
} catch (IOException e) {
- e.printStackTrace();
+ // we swallow exception to support the cached run
} finally {
if (is != null) {
try {
is.close();
- } catch (IOException e) {
+ } catch (IOException e) { // we swallow exception to support the cached run
}
}
}
@@ -623,7 +615,7 @@ public class ParserChunker2MatcherProces
POSModel model = new POSModel(is);
posTagger = new POSTaggerME(model);
} catch (IOException e) {
- e.printStackTrace();
+ // we swallow exception to support the cached run
} finally {
if (is != null) {
try {
@@ -641,12 +633,12 @@ public class ParserChunker2MatcherProces
ParserModel model = new ParserModel(is);
parser = ParserFactory.create(model);
} catch (IOException e) {
- e.printStackTrace();
+ //e.printStackTrace();
} finally {
if (is != null) {
try {
is.close();
- } catch (IOException e) {
+ } catch (IOException e) { // we swallow exception to support the cached run
}
}
}
@@ -659,12 +651,12 @@ public class ParserChunker2MatcherProces
ChunkerModel model = new ChunkerModel(is);
chunker = new ChunkerME(model);
} catch (IOException e) {
- e.printStackTrace();
+ //e.printStackTrace();
} finally {
if (is != null) {
try {
is.close();
- } catch (IOException e) {
+ } catch (IOException e) { // we swallow exception to support the cached run
}
}
}