You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2023/07/13 16:23:04 UTC
[opennlp-sandbox] 01/01: conducts code cleanup of some sandbox classes removes unnecessary duplicates modernizes some code style towards Java 8+ adds missing break statements in loops to terminate earlier simplifies some methods fixes problematic JavaDoc removes unused imports
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch conduct_code_cleanup
in repository https://gitbox.apache.org/repos/asf/opennlp-sandbox.git
commit 5ac984626156151e413574cf0d567b90f378bb0f
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Thu Jul 13 18:22:55 2023 +0200
conducts code cleanup of some sandbox classes
removes unnecessary duplicates
modernizes some code style towards Java 8+
adds missing break statements in loops to terminate earlier
simplifies some methods
fixes problematic JavaDoc
removes unused imports
---
.../caseditor/CorpusExplorerView.java | 4 -
.../corpus_server/caseditor/TaskQueueView.java | 2 -
.../apache/opennlp/caseditor/TypeInputDialog.java | 1 -
.../ConfirmedNameDetectionFieldEditor.java | 2 -
.../caseditor/namefinder/NameFinderViewPage.java | 2 -
.../corpus_server/impl/LuceneSearchService.java | 27 +--
.../languagemodel/NaiveSentenceLanguageModel.java | 2 +-
.../tools/coref/resolver/ResolverUtils.java | 5 +-
.../muc/Muc6FullParseCorefSampleStreamFactory.java | 1 -
.../java/opennlp/tools/dl/NeuralDocCatTrainer.java | 14 +-
.../apps/object_dedup/SimilarityAccessorBase.java | 2 +-
.../apps/relevanceVocabs/PhraseProcessor.java | 2 +-
.../review_builder/MinedSentenceProcessor.java | 19 +-
.../apps/review_builder/ReviewBuilderRunner.java | 10 +-
.../review_builder/WebPageReviewExtractor.java | 6 +-
.../enron_email_recognizer/EmailNormalizer.java | 6 +-
.../EmailTrainingSetFormer.java | 6 +-
.../JSMLearnerOnLatticeWithDeduction.java | 8 +-
.../opennlp/tools/nl2code/NL2ObjCreateAssign.java | 11 +-
.../parse_thicket/apps/MinedSentenceProcessor.java | 207 ---------------------
.../apps/MultiSentenceSearchResultsProcessor.java | 2 +-
.../parse_thicket/apps/SnippetToParagraph.java | 9 -
.../apps/WebPageContentSentenceExtractor.java | 15 --
.../matching/PhraseGroupGeneralizer.java | 2 +-
.../pattern_structure/PhrasePatternStructure.java | 18 +-
.../tools/similarity/apps/ContentGenerator.java | 7 +-
.../similarity/apps/ContentGeneratorSupport.java | 7 +-
.../similarity/apps/RelatedSentenceFinder.java | 7 +-
.../taxo_builder/TaxonomyExtenderViaMebMining.java | 8 +-
.../tools/similarity/apps/utils/ValueSortMap.java | 18 +-
.../ParseTreeMatcherDeterministic.java | 2 +-
.../textsimilarity/SentencePairMatchResult.java | 3 +-
.../tools/word2vec/W2VDistanceMeasurer.java | 5 +-
.../datareader/SemcorReaderExtended.java | 14 +-
34 files changed, 96 insertions(+), 358 deletions(-)
diff --git a/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/CorpusExplorerView.java b/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/CorpusExplorerView.java
index 352d115..dbb193a 100644
--- a/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/CorpusExplorerView.java
+++ b/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/CorpusExplorerView.java
@@ -24,16 +24,12 @@ import org.eclipse.core.runtime.jobs.JobChangeAdapter;
import org.eclipse.jface.layout.GridDataFactory;
import org.eclipse.jface.preference.IPreferenceStore;
import org.eclipse.jface.viewers.ILabelProviderListener;
-import org.eclipse.jface.viewers.IOpenListener;
import org.eclipse.jface.viewers.ITableLabelProvider;
-import org.eclipse.jface.viewers.OpenEvent;
import org.eclipse.jface.viewers.StructuredSelection;
import org.eclipse.jface.viewers.TableViewer;
import org.eclipse.swt.SWT;
import org.eclipse.swt.events.KeyEvent;
import org.eclipse.swt.events.KeyListener;
-import org.eclipse.swt.events.ModifyEvent;
-import org.eclipse.swt.events.ModifyListener;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.events.SelectionListener;
import org.eclipse.swt.graphics.Image;
diff --git a/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/TaskQueueView.java b/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/TaskQueueView.java
index 53c8d87..418a9a8 100644
--- a/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/TaskQueueView.java
+++ b/caseditor-corpus-server-plugin/src/main/java/org/apache/opennlp/corpus_server/caseditor/TaskQueueView.java
@@ -28,9 +28,7 @@ import javax.ws.rs.core.Response;
import org.eclipse.jface.layout.GridDataFactory;
import org.eclipse.jface.viewers.ILabelProviderListener;
-import org.eclipse.jface.viewers.IOpenListener;
import org.eclipse.jface.viewers.ITableLabelProvider;
-import org.eclipse.jface.viewers.OpenEvent;
import org.eclipse.jface.viewers.StructuredSelection;
import org.eclipse.jface.viewers.TableViewer;
import org.eclipse.swt.SWT;
diff --git a/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/TypeInputDialog.java b/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/TypeInputDialog.java
index 6c37546..a903da4 100644
--- a/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/TypeInputDialog.java
+++ b/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/TypeInputDialog.java
@@ -18,7 +18,6 @@
package org.apache.opennlp.caseditor;
import org.apache.uima.cas.TypeSystem;
-import org.eclipse.jface.dialogs.IInputValidator;
import org.eclipse.jface.dialogs.InputDialog;
import org.eclipse.swt.widgets.Shell;
diff --git a/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/ConfirmedNameDetectionFieldEditor.java b/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/ConfirmedNameDetectionFieldEditor.java
index e70ec1a..8eb4346 100644
--- a/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/ConfirmedNameDetectionFieldEditor.java
+++ b/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/ConfirmedNameDetectionFieldEditor.java
@@ -23,9 +23,7 @@ import org.eclipse.swt.SWT;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.Button;
import org.eclipse.swt.widgets.Composite;
-import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Group;
-import org.eclipse.swt.widgets.Listener;
import static org.apache.opennlp.caseditor.OpenNLPPreferenceConstants.*;
diff --git a/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderViewPage.java b/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderViewPage.java
index 1e86482..312e604 100644
--- a/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderViewPage.java
+++ b/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderViewPage.java
@@ -32,8 +32,6 @@ import org.eclipse.jface.action.IAction;
import org.eclipse.jface.action.IToolBarManager;
import org.eclipse.jface.preference.IPreferenceStore;
import org.eclipse.jface.viewers.ISelection;
-import org.eclipse.jface.viewers.ISelectionChangedListener;
-import org.eclipse.jface.viewers.SelectionChangedEvent;
import org.eclipse.jface.viewers.StructuredSelection;
import org.eclipse.jface.viewers.TableViewer;
import org.eclipse.jface.viewers.TableViewerColumn;
diff --git a/corpus-server/corpus-server-impl/src/main/java/org/apache/opennlp/corpus_server/impl/LuceneSearchService.java b/corpus-server/corpus-server-impl/src/main/java/org/apache/opennlp/corpus_server/impl/LuceneSearchService.java
index d4b3672..d8fceca 100644
--- a/corpus-server/corpus-server-impl/src/main/java/org/apache/opennlp/corpus_server/impl/LuceneSearchService.java
+++ b/corpus-server/corpus-server-impl/src/main/java/org/apache/opennlp/corpus_server/impl/LuceneSearchService.java
@@ -105,37 +105,18 @@ public class LuceneSearchService implements SearchService {
File mappingTmpFile = File.createTempFile("lucas-mapping", corpusId + ".xml");
mappingTmpFile.deleteOnExit();
- InputStream mappingFileIn = new ByteArrayInputStream(corpusStore.getIndexMapping());
- OutputStream mappingTmpOut = null;
-
- try {
- mappingTmpOut = new FileOutputStream(mappingTmpFile);
-
+ try (InputStream mappingFileIn = new ByteArrayInputStream(corpusStore.getIndexMapping());
+ OutputStream mappingTmpOut = new FileOutputStream(mappingTmpFile)) {
+
byte[] buffer = new byte[1024];
int len;
while ((len = mappingFileIn.read(buffer)) > 0) {
mappingTmpOut.write(buffer, 0, len);
}
- }
- catch (IOException e) {
+ } catch (IOException e) {
// TODO: Or just ignore it ?! and do not create the indexer for this corpus?!
throw e;
}
- finally {
- if (mappingFileIn != null) {
- try {
- mappingFileIn.close();
- }
- catch (IOException e) {}
- }
-
- if (mappingTmpOut != null) {
- try {
- mappingTmpOut.close();
- }
- catch (IOException e) {}
- }
- }
specifier.getAnalysisEngineMetaData().
getConfigurationParameterSettings().setParameterValue("mappingFile",
diff --git a/nlp-utils/src/main/java/org/apache/opennlp/utils/languagemodel/NaiveSentenceLanguageModel.java b/nlp-utils/src/main/java/org/apache/opennlp/utils/languagemodel/NaiveSentenceLanguageModel.java
index d29b2e5..6700cda 100644
--- a/nlp-utils/src/main/java/org/apache/opennlp/utils/languagemodel/NaiveSentenceLanguageModel.java
+++ b/nlp-utils/src/main/java/org/apache/opennlp/utils/languagemodel/NaiveSentenceLanguageModel.java
@@ -29,7 +29,7 @@ public class NaiveSentenceLanguageModel<T> implements LanguageModel<T[]> {
@Override
public double calculateProbability(Collection<T[]> vocabulary, T[] sentence) {
- return vocabulary.isEmpty() ? 0 : Collections.frequency(vocabulary, sentence) / vocabulary.size();
+ return vocabulary.isEmpty() ? 0 : (double) Collections.frequency(vocabulary, sentence) / vocabulary.size();
}
}
diff --git a/opennlp-coref/src/main/java/opennlp/tools/coref/resolver/ResolverUtils.java b/opennlp-coref/src/main/java/opennlp/tools/coref/resolver/ResolverUtils.java
index 2cd657a..31fe435 100644
--- a/opennlp-coref/src/main/java/opennlp/tools/coref/resolver/ResolverUtils.java
+++ b/opennlp-coref/src/main/java/opennlp/tools/coref/resolver/ResolverUtils.java
@@ -547,10 +547,7 @@ public class ResolverUtils {
*/
public static boolean definiteArticle(String tok, String tag) {
tok = tok.toLowerCase();
- if (tok.equals("the") || tok.equals("these") || tok.equals("these") || tag.equals("PRP$")) {
- return (true);
- }
- return (false);
+ return tok.equals("the") || tok.equals("these") || tok.equals("those") || tag.equals("PRP$");
}
public static String getNumberCompatibilityFeature(MentionContext ec, DiscourseEntity de) {
diff --git a/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java b/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java
index b83c522..f6206b6 100644
--- a/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java
+++ b/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java
@@ -18,7 +18,6 @@
package opennlp.tools.formats.muc;
import java.io.File;
-import java.io.FileFilter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
diff --git a/opennlp-dl/src/main/java/opennlp/tools/dl/NeuralDocCatTrainer.java b/opennlp-dl/src/main/java/opennlp/tools/dl/NeuralDocCatTrainer.java
index dd2c39c..e162417 100644
--- a/opennlp-dl/src/main/java/opennlp/tools/dl/NeuralDocCatTrainer.java
+++ b/opennlp-dl/src/main/java/opennlp/tools/dl/NeuralDocCatTrainer.java
@@ -55,24 +55,24 @@ public class NeuralDocCatTrainer {
public static class Args {
@Option(name = "-batchSize", usage = "Number of examples in minibatch")
- int batchSize = 128;
+ final int batchSize = 128;
@Option(name = "-nEpochs", usage = "Number of epochs (i.e. full passes over the training data) to train on." +
" Applicable for training only.")
- int nEpochs = 2;
+ final int nEpochs = 2;
@Option(name = "-maxSeqLen", usage = "Max Sequence Length. Sequences longer than this will be truncated")
- int maxSeqLen = 256; //Truncate text with length (# words) greater than this
+ final int maxSeqLen = 256; //Truncate text with length (# words) greater than this
@Option(name = "-vocabSize", usage = "Vocabulary Size.")
- int vocabSize = 20000; //vocabulary size
+ final int vocabSize = 20000; //vocabulary size
@Option(name = "-nRNNUnits", usage = "Number of RNN cells to use.")
- int nRNNUnits = 128;
+ final int nRNNUnits = 128;
@Option(name = "-lr", aliases = "-learnRate", usage = "Learning Rate." +
" Adjust it when the scores bounce to NaN or Infinity.")
- double learningRate = 2e-3;
+ final double learningRate = 2e-3;
@Option(name = "-glovesPath", required = true, usage = "Path to GloVe vectors file." +
" Download and unzip from https://nlp.stanford.edu/projects/glove/")
@@ -87,7 +87,7 @@ public class NeuralDocCatTrainer {
String trainDir = null;
@Option(name = "-validDir", usage = "Path to validation data directory. Optional.")
- String validDir = null;
+ final String validDir = null;
@Option(name = "-labels", required = true, handler = StringArrayOptionHandler.class,
usage = "Names of targets or labels separated by spaces. " +
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/apps/object_dedup/SimilarityAccessorBase.java b/opennlp-similarity/src/main/java/opennlp/tools/apps/object_dedup/SimilarityAccessorBase.java
index a69ba08..7ec4c26 100755
--- a/opennlp-similarity/src/main/java/opennlp/tools/apps/object_dedup/SimilarityAccessorBase.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/apps/object_dedup/SimilarityAccessorBase.java
@@ -95,7 +95,7 @@ public class SimilarityAccessorBase {
public List<String> getWordsThatShouldBeOnBothSidesEvents()
{
-/*
+ /*
names.addAll(Arrays.asList(new String[] { "woman", "man", "women", "men", "womans", "mans", "womens", "mens",
"boy", "girl", "boys", "girls", "men's", "women's", "woman's", "ice", // for disney
"flight", "intermediate", "advanced", "beginner",
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/apps/relevanceVocabs/PhraseProcessor.java b/opennlp-similarity/src/main/java/opennlp/tools/apps/relevanceVocabs/PhraseProcessor.java
index 8b00ff9..013ad90 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/apps/relevanceVocabs/PhraseProcessor.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/apps/relevanceVocabs/PhraseProcessor.java
@@ -143,7 +143,7 @@ public class PhraseProcessor {
&& phraseBeingFormed )
break;
else if (ch.getPOSs().get(i).startsWith("DT") || ch.getPOSs().get(i).startsWith("CC"))
- continue;
+ continue;
}
query = new StringBuilder(query.toString().trim());
int len = query.toString().split(" ").length;
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/MinedSentenceProcessor.java b/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/MinedSentenceProcessor.java
index dcb3ef4..27b3f49 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/MinedSentenceProcessor.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/MinedSentenceProcessor.java
@@ -22,6 +22,7 @@ import org.apache.commons.lang.StringUtils;
import opennlp.tools.similarity.apps.utils.Utils;
public class MinedSentenceProcessor {
+
public static String acceptableMinedSentence(String sent) {
// if too many commas => seo text
@@ -54,7 +55,6 @@ public class MinedSentenceProcessor {
return null;
}
- String[] pipes = StringUtils.split(sent, '|');
if (StringUtils.split(sent, '|').length > 2
|| StringUtils.split(sent, '>').length > 2) {
System.out.println("Rejection: too many |s or >s ");
@@ -93,7 +93,6 @@ public class MinedSentenceProcessor {
|| sentTry.contains("[edit")
|| sentTry.contains("edit categories")
|| sentTry.contains("free license")
- || sentTry.contains("permission is granted")
|| sentTry.contains("under the terms")
|| sentTry.contains("rights reserved")
|| sentTry.contains("wikipedia") || sentTry.endsWith("the")
@@ -101,19 +100,19 @@ public class MinedSentenceProcessor {
|| sentTry.contains("recipient of") || sentTry.contains("this message")
|| sentTry.contains("mailing list") || sentTry.contains("purchase order")
|| sentTry.contains("mon-fri") || sentTry.contains("email us") || sentTry.contains("privacy pol") || sentTry.contains("back to top")
- || sentTry.contains("click here") || sentTry.contains("for details") || sentTry.contains("assistance?") || sentTry.contains("chat live")
+ || sentTry.contains("for details") || sentTry.contains("assistance?") || sentTry.contains("chat live")
|| sentTry.contains("free shipping") || sentTry.contains("company info") || sentTry.contains("satisfaction g") || sentTry.contains("contact us")
- ||sentTry.startsWith( "fax") ||sentTry.startsWith( "write") || sentTry.startsWith( "email")|| sentTry.contains("conditions") || sentTry.contains("chat live")
- ||sentTry.startsWith( "we ") || sentTry.contains("the recipient") || sentTry.contains("day return") || sentTry.contains("days return")
+ ||sentTry.startsWith("write") || sentTry.startsWith( "email")|| sentTry.contains("conditions")
+ ||sentTry.startsWith("we ") || sentTry.contains("the recipient") || sentTry.contains("day return") || sentTry.contains("days return")
- ||sentTry.startsWith( "fax") || sentTry.contains("refund it") || sentTry.contains("your money")
- ||sentTry.startsWith( "free") || sentTry.contains("purchase orders")
- ||sentTry.startsWith( "exchange it ") || sentTry.contains("return it") || sentTry.contains("credit card")
+ ||sentTry.startsWith("fax") || sentTry.contains("refund it") || sentTry.contains("your money")
+ ||sentTry.startsWith("free") || sentTry.contains("purchase orders")
+ ||sentTry.startsWith("exchange it ") || sentTry.contains("return it") || sentTry.contains("credit card")
|| sentTry.contains("storeshop") || sentTry.startsWith( "find") || sentTry.startsWith( "shop") || sentTry.startsWith( "unlimited")
|| sentTry.contains("for a limited time") || sentTry.contains("prime members") || sentTry.contains("amazon members") || sentTry.contains("unlimited free")
|| sentTry.contains("shipping") || sentTry.startsWith( "amazon")
-// not a script text
+ // not a script text
|| sentTry.contains("document.body") || sentTry.contains(" var ") || sentTry.contains("search suggestions") ||sentTry.startsWith( "Search")
)
@@ -194,7 +193,7 @@ public class MinedSentenceProcessor {
public static String normalizeForSentenceSplitting(String pageContent) {
- pageContent.replace("Jan.", "January").replace("Feb.", "February")
+ pageContent = pageContent.replace("Jan.", "January").replace("Feb.", "February")
.replace("Mar.", "March").replace("Apr.", "April")
.replace("Jun.", "June").replace("Jul.", "July")
.replace("Aug.", "August").replace("Sep.", "September")
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/ReviewBuilderRunner.java b/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/ReviewBuilderRunner.java
index 6a11ad7..1cb64b2 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/ReviewBuilderRunner.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/ReviewBuilderRunner.java
@@ -25,7 +25,7 @@ import opennlp.tools.parse_thicket.Triple;
public class ReviewBuilderRunner {
- private final List<Triple> input = new ArrayList<>();
+ private final List<Triple<String, Integer, String>> input = new ArrayList<>();
public ReviewBuilderRunner(){
@@ -168,14 +168,10 @@ public class ReviewBuilderRunner {
//ProductFinderInAWebPage init = new ProductFinderInAWebPage("C:/workspace/relevanceEngine/src/test/resources");
ReviewBuilderRunner r = new ReviewBuilderRunner();
WebPageReviewExtractor extractor = new WebPageReviewExtractor("C:/workspace/relevanceEngine/src/test/resources");
- for(Triple query_ID : r.input ){
- String query = (String) query_ID.getFirst();
+ for(Triple<String, Integer, String> query_ID : r.input ){
+ String query = query_ID.getFirst();
List<String> res = extractor.formReviewsForAProduct(query);
-
ProfileReaderWriter.writeReportListStr(res, "formedReviewSentences"+ query +".csv");
}
-
-
-
}
}
\ No newline at end of file
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/WebPageReviewExtractor.java b/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/WebPageReviewExtractor.java
index 078d56c..4448f58 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/WebPageReviewExtractor.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/apps/review_builder/WebPageReviewExtractor.java
@@ -170,7 +170,7 @@ public class WebPageReviewExtractor extends WebPageExtractor {
private List<String> cleanProductFeatures(List<String> productFeaturesList) {
List<String> results = new ArrayList<>();
for (String feature: productFeaturesList) {
- if (feature.startsWith("Unlimited Free") || feature.startsWith("View Larger") || feature.startsWith("View Larger") || feature.indexOf("shipping")>0)
+ if (feature.startsWith("Unlimited Free") || feature.startsWith("View Larger") || feature.indexOf("shipping")>0)
continue;
results.add(feature);
}
@@ -324,7 +324,7 @@ public class WebPageReviewExtractor extends WebPageExtractor {
if (count%2==0 && count<features.length)
if (features[count]!=null){
buf.append(features[count]);
- if (!(features[count].endsWith("!") ||features[count].endsWith("?")||features[count].endsWith("?")
+ if (!(features[count].endsWith("!") ||features[count].endsWith("?")
||features[count].endsWith(".\"") ))
buf.append(". ");
}
@@ -364,7 +364,7 @@ public class WebPageReviewExtractor extends WebPageExtractor {
if (count%2==0 && count<features.length)
if (features[count]!=null){
bufs[currentRevIndex].append(features[count]);
- if (!(features[count].endsWith("!") ||features[count].endsWith("?")||features[count].endsWith("?")
+ if (!(features[count].endsWith("!") ||features[count].endsWith("?")
||features[count].endsWith(".\"") ))
bufs[currentRevIndex].append(". ");
}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailNormalizer.java b/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailNormalizer.java
index 808788f..6e1ebe9 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailNormalizer.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailNormalizer.java
@@ -76,13 +76,15 @@ public class EmailNormalizer {
for(String l: lines){
boolean bAccept = true;
for(String h: HEADERS){
- if (l.startsWith(h)){
+ if (l.startsWith(h)) {
bAccept = false;
+ break;
}
}
for(String h: PROHIBITED_STRINGS){
- if (l.indexOf(h)>0){
+ if (l.indexOf(h) > 0) {
bAccept = false;
+ break;
}
}
if (bAccept)
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailTrainingSetFormer.java b/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailTrainingSetFormer.java
index 1a2f89e..1a8ce6d 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailTrainingSetFormer.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/enron_email_recognizer/EmailTrainingSetFormer.java
@@ -61,13 +61,15 @@ public class EmailTrainingSetFormer {
for(String l: lines){
boolean bAccept = true;
for(String h: EmailNormalizer.HEADERS){
- if (l.startsWith(h)){
+ if (l.startsWith(h)) {
bAccept = false;
+ break;
}
}
for(String h: EmailNormalizer.PROHIBITED_STRINGS){
- if (l.indexOf(h)>0){
+ if (l.indexOf(h) > 0) {
bAccept = false;
+ break;
}
}
if (bAccept)
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithDeduction.java b/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithDeduction.java
index 03e8b1e..d9c8b83 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithDeduction.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithDeduction.java
@@ -179,8 +179,10 @@ public class JSMLearnerOnLatticeWithDeduction extends JSMLearnerOnLatticeBase{
for(String posText:posTexts){
boolean multiwordOccurs = true;
for(String keyword: keywords){
- if (!posText.contains(keyword))
+ if (!posText.contains(keyword)) {
multiwordOccurs = false;
+ break;
+ }
break;
}
if (multiwordOccurs)
@@ -191,8 +193,10 @@ public class JSMLearnerOnLatticeWithDeduction extends JSMLearnerOnLatticeBase{
for(String negText:negTexts){
boolean multiwordOccurs = true;
for(String keyword: keywords){
- if (!negText.contains(keyword))
+ if (!negText.contains(keyword)) {
multiwordOccurs = false;
+ break;
+ }
break;
}
if (multiwordOccurs)
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/nl2code/NL2ObjCreateAssign.java b/opennlp-similarity/src/main/java/opennlp/tools/nl2code/NL2ObjCreateAssign.java
index 0e5811e..1a16adb 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/nl2code/NL2ObjCreateAssign.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/nl2code/NL2ObjCreateAssign.java
@@ -60,7 +60,6 @@ public class NL2ObjCreateAssign extends NL2Obj {
sentence = sentence.split(":")[0]+".";
}
-
List<ObjectPhrase> oPhrases = new ArrayList<>();
parser = ParserChunker2MatcherProcessor.getInstance();
List<List<ParseTreeChunk>> lingPhrases =
@@ -79,8 +78,8 @@ public class NL2ObjCreateAssign extends NL2Obj {
List<String> lems = verbChunk.getLemmas();
String declarativeAction = verbChunk.getLemmas().get(0).toLowerCase();
if (declarativeAction.equals("define")){
- if (verbChunk.getLemmas().get(1).toLowerCase().equals("class") ||
- verbChunk.getLemmas().get(2).toLowerCase().equals("class")){
+ if (verbChunk.getLemmas().get(1).equalsIgnoreCase("class") ||
+ verbChunk.getLemmas().get(2).equalsIgnoreCase("class")){
// new class
String className = verbChunk.getLemmas().get(verbChunk.getLemmas().size()-1).toLowerCase();
className = className.substring(0, 1).toUpperCase()+className.substring(1, className.length());
@@ -91,12 +90,12 @@ public class NL2ObjCreateAssign extends NL2Obj {
}
String dataType = verbChunk.getLemmas().get(1).toLowerCase();
- if (classBeingDefined && Arrays.asList(DATA_TYPES_LIST).contains(dataType) && verbChunk.getLemmas().get(2).toLowerCase().equals("attribute")){
+ if (classBeingDefined && Arrays.asList(DATA_TYPES_LIST).contains(dataType) && verbChunk.getLemmas().get(2).equalsIgnoreCase("attribute")){
op.setOperatorFor(dataType + " "+verbChunk.getLemmas().get(verbChunk.getLemmas().size()-1).toLowerCase());
classBeingDefined = true;
break;
}
- if (Arrays.asList(DATA_TYPES_LIST).contains(dataType) && verbChunk.getLemmas().get(2).toLowerCase().equals("attribute")){
+ if (Arrays.asList(DATA_TYPES_LIST).contains(dataType) && verbChunk.getLemmas().get(2).equalsIgnoreCase("attribute")){
op.setOperatorFor(dataType + " "+verbChunk.getLemmas().get(verbChunk.getLemmas().size()-1).toLowerCase());
classBeingDefined = true;
break;
@@ -104,7 +103,7 @@ public class NL2ObjCreateAssign extends NL2Obj {
} else if (declarativeAction.equals("create")){
// now substituting array
- if (verbChunk.getLemmas().get(1).toLowerCase().equals("array")){
+ if (verbChunk.getLemmas().get(1).equalsIgnoreCase("array")){
if(lems.contains("class")){
int indClass = lems.indexOf("class");
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/MinedSentenceProcessor.java b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/MinedSentenceProcessor.java
deleted file mode 100644
index fddef77..0000000
--- a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/MinedSentenceProcessor.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.parse_thicket.apps;
-
-import opennlp.tools.similarity.apps.utils.Utils;
-
-import org.apache.commons.lang.StringUtils;
-
-public class MinedSentenceProcessor {
- public static String acceptableMinedSentence(String sent) {
- // if too many commas => seo text
-
- String[] commas = StringUtils.split(sent, ',');
- String[] spaces = StringUtils.split(sent, ' ');
- if ((float) commas.length / (float) spaces.length > 0.7) {
- System.out.println("Rejection: too many commas");
- return null;
- }
-
- String[] otherDelimiters = StringUtils.split(sent, '/');
- if ((float) otherDelimiters.length / (float) spaces.length > 0.7) {
- System.out.println("Rejection: too many delimiters");
- return null;
- }
-
- otherDelimiters = StringUtils.split(sent, '.');
- if ((float) otherDelimiters.length / (float) spaces.length > 0.7) {
- System.out.println("Rejection: too many delimiters");
- return null;
- }
- otherDelimiters = StringUtils.split(sent, '!');
- if ((float) otherDelimiters.length / (float) spaces.length > 0.7) {
- System.out.println("Rejection: too many delimiters");
- return null;
- }
- otherDelimiters = StringUtils.split(sent, '=');
- if ((float) otherDelimiters.length / (float) spaces.length > 0.7) {
- System.out.println("Rejection: too many delimiters");
- return null;
- }
-
- String[] pipes = StringUtils.split(sent, '|');
- if (StringUtils.split(sent, '|').length > 2
- || StringUtils.split(sent, '>').length > 2) {
- System.out.println("Rejection: too many |s or >s ");
- return null;
- }
- String sentTry = sent.toLowerCase();
- // if too many long spaces
- String sentSpaces = sentTry.replace(" ", "");
- if (sentSpaces.length() - sentTry.length() > 10) // too many spaces -
- // suspicious
- return null;
-
- if (sentTry.contains("click here") || sentTry.contains(" wikip")
- || sentTry.contains("copyright")
- || sentTry.contains("operating hours")
- || sentTry.contains("days per week")
- || sentTry.contains("click for") || sentTry.contains("photos")
- || sentTry.contains("find the latest")
- || sentTry.startsWith("subscribe")
- || sentTry.contains("Terms of Service")
- || sentTry.contains("clicking here")
- || sentTry.contains("skip to") || sentTry.contains("sidebar")
- || sentTry.contains("Tags:") || sentTry.startsWith("Posted by")
- || sentTry.contains("available online")
- || sentTry.contains("get online")
- || sentTry.contains("buy online")
- || sentTry.contains("not valid") || sentTry.contains("discount")
- || sentTry.contains("official site")
- || sentTry.contains("this video")
- || sentTry.contains("this book")
- || sentTry.contains("this product")
- || sentTry.contains("paperback") || sentTry.contains("hardcover")
- || sentTry.contains("audio cd")
- || sentTry.contains("related searches")
- || sentTry.contains("permission is granted")
- || sentTry.contains("[edit")
- || sentTry.contains("edit categories")
- || sentTry.contains("free license")
- || sentTry.contains("permission is granted")
- || sentTry.contains("under the terms")
- || sentTry.contains("rights reserved")
- || sentTry.contains("wikipedia") || sentTry.endsWith("the")
- || sentTry.endsWith("the.") || sentTry.startsWith("below")
- || sentTry.contains("recipient of") || sentTry.contains("this message")
- || sentTry.contains("mailing list") || sentTry.contains("purchase order")
- || sentTry.contains("mon-fri") || sentTry.contains("email us") || sentTry.contains("privacy pol") || sentTry.contains("back to top")
- || sentTry.contains("click here") || sentTry.contains("for details") || sentTry.contains("assistance?") || sentTry.contains("chat live")
- || sentTry.contains("free shipping") || sentTry.contains("company info") || sentTry.contains("satisfaction g") || sentTry.contains("contact us")
- ||sentTry.startsWith( "fax") ||sentTry.startsWith( "write") || sentTry.startsWith( "email")|| sentTry.contains("conditions") || sentTry.contains("chat live")
- ||sentTry.startsWith( "we ") || sentTry.contains("the recipient") || sentTry.contains("day return") || sentTry.contains("days return")
-
- ||sentTry.startsWith( "fax") || sentTry.contains("refund it") || sentTry.contains("your money")
- ||sentTry.startsWith( "free") || sentTry.contains("purchase orders")
- ||sentTry.startsWith( "exchange it ") || sentTry.contains("return it") || sentTry.contains("credit card")
-
- || sentTry.contains("storeshop") || sentTry.startsWith( "find") || sentTry.startsWith( "shop") || sentTry.startsWith( "unlimited")
- || sentTry.contains("for a limited time") || sentTry.contains("prime members") || sentTry.contains("amazon members") || sentTry.contains("unlimited free")
- || sentTry.contains("shipping") || sentTry.startsWith( "amazon")
-// not a script text
- || sentTry.contains("document.body") || sentTry.contains(" var ") || sentTry.contains("search suggestions") ||sentTry.startsWith( "Search")
-
- )
- return null;
-
- //Millions of Amazon Prime members enjoy instant videos, free Kindle books and unlimited free two-day shipping.
-
- // count symbols indicating wrong parts of page to mine for text
- // if short and contains too many symbols indicating wrong area: reject
- String sentWrongSym = sentTry.replace(">", "&&&").replace("�", "&&&")
- .replace("|", "&&&").replace(":", "&&&").replace("/", "&&&")
- .replace("-", "&&&").replace("%", "&&&");
- if ((sentWrongSym.length() - sentTry.length()) >= 4
- && sentTry.length() < 200) // twice ot more
- return null;
-
- sent = sent.replace('[', ' ').replace(']', ' ')
- .replace("_should_find_orig_", "").replace(". .", ". ")
- .replace("amp;", " ").replace("1.", " ").replace("2.", " ")
- .replace("3.", " ").replace("4.", " ").replace("2009", "2011")
- .replace("2008", "2011").replace("2006", "2011")
- .replace("2007", "2011").replace("VIDEO:", " ").replace("Video:", " ")
- .replace("no comments", " ").replace(" ", " ").replace(" ", " ")
- .replace("(more.)", "").replace("more.", "").replace("<more>", "")
- .replace("[more]", "").replace(".,", ".").replace("<", "")
- .replace("p>", "").replace("product description", "");
-
- // TODO .replace("a.", ".");
-
- int endIndex = sent.indexOf(" posted");
- if (endIndex > 0)
- sent = sent.substring(0, endIndex);
-
- return sent;
- }
-
- public static String processSentence(String pageSentence) {
- if (pageSentence == null)
- return "";
- pageSentence = Utils.fullStripHTML(pageSentence);
- pageSentence = StringUtils.chomp(pageSentence, "..");
- pageSentence = StringUtils.chomp(pageSentence, ". .");
- pageSentence = StringUtils.chomp(pageSentence, " .");
- pageSentence = StringUtils.chomp(pageSentence, ".");
- pageSentence = StringUtils.chomp(pageSentence, "...");
- pageSentence = StringUtils.chomp(pageSentence, " ....");
- pageSentence = pageSentence.replace("::", ":").replace(".,", ". ")
- .replace("(.)", "");
-
- pageSentence = pageSentence.trim();
- pageSentence = pageSentence.replaceAll("\\s+", " "); // make single
- // spaces
- // everywhere
-
- String[] pipes = StringUtils.split(pageSentence, '|'); // removed
- // shorter part
- // of sentence
- // at the end
- // after pipe
- if (pipes.length == 2
- && ((float) pipes[0].length() / (float) pipes[1].length() > 3.0)) {
- int pipePos = pageSentence.indexOf("|");
- if (pipePos > -1)
- pageSentence = pageSentence.substring(0, pipePos - 1).trim();
-
- }
-
- if (!StringUtils.contains(pageSentence, '.')
- && !StringUtils.contains(pageSentence, '?')
- && !StringUtils.contains(pageSentence, '!'))
- pageSentence = pageSentence + ". ";
-
- pageSentence = pageSentence.replace(" .", ".").replace("..", ".").trim();
- if (!pageSentence.endsWith("."))
- pageSentence += ". ";
- return pageSentence;
- }
-
-
- public static String normalizeForSentenceSplitting(String pageContent) {
- pageContent.replace("Jan.", "January").replace("Feb.", "February")
- .replace("Mar.", "March").replace("Apr.", "April")
- .replace("Jun.", "June").replace("Jul.", "July")
- .replace("Aug.", "August").replace("Sep.", "September")
- .replace("Oct.", "October").replace("Nov.", "November")
- .replace("Dec.", "December");
-
- return pageContent;
-
- }
-}
\ No newline at end of file
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/MultiSentenceSearchResultsProcessor.java b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/MultiSentenceSearchResultsProcessor.java
index 47f5fad..b027363 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/MultiSentenceSearchResultsProcessor.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/MultiSentenceSearchResultsProcessor.java
@@ -59,7 +59,7 @@ public class MultiSentenceSearchResultsProcessor {
count++;
String[] pageSentsAndSnippet = formTextForReRankingFromHit(hit);
- Double score = 0.0;
+ double score = 0.0;
try {
SentencePairMatchResult match;
if (pageSentsAndSnippet!=null && pageSentsAndSnippet[0].length()>50){
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/SnippetToParagraph.java b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/SnippetToParagraph.java
index d01fec2..c22ba9f 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/SnippetToParagraph.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/SnippetToParagraph.java
@@ -318,15 +318,6 @@ public class SnippetToParagraph extends ContentGeneratorSupport /*RelatedSentenc
return sents;
}
- private List<String> cleanProductFeatures(List<String> productFeaturesList) {
- List<String> results = new ArrayList<>();
- for(String feature: productFeaturesList){
- if (feature.startsWith("Unlimited Free") || feature.startsWith("View Larger") || feature.startsWith("View Larger") || feature.indexOf("shipping")>0)
- continue;
- results.add(feature);
- }
- return results;
- }
public static class TextChunk {
public TextChunk(String s, int length) {
this.text = s;
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/WebPageContentSentenceExtractor.java b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/WebPageContentSentenceExtractor.java
index 83851c7..df0c6c0 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/WebPageContentSentenceExtractor.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/apps/WebPageContentSentenceExtractor.java
@@ -26,9 +26,6 @@ import opennlp.tools.similarity.apps.HitBase;
import org.apache.commons.lang.StringUtils;
public class WebPageContentSentenceExtractor extends WebPageExtractor {
-
-
-
public List<String> extractSentencesWithPotentialReviewPhrases(String url)
{
@@ -84,16 +81,6 @@ public class WebPageContentSentenceExtractor extends WebPageExtractor {
return sents;
}
- private List<String> cleanProductFeatures(List<String> productFeaturesList) {
- List<String> results = new ArrayList<>();
- for(String feature: productFeaturesList){
- if (feature.startsWith("Unlimited Free") || feature.startsWith("View Larger") || feature.startsWith("View Larger") || feature.indexOf("shipping")>0)
- continue;
- results.add(feature);
- }
- return results;
- }
-
// extracts paragraphs from web page
protected String[] cleanListOfSents(String[] longestSents)
{
@@ -123,8 +110,6 @@ public class WebPageContentSentenceExtractor extends WebPageExtractor {
return sentsClean.toArray(new String[0]);
}
-
-
private String startWithCapitalSent(String sent) {
String firstChar = sent.substring(0,1);
String remainder = sent.substring(1);
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/matching/PhraseGroupGeneralizer.java b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/matching/PhraseGroupGeneralizer.java
index 01fe299..1fe19d2 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/matching/PhraseGroupGeneralizer.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/matching/PhraseGroupGeneralizer.java
@@ -37,7 +37,7 @@ public class PhraseGroupGeneralizer implements IGeneralizer<List<ParseTreeChunk>
*
* @param o1
* @param o2
- * @return List<List<ParseTreeChunk>> list of POS-words pairs for each
+ * @return {@link List ParseTreeChunk} list of POS-words pairs for each
* resultant matched / overlapped phrase
*/
@Override
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/pattern_structure/PhrasePatternStructure.java b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/pattern_structure/PhrasePatternStructure.java
index d910352..dbda0d8 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/pattern_structure/PhrasePatternStructure.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/parse_thicket/pattern_structure/PhrasePatternStructure.java
@@ -163,14 +163,18 @@ public class PhrasePatternStructure {
List<ParseTreeChunk> nps = new ArrayList<>(), vps = new ArrayList<>(), pps = new ArrayList<>();
for(List<ParseTreeNode> ps:phrs) {
ParseTreeChunk ch = convertNodeListIntoChunk(ps);
- String ptype = ps.get(0).getPhraseType();
LOG.debug(ps.toString());
- if (ptype.equals("NP")){
- nps.add(ch);
- } else if (ptype.equals("VP")){
- vps.add(ch);
- } else if (ptype.equals("PP")){
- pps.add(ch);
+ String ptype = ps.get(0).getPhraseType();
+ switch (ptype) {
+ case "NP":
+ nps.add(ch);
+ break;
+ case "VP":
+ vps.add(ch);
+ break;
+ case "PP":
+ pps.add(ch);
+ break;
}
}
results.add(nps); results.add(vps); results.add(pps);
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java
index 33f387b..d26748d 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java
@@ -146,9 +146,12 @@ public class ContentGenerator /*extends RelatedSentenceFinder*/ {
boolean bAccept = true;
for (String w : qs) {
if (w.toLowerCase().equals(w)) // idf only two words then
- // has to be person name,
- // title or geolocation
+ // has to be person name,
+ // title or geolocation
+ {
bAccept = false;
+ break;
+ }
}
if (!bAccept)
continue;
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java
index b846abf..abdb60c 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java
@@ -77,9 +77,12 @@ public class ContentGeneratorSupport {
boolean bAccept = true;
for (String w : qs) {
if (w.toLowerCase().equals(w)) // idf only two words then
- // has to be person name,
- // title or geolocation
+ // has to be person name,
+ // title or geolocation
+ {
bAccept = false;
+ break;
+ }
}
if (!bAccept)
continue;
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java
index 209e29a..2404804 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java
@@ -217,9 +217,12 @@ public class RelatedSentenceFinder {
boolean bAccept = true;
for (String w : qs) {
if (w.toLowerCase().equals(w)) // idf only two words then
- // has to be person name,
- // title or geolocation
+ // has to be person name,
+ // title or geolocation
+ {
bAccept = false;
+ break;
+ }
}
if (!bAccept)
continue;
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java
index cbd1229..4442b40 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java
@@ -34,10 +34,12 @@ import opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcess
/**
* Results of taxonomy learning are two maps 0) For an entity like tax it gives
* all lists of associated parameters obtained from the taxonomy kernel (done
- * manually) Now, given 0, we obtain the derived list of parameters as
+ * manually).
+ * <p>
+ * Now, given 0, we obtain the derived list of parameters as
* commonalities of search results snapshots output map 1) for the entity,
- * derived list output map 2) for such manual list of words -> derived list of
- * words
+ * derived list output map 2) for such manual list of words - derived list of
+ * words.
*/
public class TaxonomyExtenderViaMebMining extends BingQueryRunner {
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/ValueSortMap.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/ValueSortMap.java
index fc2fdeb..3ffcb94 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/ValueSortMap.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/ValueSortMap.java
@@ -164,10 +164,10 @@ public class ValueSortMap {
Map<K, V> hmTmpMap = new HashMap<>(iSize);
Map<K, V> hmNullValueMap = new HashMap<>();
+ keySet = inMap.keySet();
+ itKeyList = keySet.iterator();// Add Null Values in the last of the LinkedHasMap
if (bAllDistinct) {
// There are no multiple same values of the passed map (without considering null)
- keySet = inMap.keySet();
- itKeyList = keySet.iterator();
while (itKeyList.hasNext()) {
key = itKeyList.next();
value = inMap.get(key);
@@ -192,14 +192,8 @@ public class ValueSortMap {
sortedMap.put(key, value);
}
- if (ascendingOrder == null || ascendingOrder) {
- // Add Null Values in the last of the LinkedHasMap
- sortedMap.putAll(hmNullValueMap);
- }
} else {
// There are some multiple values (without considering null)
- keySet = inMap.keySet();
- itKeyList = keySet.iterator();
while (itKeyList.hasNext()) {
key = itKeyList.next();
value = inMap.get(key);
@@ -234,10 +228,10 @@ public class ValueSortMap {
}
}
- if (ascendingOrder == null || ascendingOrder) {
- // Add Null Values in the last of the LinkedHasMap
- sortedMap.putAll(hmNullValueMap);
- }
+ }
+ if (ascendingOrder == null || ascendingOrder) {
+ // Add Null Values in the last of the LinkedHasMap
+ sortedMap.putAll(hmNullValueMap);
}
return sortedMap;
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/ParseTreeMatcherDeterministic.java b/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/ParseTreeMatcherDeterministic.java
index 9a78670..d27f3f4 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/ParseTreeMatcherDeterministic.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/ParseTreeMatcherDeterministic.java
@@ -212,7 +212,7 @@ public class ParseTreeMatcherDeterministic {
*
* @param sent1
* @param sent2
- * @return List<List<ParseTreeChunk>> list of POS-words pairs for each resultant matched / overlapped phrase.
+ * @return {@link List ParseTreeChunk} list of POS-words pairs for each resultant matched / overlapped phrase.
*/
public List<List<ParseTreeChunk>> matchTwoSentencesGroupedChunksDeterministic(
List<List<ParseTreeChunk>> sent1, List<List<ParseTreeChunk>> sent2) {
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/SentencePairMatchResult.java b/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/SentencePairMatchResult.java
index 2a74997..de13ae3 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/SentencePairMatchResult.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/SentencePairMatchResult.java
@@ -86,9 +86,10 @@ public class SentencePairMatchResult {
for (LemmaPair word : resForMinedSent1) {
if (word.getPOS().startsWith("VB") && word.getStartPos() < 1
- && word.getEndPos() < 1) {
+ && word.getEndPos() < 1) {
imperativeVerb = true;
// LOG.info("Found imperative verb=" + word);
+ break;
}
}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/word2vec/W2VDistanceMeasurer.java b/opennlp-similarity/src/main/java/opennlp/tools/word2vec/W2VDistanceMeasurer.java
index a898172..69ad781 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/word2vec/W2VDistanceMeasurer.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/word2vec/W2VDistanceMeasurer.java
@@ -29,14 +29,12 @@ import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable;
import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
import org.deeplearning4j.models.word2vec.VocabWord;
import org.deeplearning4j.models.word2vec.Word2Vec;
-import org.deeplearning4j.models.word2vec.wordstore.VocabCache;
import org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache;
import org.deeplearning4j.text.sentenceiterator.FileSentenceIterator;
import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
-import org.nd4j.common.primitives.Pair;
public class W2VDistanceMeasurer {
static W2VDistanceMeasurer instance;
@@ -62,8 +60,7 @@ public class W2VDistanceMeasurer {
String pathToW2V = resourceDir + "/w2v/GoogleNews-vectors-negative300.bin.gz";
File gModel = new File(pathToW2V);
try {
- Pair<InMemoryLookupTable, VocabCache> pair = WordVectorSerializer.loadTxt(Files.newInputStream(gModel.toPath()));
- vec = WordVectorSerializer.fromPair(pair);
+ vec = WordVectorSerializer.fromPair(WordVectorSerializer.loadTxt(Files.newInputStream(gModel.toPath())));
} catch (IOException e) {
System.out.println("Word2vec model is not loaded");
vec = null;
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/datareader/SemcorReaderExtended.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/datareader/SemcorReaderExtended.java
index 538506c..740d932 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/datareader/SemcorReaderExtended.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/datareader/SemcorReaderExtended.java
@@ -136,12 +136,11 @@ public class SemcorReaderExtended {
if (nWord.getNodeName().equals(ELEMENT_WORDFORM)) {
Element eWord = (Element) nWord;
-
+ String word = eWord.getTextContent();
+ String cmd = eWord.getAttribute(ATTRIBUTE_CMD);
+ String pos = eWord.getAttribute(ATTRIBUTE_POS);
if (eWord.getAttribute(ATTRIBUTE_CMD).equals("done")) {
// if the word is already disambiguated
- String word = eWord.getTextContent();
- String cmd = eWord.getAttribute(ATTRIBUTE_CMD);
- String pos = eWord.getAttribute(ATTRIBUTE_POS);
String lemma = eWord.getAttribute(ATTRIBUTE_LEMMA);
String wnsn = eWord.getAttribute(ATTRIBUTE_WNSN);
String lexsn = eWord.getAttribute(ATTRIBUTE_LEXSN);
@@ -149,21 +148,16 @@ public class SemcorReaderExtended {
Word iword = new Word(paragraphID, sentenceID, wnum,
Word.Type.WORD, word, cmd, pos, lemma, wnsn, lexsn);
isentence.addIword(iword);
- wnum++;
// System.out.println("*** " + iword.toString() + " ***");
} else {
// if the word is not disambiguated
- String word = eWord.getTextContent();
- String cmd = eWord.getAttribute(ATTRIBUTE_CMD);
- String pos = eWord.getAttribute(ATTRIBUTE_POS);
-
Word iword = new Word(paragraphID, sentenceID, wnum,
Word.Type.WORD, word, cmd, pos);
isentence.addIword(iword);
- wnum++;
}
+ wnum++;
} else if (nWord.getNodeName().equals(ELEMENT_PUNCTUATION)) {
Element eWord = (Element) nWord;