You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2023/02/24 18:00:39 UTC
[opennlp-sandbox] branch sanitize_some_TODOs_and_unhealthy_code created (now 5348212)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a change to branch sanitize_some_TODOs_and_unhealthy_code
in repository https://gitbox.apache.org/repos/asf/opennlp-sandbox.git
at 5348212 sanitize some TODOs and unhealthy code
This branch includes the following new commits:
new 5348212 sanitize some TODOs and unhealthy code
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[opennlp-sandbox] 01/01: sanitize some TODOs and unhealthy code
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch sanitize_some_TODOs_and_unhealthy_code
in repository https://gitbox.apache.org/repos/asf/opennlp-sandbox.git
commit 5348212e75d579dc4a95343f6e7840b8c5ee6ede
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Fri Feb 24 19:00:29 2023 +0100
sanitize some TODOs and unhealthy code
- addresses open TODO in `PredictTest` which is now only display if platform arch is 'aarch64' (aka Apple Silicon)
- fixes incomplete resource handling of IO streams
- adds Override annotation where useful
- extracts regex Patterns instead of inline recompilation of those in loops
- removes "dead" (unused) `FileHandler` class which was flawed for several reasons anyway
- removes unused ImageIO / AWT bound code from `Utils` class
- clears some unused variables/fields
- removes some unclear TODO leftovers
- fixes some broken indentation along the path
---
.../opennlp/tools/coref/mention/DefaultParse.java | 34 +-
.../muc/Muc6FullParseCorefSampleStreamFactory.java | 3 +-
.../formats/muc/NameFinderCorefEnhancerStream.java | 1 +
.../JSMLearnerOnLatticeWithAbduction.java | 21 +-
.../tools/similarity/apps/ContentGenerator.java | 3 +-
.../similarity/apps/ContentGeneratorSupport.java | 29 +-
.../similarity/apps/RelatedSentenceFinder.java | 111 +++---
.../similarity/apps/RelatedSentenceFinderML.java | 4 -
.../solr/WordDocBuilderSingleImageSearchCall.java | 68 ++--
.../taxo_builder/TaxoQuerySnapshotMatcher.java | 4 +-
.../taxo_builder/TaxonomyExtenderViaMebMining.java | 33 +-
.../tools/similarity/apps/utils/FileHandler.java | 373 ---------------------
.../opennlp/tools/similarity/apps/utils/Utils.java | 36 +-
.../cmdline/disambiguator/DisambiguatorTool.java | 6 +-
.../disambiguator/IMSWSDContextGenerator.java | 10 +-
.../disambiguator/OSCCWSDContextGenerator.java | 7 +-
.../tools/disambiguator/WSDDefaultParameters.java | 4 +-
.../java/opennlp/tools/disambiguator/WSDModel.java | 3 +-
.../tools/disambiguator/WSDSampleStream.java | 3 +-
.../disambiguator/WSDisambiguatorFactory.java | 3 +-
.../tools/disambiguator/WSDisambiguatorME.java | 4 +-
.../java/opennlp/tools/disambiguator/WordPOS.java | 4 +-
.../main/java/org/apache/opennlp/ModelUtil.java | 22 +-
.../org/apache/opennlp/namefinder/PredictTest.java | 16 +-
24 files changed, 178 insertions(+), 624 deletions(-)
diff --git a/opennlp-coref/src/main/java/opennlp/tools/coref/mention/DefaultParse.java b/opennlp-coref/src/main/java/opennlp/tools/coref/mention/DefaultParse.java
index 725a213..d3566e1 100644
--- a/opennlp-coref/src/main/java/opennlp/tools/coref/mention/DefaultParse.java
+++ b/opennlp-coref/src/main/java/opennlp/tools/coref/mention/DefaultParse.java
@@ -23,7 +23,6 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
-import java.util.Stack;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.chunking.Parser;
@@ -56,10 +55,12 @@ public class DefaultParse extends AbstractParse {
// Should we just maintain a parse id map !?
}
+ @Override
public int getSentenceNumber() {
return sentenceNumber;
}
+ @Override
public List<opennlp.tools.coref.mention.Parse> getNamedEntities() {
List<Parse> names = new ArrayList<>();
List<Parse> kids = new LinkedList<>(Arrays.asList(parse.getChildren()));
@@ -75,10 +76,12 @@ public class DefaultParse extends AbstractParse {
return createParses(names.toArray(new Parse[names.size()]));
}
+ @Override
public List<opennlp.tools.coref.mention.Parse> getChildren() {
return createParses(parse.getChildren());
}
+ @Override
public List<opennlp.tools.coref.mention.Parse> getSyntacticChildren() {
List<Parse> kids = new ArrayList<>(Arrays.asList(parse.getChildren()));
for (int ci = 0; ci < kids.size(); ci++) {
@@ -92,6 +95,7 @@ public class DefaultParse extends AbstractParse {
return createParses(kids.toArray(new Parse[kids.size()]));
}
+ @Override
public List<opennlp.tools.coref.mention.Parse> getTokens() {
List<Parse> tokens = new ArrayList<>();
List<Parse> kids = new LinkedList<>(Arrays.asList(parse.getChildren()));
@@ -107,6 +111,7 @@ public class DefaultParse extends AbstractParse {
return createParses(tokens.toArray(new Parse[tokens.size()]));
}
+ @Override
public String getSyntacticType() {
if (ENTITY_SET.contains(parse.getType())) {
return null;
@@ -129,6 +134,7 @@ public class DefaultParse extends AbstractParse {
return newParses;
}
+ @Override
public String getEntityType() {
if (ENTITY_SET.contains(parse.getType())) {
return parse.getType();
@@ -138,6 +144,7 @@ public class DefaultParse extends AbstractParse {
}
}
+ @Override
public boolean isParentNAC() {
Parse parent = parse.getParent();
while (parent != null) {
@@ -149,6 +156,7 @@ public class DefaultParse extends AbstractParse {
return false;
}
+ @Override
public opennlp.tools.coref.mention.Parse getParent() {
Parse parent = parse.getParent();
if (parent == null) {
@@ -159,32 +167,32 @@ public class DefaultParse extends AbstractParse {
}
}
+ @Override
public boolean isNamedEntity() {
// TODO: We should use here a special tag to, where
// the type can be extracted from. Then it just depends
// on the training data and not the values inside NAME_TYPES.
-
- if (ENTITY_SET.contains(parse.getType())) {
- return true;
- }
- else {
- return false;
- }
+
+ return ENTITY_SET.contains(parse.getType());
}
+ @Override
public boolean isNounPhrase() {
return parse.getType().equals("NP") || parse.getType().startsWith("NP#");
}
+ @Override
public boolean isSentence() {
return parse.getType().equals(Parser.TOP_NODE);
}
+ @Override
public boolean isToken() {
return parse.isPosTag();
}
+ @Override
public int getEntityId() {
String type = parse.getType();
@@ -198,16 +206,17 @@ public class DefaultParse extends AbstractParse {
}
}
+ @Override
public Span getSpan() {
return parse.getSpan();
}
+ @Override
public int compareTo(opennlp.tools.coref.mention.Parse p) {
if (p == this) {
return 0;
}
-
if (getSentenceNumber() < p.getSentenceNumber()) {
return -1;
}
@@ -221,11 +230,6 @@ public class DefaultParse extends AbstractParse {
System.out.println("Maybe incorrect measurement!");
- Stack<Parse> parents = new Stack<>();
-
-
-
-
// get parent and update distance
// if match return distance
// if not match do it again
@@ -241,6 +245,7 @@ public class DefaultParse extends AbstractParse {
}
+ @Override
public opennlp.tools.coref.mention.Parse getPreviousToken() {
Parse parent = parse.getParent();
Parse node = parse;
@@ -267,6 +272,7 @@ public class DefaultParse extends AbstractParse {
}
}
+ @Override
public opennlp.tools.coref.mention.Parse getNextToken() {
Parse parent = parse.getParent();
Parse node = parse;
diff --git a/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java b/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java
index d715871..9f5a9d0 100644
--- a/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java
+++ b/opennlp-coref/src/main/java/opennlp/tools/formats/muc/Muc6FullParseCorefSampleStreamFactory.java
@@ -73,6 +73,7 @@ public class Muc6FullParseCorefSampleStreamFactory extends AbstractSampleStreamF
super(Parameters.class);
}
+ @Override
public ObjectStream<CorefSample> create(String[] args) {
Parameters params = ArgumentParser.parse(args, Parameters.class);
@@ -85,7 +86,7 @@ public class Muc6FullParseCorefSampleStreamFactory extends AbstractSampleStreamF
ObjectStream<String> mucDocStream = new FileToStringSampleStream(
new DirectorySampleStream(params.getData(), new FileFilter() {
-
+ @Override
public boolean accept(File file) {
return file.getName().toLowerCase().endsWith(".sgm");
}
diff --git a/opennlp-coref/src/main/java/opennlp/tools/formats/muc/NameFinderCorefEnhancerStream.java b/opennlp-coref/src/main/java/opennlp/tools/formats/muc/NameFinderCorefEnhancerStream.java
index e9e0bc4..4e24777 100644
--- a/opennlp-coref/src/main/java/opennlp/tools/formats/muc/NameFinderCorefEnhancerStream.java
+++ b/opennlp-coref/src/main/java/opennlp/tools/formats/muc/NameFinderCorefEnhancerStream.java
@@ -43,6 +43,7 @@ public class NameFinderCorefEnhancerStream extends FilterObjectStream<RawCorefSa
this.tags = tags;
}
+ @Override
public RawCorefSample read() throws IOException {
RawCorefSample sample = samples.read();
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithAbduction.java b/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithAbduction.java
index 76ae8ed..cd7c818 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithAbduction.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/jsmlearning/JSMLearnerOnLatticeWithAbduction.java
@@ -20,25 +20,17 @@ package opennlp.tools.jsmlearning;
import java.util.Arrays;
import java.util.List;
-public class JSMLearnerOnLatticeWithAbduction extends JSMLearnerOnLatticeWithDeduction{
-
-
-
+public class JSMLearnerOnLatticeWithAbduction extends JSMLearnerOnLatticeWithDeduction {
+ @Override
public JSMDecision buildLearningModel(List<String> posTexts, List<String> negTexts,
String unknown, String[] separationKeywords){
- JSMDecision decision = super.buildLearningModel(posTexts, negTexts, unknown, separationKeywords);
- // verify each hypothesis
- //TODO
- return decision;
-
+ //TODO verify each hypothesis
+ return super.buildLearningModel(posTexts, negTexts, unknown, separationKeywords);
}
-
-
public static void main (String[] args) {
-
String[] posArr = new String[] {"I rent an office space. This office is for my business. I can deduct office rental expense from my business profit to calculate net income. ",
"To run my business, I have to rent an office. The net business profit is calculated as follows. Rental expense needs to be subtracted from revenue. ",
"To store goods for my retail business I rent some space. When I calculate the net income, I take revenue and subtract business expenses such as office rent. ",
@@ -60,10 +52,5 @@ public class JSMLearnerOnLatticeWithAbduction extends JSMLearnerOnLatticeWithDed
// Finally, do prediction
JSMDecision dec = // may be determined by ...
jsm.buildLearningModel(Arrays.asList(posArr), Arrays.asList(negArr), unknown , new String[]{"property"});
-
-
-
-
-
}
}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java
index b71d0b2..00a6d33 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGenerator.java
@@ -93,8 +93,7 @@ public class ContentGenerator /*extends RelatedSentenceFinder*/ {
for (HitBase item : searchResult) { // got some text from .html
if (item.getAbstractText() != null
&& !(item.getUrl().indexOf(".pdf") > 0)) { // exclude pdf
- opinionSentencesToAdd
- .add(buildParagraphOfGeneratedText(item, sentence, null));
+ opinionSentencesToAdd.add(buildParagraphOfGeneratedText(item, sentence, null));
}
}
}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java
index 4389ab6..0575bbd 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/ContentGeneratorSupport.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.logging.Logger;
+import java.util.regex.Pattern;
import opennlp.tools.similarity.apps.utils.StringDistanceMeasurer;
import opennlp.tools.similarity.apps.utils.Utils;
@@ -30,15 +31,16 @@ import opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcess
import org.apache.commons.lang.StringUtils;
-/*
- * This class supports content generation by static functions
- *
+/**
+ * This class supports content generation by static functions.
*/
-
public class ContentGeneratorSupport {
private static final Logger LOG = Logger
.getLogger("opennlp.tools.similarity.apps.ContentGeneratorSupport");
+ //TODO - verify regexp!!
+ private static final Pattern SPACES_PATTERN = Pattern.compile("([a-z])(\\s{2,3})([A-Z])");
+
/**
* Takes a sentence and extracts noun phrases and entity names to from search
* queries for finding relevant sentences on the web, which are then subject
@@ -50,10 +52,7 @@ public class ContentGeneratorSupport {
* @return List<String> of search expressions
*/
public static List<String> buildSearchEngineQueryFromSentence(String sentence) {
- ParseTreeChunk matcher = new ParseTreeChunk();
- ParserChunker2MatcherProcessor pos = ParserChunker2MatcherProcessor
- .getInstance();
- List<List<ParseTreeChunk>> sent1GrpLst = null;
+ ParserChunker2MatcherProcessor pos = ParserChunker2MatcherProcessor.getInstance();
List<ParseTreeChunk> nPhrases = pos
.formGroupedPhrasesFromChunksForSentence(sentence).get(0);
@@ -135,10 +134,11 @@ public class ContentGeneratorSupport {
public static String cleanSpacesInCleanedHTMLpage(String pageContent){ //was 4 spaces
//was 3 spaces => now back to 2
- //TODO - verify regexp!!
- pageContent = pageContent.trim().replaceAll("([a-z])(\\s{2,3})([A-Z])", "$1. $3")
- .replace("..", ".").replace(". . .", " ").
- replace(". .",". ").trim(); // sometimes html breaks are converted into ' ' (two spaces), so
+ pageContent = pageContent.trim();
+ pageContent = SPACES_PATTERN.matcher(pageContent).replaceAll("$1. $3")
+ .replace("..", ".").replace(". . .", " ")
+ .replace(". .",". ").trim();
+ // sometimes html breaks are converted into ' ' (two spaces), so
// we need to put '.'
return pageContent;
}
@@ -209,12 +209,11 @@ public class ContentGeneratorSupport {
for (Fragment f2 : fragmList2) {
String sf1 = f1.getResultText();
String sf2 = f2.getResultText();
- if (StringUtils.isEmpty(sf1) || StringUtils.isEmpty(sf1))
+ if (StringUtils.isEmpty(sf1) || StringUtils.isEmpty(sf2))
continue;
if (meas.measureStringDistance(sf1, sf2) > dupeThresh) {
fragmList2Results.remove(f2);
- LOG.info("Removed duplicates from formed fragments list: "
- + sf2);
+ LOG.info("Removed duplicates from formed fragments list: " + sf2);
}
}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java
index 45bcbdb..80f02ed 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinder.java
@@ -311,8 +311,7 @@ public class RelatedSentenceFinder {
* @param hits List<HitBase> of search results objects
* @return List<String> of search results objects where dupes are removed
*/
- public static List<HitBase> removeDuplicatesFromResultantHits(
- List<HitBase> hits) {
+ public static List<HitBase> removeDuplicatesFromResultantHits(List<HitBase> hits) {
StringDistanceMeasurer meas = new StringDistanceMeasurer();
double dupeThresh = // 0.8; // if more similar, then considered dupes was
0.7;
@@ -447,7 +446,7 @@ public class RelatedSentenceFinder {
// or get original snippet
pageSentence = fragment;
if (pageSentence != null)
- pageSentence.replace("_should_find_orig_", "");
+ pageSentence = pageSentence.replace("_should_find_orig_", "");
// resultant sentence SHOULD NOT be longer than for times the size of
// snippet fragment
@@ -463,9 +462,7 @@ public class RelatedSentenceFinder {
+ " " + title, originalSentence);
List<List<ParseTreeChunk>> match = matchRes.getMatchResult();
if (!matchRes.isVerbExists() || matchRes.isImperativeVerb()) {
- System.out
- .println("Rejected Sentence : No verb OR Yes imperative verb :"
- + pageSentence);
+ System.out.println("Rejected Sentence : No verb OR Yes imperative verb :" + pageSentence);
continue;
}
@@ -520,12 +517,9 @@ public class RelatedSentenceFinder {
+ "| with title= " + title);
System.out.println("For fragment = " + fragment);
} else
- System.out
- .println("Rejected sentence due to wrong area at webpage: "
- + pageSentence);
+ System.out.println("Rejected sentence due to wrong area at webpage: " + pageSentence);
} else
- System.out.println("Rejected sentence due to low score: "
- + pageSentence);
+ System.out.println("Rejected sentence due to low score: " + pageSentence);
// }
} catch (Throwable t) {
t.printStackTrace();
@@ -902,63 +896,58 @@ public class RelatedSentenceFinder {
t.printStackTrace();
}
- return result;
-}
+ return result;
+ }
-public HitBase buildParagraphOfGeneratedText(HitBase item,
- String originalSentence, List<String> sentsAll) {
- List<Fragment> results = new ArrayList<>() ;
-
- Triple<List<String>, String, String[]> fragmentExtractionResults = formCandidateFragmentsForPage(item, originalSentence, sentsAll);
+ public HitBase buildParagraphOfGeneratedText(HitBase item,
+ String originalSentence, List<String> sentsAll) {
+ List<Fragment> results = new ArrayList<>() ;
- List<String> allFragms = fragmentExtractionResults.getFirst();
+ Triple<List<String>, String, String[]> fragmentExtractionResults = formCandidateFragmentsForPage(item, originalSentence, sentsAll);
- for (String fragment : allFragms) {
- String[] candidateSentences = formCandidateSentences(fragment, fragmentExtractionResults);
- if (candidateSentences == null)
- continue;
- Fragment res = verifyCandidateSentencesAndFormParagraph(candidateSentences, item, fragment, originalSentence, sentsAll);
- if (res!=null)
- results.add(res);
+ List<String> allFragms = fragmentExtractionResults.getFirst();
+ for (String fragment : allFragms) {
+ String[] candidateSentences = formCandidateSentences(fragment, fragmentExtractionResults);
+ if (candidateSentences == null)
+ continue;
+ Fragment res = verifyCandidateSentencesAndFormParagraph(candidateSentences, item, fragment, originalSentence, sentsAll);
+ if (res!=null)
+ results.add(res);
+
+ }
+ item.setFragments(results);
+ return item;
}
-
- item.setFragments(results );
- return item;
-}
-
-
-
-
-public static void main(String[] args) {
- RelatedSentenceFinder f = new RelatedSentenceFinder();
-
- List<HitBase> hits;
- try {
- // uncomment the sentence you would like to serve as a seed sentence for
- // content generation for an event description
-
- // uncomment the sentence you would like to serve as a seed sentence for
- // content generation for an event description
- hits = f.generateContentAbout("Albert Einstein"
- // "Britney Spears - The Femme Fatale Tour"
- // "Rush Time Machine",
- // "Blue Man Group" ,
- // "Belly Dance With Zaharah",
- // "Hollander Musicology Lecture: Danielle Fosler-Lussier, Guest Lecturer",
- // "Jazz Master and arguably the most famous jazz musician alive, trumpeter Wynton Marsalis",
- );
- System.out.println(HitBase.toString(hits));
- System.out.println(HitBase.toResultantString(hits));
- // WordFileGenerator.createWordDoc("Essey about Albert Einstein",
- // hits.get(0).getTitle(), hits);
-
- } catch (Exception e) {
- e.printStackTrace();
- }
-}
+ public static void main(String[] args) {
+ RelatedSentenceFinder f = new RelatedSentenceFinder();
+
+ List<HitBase> hits;
+ try {
+ // uncomment the sentence you would like to serve as a seed sentence for
+ // content generation for an event description
+
+ // uncomment the sentence you would like to serve as a seed sentence for
+ // content generation for an event description
+ hits = f.generateContentAbout("Albert Einstein"
+ // "Britney Spears - The Femme Fatale Tour"
+ // "Rush Time Machine",
+ // "Blue Man Group" ,
+ // "Belly Dance With Zaharah",
+ // "Hollander Musicology Lecture: Danielle Fosler-Lussier, Guest Lecturer",
+ // "Jazz Master and arguably the most famous jazz musician alive, trumpeter Wynton Marsalis",
+ );
+ System.out.println(HitBase.toString(hits));
+ System.out.println(HitBase.toResultantString(hits));
+ // WordFileGenerator.createWordDoc("Essey about Albert Einstein",
+ // hits.get(0).getTitle(), hits);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
}
\ No newline at end of file
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinderML.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinderML.java
index a075bc2..dbc93f5 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinderML.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/RelatedSentenceFinderML.java
@@ -20,7 +20,6 @@ package opennlp.tools.similarity.apps;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.logging.Logger;
import opennlp.tools.similarity.apps.utils.Utils;
import opennlp.tools.textsimilarity.TextProcessor;
@@ -29,8 +28,6 @@ import opennlp.tools.textsimilarity.TextProcessor;
* This class does content generation in ES, DE etc
*/
public class RelatedSentenceFinderML extends RelatedSentenceFinder{
- private static final Logger LOG = Logger.getLogger("opennlp.tools.similarity.apps.RelatedSentenceFinderML");
-
public RelatedSentenceFinderML(int ms, int msr, float thresh, String key) {
this.MAX_STEPS = ms;
@@ -45,7 +42,6 @@ public class RelatedSentenceFinderML extends RelatedSentenceFinder{
public List<HitBase> generateContentAbout(String sentence) throws Exception {
List<HitBase> opinionSentencesToAdd = new ArrayList<>();
System.out.println(" \n=== Entity to write about = " + sentence);
- List<String> nounPhraseQueries = new ArrayList<>();
List<HitBase> searchResult = yrunner.runSearch(sentence, 100);
if (MAX_SEARCH_RESULTS<searchResult.size())
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/solr/WordDocBuilderSingleImageSearchCall.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/solr/WordDocBuilderSingleImageSearchCall.java
index 79aa5d1..b0eaa29 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/solr/WordDocBuilderSingleImageSearchCall.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/solr/WordDocBuilderSingleImageSearchCall.java
@@ -16,7 +16,6 @@
*/
package opennlp.tools.similarity.apps.solr;
-
import java.io.File;
import java.util.ArrayList;
import java.util.List;
@@ -25,13 +24,15 @@ import net.billylieurance.azuresearch.AzureSearchImageResult;
import net.billylieurance.azuresearch.AzureSearchResultSet;
import org.docx4j.openpackaging.packages.WordprocessingMLPackage;
+import org.docx4j.openpackaging.parts.WordprocessingML.MainDocumentPart;
import opennlp.tools.similarity.apps.ContentGeneratorSupport;
import opennlp.tools.similarity.apps.Fragment;
import opennlp.tools.similarity.apps.HitBase;
public class WordDocBuilderSingleImageSearchCall extends WordDocBuilder{
-
+
+ @Override
public String buildWordDoc(List<HitBase> content, String title){
String outputDocFilename = absPath+"/written/"+ title.replace(' ','_').replace('\"', ' ').trim()+ ".docx";
@@ -41,20 +42,20 @@ public class WordDocBuilderSingleImageSearchCall extends WordDocBuilder{
int count=0;
try {
wordMLPackage = WordprocessingMLPackage.createPackage();
- wordMLPackage.getMainDocumentPart().addStyledParagraphOfText("Title", title.toUpperCase());
+ MainDocumentPart mdp = wordMLPackage.getMainDocumentPart();
+ mdp.addStyledParagraphOfText("Title", title.toUpperCase());
for(HitBase para: content){
if (para.getFragments()==null || para.getFragments().size()<1) // no found content in this hit
continue;
try {
if (!para.getTitle().endsWith("..") /*|| StringUtils.isAlphanumeric(para.getTitle())*/){
String sectTitle = ContentGeneratorSupport.getPortionOfTitleWithoutDelimiters(para.getTitle());
- wordMLPackage.getMainDocumentPart().addStyledParagraphOfText("Subtitle",
- sectTitle);
+ mdp.addStyledParagraphOfText("Subtitle", sectTitle);
}
String paraText = para.getFragments().toString().replace("[", "").replace("]", "").replace(" | ", "")
.replace(".,", ".").replace(".\"", "\"").replace(". .", ".")
.replace(",.", ".");
- wordMLPackage.getMainDocumentPart().addParagraphOfText(paraText);
+ mdp.addParagraphOfText(paraText);
try {
addImageByImageURLToPackage(count, wordMLPackage, imageURLs);
@@ -67,33 +68,22 @@ public class WordDocBuilderSingleImageSearchCall extends WordDocBuilder{
count++;
}
// now add URLs
- wordMLPackage.getMainDocumentPart().addStyledParagraphOfText("Subtitle", "REFERENCES");
+ mdp.addStyledParagraphOfText("Subtitle", "REFERENCES");
for(HitBase para: content){
if (para.getFragments()==null || para.getFragments().size()<1) // no found content in this hit
continue;
try {
- wordMLPackage.getMainDocumentPart().addStyledParagraphOfText("Subtitle",
- para.getTitle());
+ mdp.addStyledParagraphOfText("Subtitle", para.getTitle());
String paraText = para.getUrl();
- wordMLPackage.getMainDocumentPart().addParagraphOfText(paraText);
-
-
+ mdp.addParagraphOfText(paraText);
+
} catch (Exception e) {
e.printStackTrace();
}
}
-
wordMLPackage.save(new File(outputDocFilename));
System.out.println("Finished creating docx ="+outputDocFilename);
- //TODO pdf export
- /*
- FOSettings foSettings = Docx4J.createFOSettings();
- foSettings.setWmlPackage(wordMLPackage);
- OutputStream os = new java.io.FileOutputStream(outputDocFilename.replace(".docx", ".pdf"));
- Docx4J.toFO(foSettings, os, Docx4J.FLAG_NONE);
- System.out.println("Finished creating docx's PDF ="+outputDocFilename);
- */
} catch (Exception e) {
e.printStackTrace();
@@ -102,9 +92,8 @@ public class WordDocBuilderSingleImageSearchCall extends WordDocBuilder{
return outputDocFilename;
}
- protected void addImageByImageURLToPackage(int count,
- WordprocessingMLPackage wordMLPackage,
- List<String> imageURLs) {
+ protected void addImageByImageURLToPackage(int count, WordprocessingMLPackage wordMLPackage,
+ List<String> imageURLs) {
if (count>imageURLs.size()-1)
return;
@@ -112,7 +101,7 @@ public class WordDocBuilderSingleImageSearchCall extends WordDocBuilder{
String destinationFile = url.replace("http://", "").replace("/", "_");
saveImageFromTheWeb(url, absPath+IMG_REL_PATH+destinationFile);
File file = new File(absPath+IMG_REL_PATH+destinationFile);
- try {
+ try {
byte[] bytes = convertImageToByteArray(file);
addImageToPackage(wordMLPackage, bytes);
} catch (Exception e) {
@@ -130,20 +119,19 @@ public class WordDocBuilderSingleImageSearchCall extends WordDocBuilder{
return imageURLs;
}
-
- public static void main(String[] args){
- WordDocBuilderSingleImageSearchCall b = new WordDocBuilderSingleImageSearchCall();
- List<HitBase> content = new ArrayList<>();
- for(int i = 0; i<10; i++){
- HitBase h = new HitBase();
- h.setTitle("albert einstein "+i);
- List<Fragment> frs = new ArrayList<>();
- frs.add(new Fragment(" content "+i, 0));
- h.setFragments(frs);
- content.add(h);
- }
-
- b.buildWordDoc(content, "albert einstein");
- }
+ public static void main(String[] args){
+ WordDocBuilderSingleImageSearchCall b = new WordDocBuilderSingleImageSearchCall();
+ List<HitBase> content = new ArrayList<>();
+ for(int i = 0; i<10; i++){
+ HitBase h = new HitBase();
+ h.setTitle("albert einstein "+i);
+ List<Fragment> frs = new ArrayList<>();
+ frs.add(new Fragment(" content "+i, 0));
+ h.setFragments(frs);
+ content.add(h);
+ }
+
+ b.buildWordDoc(content, "albert einstein");
+ }
}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java
index 1be923e..fa205d7 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxoQuerySnapshotMatcher.java
@@ -19,7 +19,6 @@ package opennlp.tools.similarity.apps.taxo_builder;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -55,8 +54,7 @@ public class TaxoQuerySnapshotMatcher {
*/
public int getTaxoScore(String query, String snapshot) {
- lemma_ExtendedAssocWords = (HashMap<String, List<List<String>>>) taxo
- .getLemma_ExtendedAssocWords();
+ lemma_ExtendedAssocWords = taxo.getLemma_ExtendedAssocWords();
query = query.toLowerCase();
snapshot = snapshot.toLowerCase();
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java
index 2f53a7d..e780330 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/taxo_builder/TaxonomyExtenderViaMebMining.java
@@ -27,7 +27,6 @@ import opennlp.tools.similarity.apps.HitBase;
import opennlp.tools.similarity.apps.utils.StringCleaner;
import opennlp.tools.stemmer.PStemmer;
import opennlp.tools.textsimilarity.ParseTreeChunk;
-import opennlp.tools.textsimilarity.ParseTreeChunkListScorer;
import opennlp.tools.textsimilarity.SentencePairMatchResult;
import opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcessor;
@@ -39,11 +38,9 @@ import opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcess
* derived list output map 2) for such manual list of words -> derived list of
* words
*/
-
public class TaxonomyExtenderViaMebMining extends BingQueryRunner {
- private final ParseTreeChunkListScorer parseTreeChunkListScorer = new ParseTreeChunkListScorer();
- ParserChunker2MatcherProcessor sm;
+ private ParserChunker2MatcherProcessor sm;
private Map<String, List<List<String>>> lemma_ExtendedAssocWords = new HashMap<>();
private final Map<List<String>, List<List<String>>> assocWords_ExtendedAssocWords = new HashMap<>();
@@ -145,19 +142,17 @@ public class TaxonomyExtenderViaMebMining extends BingQueryRunner {
List<HitBase> resultList = runSearch(query, numbOfHits);
for (int i = 0; i < resultList.size(); i++) {
- {
- for (int j = i + 1; j < resultList.size(); j++) {
- HitBase h1 = resultList.get(i);
- HitBase h2 = resultList.get(j);
- String snapshot1 = StringCleaner.processSnapshotForMatching(h1
- .getTitle() + " . " + h1.getAbstractText());
- String snapshot2 = StringCleaner.processSnapshotForMatching(h2
- .getTitle() + " . " + h2.getAbstractText());
- SentencePairMatchResult matchRes = sm.assessRelevance(snapshot1,
- snapshot2);
- List<List<ParseTreeChunk>> matchResult = matchRes.getMatchResult();
- genResult.addAll(matchResult);
- }
+ for (int j = i + 1; j < resultList.size(); j++) {
+ HitBase h1 = resultList.get(i);
+ HitBase h2 = resultList.get(j);
+ String snapshot1 = StringCleaner.processSnapshotForMatching(h1
+ .getTitle() + " . " + h1.getAbstractText());
+ String snapshot2 = StringCleaner.processSnapshotForMatching(h2
+ .getTitle() + " . " + h2.getAbstractText());
+ SentencePairMatchResult matchRes = sm.assessRelevance(snapshot1,
+ snapshot2);
+ List<List<ParseTreeChunk>> matchResult = matchRes.getMatchResult();
+ genResult.addAll(matchResult);
}
}
@@ -175,9 +170,7 @@ public class TaxonomyExtenderViaMebMining extends BingQueryRunner {
public static void main(String[] args) {
TaxonomyExtenderViaMebMining self = new TaxonomyExtenderViaMebMining();
- self.extendTaxonomy("src/test/resources/taxonomies/irs_dom.ari", "tax",
- "en");
-
+ self.extendTaxonomy("src/test/resources/taxonomies/irs_dom.ari", "tax", "en");
}
}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/FileHandler.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/FileHandler.java
deleted file mode 100644
index 21bdafb..0000000
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/FileHandler.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.similarity.apps.utils;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.logging.Logger;
-
-/**
- * This class responsible to save data to files as well as read out! It is
- * capable to handle text and binary files.
- */
-public class FileHandler {
-
- private static final Logger LOG = Logger
- .getLogger("opennlp.tools.similarity.apps.utils.FileHandler");
-
- public void writeToTextFile(String data, String filepath, boolean append)
- throws IOException {
- try {
- BufferedWriter out = new BufferedWriter(new FileWriter(filepath, append));
- out.write(data + "\n");
- out.close();
- } catch (IOException e) {
- LOG.severe(e.toString());
- e.printStackTrace();
- }
- }
-
- /**
- * Writes data from an arrayList<String> to a text-file where each line of the
- * text represented by an element in the list.
- *
- * @param list
- * @param filePath
- * @param append
- * @throws Exception
- */
- public void writeToTextFile(ArrayList<String> list, String filePath, boolean append) throws Exception {
- FileWriter outFile;
- Iterator<String> it = list.iterator();
- if (!append) {
- outFile = new FileWriter(filePath);
- PrintWriter out = new PrintWriter(outFile);
- while (it.hasNext()) {
- out.println(it.next());
- }
- outFile.close();
- } else {
- int tmp = 0;
- while (it.hasNext()) {
- if (tmp == 0) {
- appendtofile("\n" + it.next(), filePath);
- } else {
- appendtofile(it.next(), filePath);
- }
- tmp++;
- }
- }
- }
-
- public void writeObjectToFile(Object obj, String filepath, boolean append) {
- if (!isFileOrDirectoryExists(getDirPathfromFullPath(filepath))) {
- createFolder(getDirPathfromFullPath(filepath));
- }
- ObjectOutputStream outputStream;
- try {
- outputStream = new ObjectOutputStream(new FileOutputStream(filepath));
- outputStream.writeObject(obj);
- } catch (IOException e) {
- LOG.severe(e.toString());
- }
- }
-
- public Object readObjectfromFile(String filePath) {
- ObjectInputStream inputStream = null;
- try {
- // Construct the ObjectInputStream object
- inputStream = new ObjectInputStream(new FileInputStream(filePath));
- Object obj;
- while ((obj = inputStream.readObject()) != null) {
- return obj;
- }
- } catch (EOFException ex) { // This exception will be caught when EOF is
- // reached
- LOG.severe("End of file reached.\n" + ex);
- } catch (ClassNotFoundException | IOException ex) {
- LOG.severe(ex.toString());
- } finally {
- // Close the ObjectInputStream
- try {
- if (inputStream != null) {
- inputStream.close();
- }
- } catch (IOException ex) {
- LOG.severe(ex.toString());
- }
- }
- return null;
- }
-
- /**
- * Creates a byte array from any object.
- * <p>
- * I wanted to use it when I write out object to files! (This is not in use
- * right now, I may move it into other class)
- *
- * @param obj
- * @return
- * @throws java.io.IOException
- */
- public byte[] getBytes(Object obj) throws java.io.IOException {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(bos);
- oos.writeObject(obj);
- oos.flush();
- oos.close();
- bos.close();
- return bos.toByteArray();
- }
-
- /**
- * Fetches all content from a text file, and return it as a String.
- *
- * @return
- */
- public String readFromTextFile(String filePath) {
- StringBuilder contents = new StringBuilder();
- // ...checks on aFile are edited
- File aFile = new File(filePath);
-
- try {
- // use buffering, reading one line at a time
- // FileReader always assumes default encoding is OK!
- // TODO be sure that the default encoding is OK!!!!! Otherwise
- // change it
-
- try (BufferedReader input = new BufferedReader(new FileReader(aFile))) {
- String line; // not declared within while loop
- /*
- * readLine is a bit quirky : it returns the content of a line MINUS the
- * newline. it returns null only for the END of the stream. it returns
- * an empty String if two newlines appear in a row.
- */
- while ((line = input.readLine()) != null) {
- contents.append(line);
- contents.append(System.getProperty("line.separator"));
- }
- }
- } catch (IOException ex) {
- LOG.severe("fileName: " + filePath +"\n " + ex);
- }
- return contents.toString();
- }
-
- /**
- * Reads text file line-wise each line will be an element in the resulting
- * list
- *
- * @param filePath
- * @return
- */
- public List<String> readLinesFromTextFile(String filePath) {
- List<String> lines = new ArrayList<>();
- // ...checks on aFile are edited
- File aFile = new File(filePath);
- try {
- // use buffering, reading one line at a time
- // FileReader always assumes default encoding is OK!
- // TODO be sure that the default encoding is OK!!!!! Otherwise
- // change it
-
- BufferedReader input = new BufferedReader(new FileReader(aFile));
- try {
- String line; // not declared within while loop
- /*
- * readLine is a bit quirky : it returns the content of a line MINUS the
- * newline. it returns null only for the END of the stream. it returns
- * an empty String if two newlines appear in a row.
- */
- while ((line = input.readLine()) != null) {
- lines.add(line);
- }
- } finally {
- input.close();
- }
- } catch (IOException ex) {
- LOG.severe(ex.toString());
- }
- return lines;
- }
-
- private void appendtofile(String data, String filePath) {
- try (BufferedWriter out = new BufferedWriter(new FileWriter(filePath, true))) {
- out.write(data + "\n");
- } catch (IOException e) {
- }
- }
-
- public void createFolder(String path) {
- if (!isFileOrDirectoryExists(path)) {
- File file = new File(path);
- try {
- file.mkdirs();
- } catch (Exception e) {
- LOG.severe("Directory already exists or the file-system is read only");
- }
- }
- }
-
- public boolean isFileOrDirectoryExists(String path) {
- File file = new File(path);
- return file.exists();
- }
-
- /**
- * Separates the directory-path from a full file-path
- *
- * @param filePath
- * @return
- */
- private String getDirPathfromFullPath(String filePath) {
- String dirPath = "";
- if (filePath != null) {
- if (filePath.contains("\\"))
- dirPath = filePath.substring(0, filePath.lastIndexOf("\\"));
- }
- return dirPath;
- }
-
- /**
- * Returns the file-names of the files in a folder (not paths only names) (Not
- * recursive)
- *
- * @param dirPath
- * @return
- */
- public ArrayList<String> getFileNamesInFolder(String dirPath) {
- ArrayList<String> fileNames = new ArrayList<>();
-
- File folder = new File(dirPath);
- File[] listOfFiles = folder.listFiles();
-
- for (File listOfFile : listOfFiles) {
- if (listOfFile.isFile()) {
- fileNames.add(listOfFile.getName());
- } else if (listOfFile.isDirectory()) {
- // TODO if I want to use it recursive I should handle this case
- }
- }
- return fileNames;
- }
-
- public void deleteAllfilesinDir(String dirName) {
- ArrayList<String> fileNameList = getFileNamesInFolder(dirName);
- if (fileNameList != null) {
- for (String s : fileNameList) {
- try {
- deleteFile(dirName + s);
- } catch (IllegalArgumentException e) {
- LOG.severe("No way to delete file: " + dirName + s + "\n" +
- e);
- }
- }
- }
- }
-
- public void deleteFile(String filePath) throws IllegalArgumentException {
- // A File object to represent the filename
- File f = new File(filePath);
- // Make sure the file or directory exists and isn't write protected
- if (!f.exists())
- throw new IllegalArgumentException("Delete: no such file or directory: "
- + filePath);
-
- if (!f.canWrite())
- throw new IllegalArgumentException("Delete: write protected: " + filePath);
- // If it is a directory, make sure it is empty
- if (f.isDirectory()) {
- String[] files = f.list();
- if (files.length > 0)
- throw new IllegalArgumentException("Delete: directory not empty: "
- + filePath);
- }
- // Attempt to delete it
- boolean success = f.delete();
- if (!success)
- throw new IllegalArgumentException("Delete: deletion failed");
- }
-
- public boolean deleteDirectory(File path) {
- if (path.exists()) {
- File[] files = path.listFiles();
- for (File file : files) {
- if (file.isDirectory()) {
- deleteDirectory(file);
- } else {
- file.delete();
- }
- }
- }
- return (path.delete());
- }
-
- /**
- * Returns the absolute-file-paths of the files in a directory (not recursive)
- *
- * @param dirPath
- * @return
- */
- public ArrayList<String> getFilePathsInFolder(String dirPath) {
- ArrayList<String> filePaths = new ArrayList<>();
-
- File folder = new File(dirPath);
- File[] listOfFiles = folder.listFiles();
- if (listOfFiles == null)
- return null;
- for (File listOfFile : listOfFiles) {
- if (listOfFile.isFile()) {
- filePaths.add(listOfFile.getAbsolutePath());
- } else if (listOfFile.isDirectory()) {
- // TODO if I want to use it recursive I should handle this case
- }
- }
- return filePaths;
- }
-
- /**
- * Returns the number of individual files in a directory (Not recursive)
- *
- * @param dirPath
- * @return
- */
- public int getFileNumInFolder(String dirPath) {
- int num;
- try {
- num = getFileNamesInFolder(dirPath).size();
- } catch (Exception e) {
- num = 0;
- }
- return num;
- }
-
-}
diff --git a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/Utils.java b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/Utils.java
index bae6357..4fd8a17 100644
--- a/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/Utils.java
+++ b/opennlp-similarity/src/main/java/opennlp/tools/similarity/apps/utils/Utils.java
@@ -17,29 +17,19 @@
package opennlp.tools.similarity.apps.utils;
-import java.awt.Graphics2D;
-import java.awt.geom.AffineTransform;
-import java.awt.image.BufferedImage;
-import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;
-import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.imageio.ImageIO;
-
import org.apache.commons.lang.StringUtils;
public class Utils {
- private static final Logger LOG = Logger
- .getLogger("opennlp.tools.similarity.apps.utils.Utils");
-
protected static final ArrayList<String[]> CHARACTER_MAPPINGS = new ArrayList<>();
static {
@@ -237,30 +227,6 @@ public class Utils {
}
}
- public static boolean createResizedCopy(String originalImage,
- String newImage, int scaledWidth, int scaledHeight) {
- boolean retVal = true;
- try {
- File o = new File(originalImage);
- BufferedImage bsrc = ImageIO.read(o);
- BufferedImage bdest = new BufferedImage(scaledWidth, scaledHeight,
- BufferedImage.TYPE_INT_RGB);
-
- Graphics2D g = bdest.createGraphics();
- AffineTransform at = AffineTransform.getScaleInstance(
- (double) scaledWidth / bsrc.getWidth(),
- (double) scaledHeight / bsrc.getHeight());
- g.drawRenderedImage(bsrc, at);
- ImageIO.write(bdest, "jpeg", new File(newImage));
-
- } catch (Exception e) {
- retVal = false;
- LOG.severe("Failed creating thumbnail for image: " + originalImage + e);
- }
-
- return retVal;
- }
-
private static int minimum(int a, int b, int c) {
int mi;
@@ -676,7 +642,7 @@ public class Utils {
public static boolean isLatinWord(String word) {
for (int i = 0; i < word.length(); i++) {
- int asciiCode = (int) word.charAt(i);
+ int asciiCode = word.charAt(i);
if (asciiCode > 128)
return false;
}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/cmdline/disambiguator/DisambiguatorTool.java b/opennlp-wsd/src/main/java/opennlp/tools/cmdline/disambiguator/DisambiguatorTool.java
index 98f32bd..0a4554f 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/cmdline/disambiguator/DisambiguatorTool.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/cmdline/disambiguator/DisambiguatorTool.java
@@ -43,13 +43,11 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.ParagraphStream;
import opennlp.tools.util.PlainTextByLineStream;
-/*
- * Command line tool for disambiguator supports MFS for now
- *
+/**
+ * Command line tool for disambiguator supports MFS for now.
*/
public class DisambiguatorTool extends CmdLineTool {
- // TODO CmdLineTool should be an interface not abstract class
@Override
public String getName() {
return "Disambiguator";
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/IMSWSDContextGenerator.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/IMSWSDContextGenerator.java
index 7cc7015..11d8f9e 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/IMSWSDContextGenerator.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/IMSWSDContextGenerator.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
+import java.util.regex.Pattern;
public class IMSWSDContextGenerator implements WSDContextGenerator {
@@ -48,13 +49,14 @@ public class IMSWSDContextGenerator implements WSDContextGenerator {
// TODO consider the windowSize
List<String> contextWords = new ArrayList<>();
+ final Pattern pattern = Pattern.compile("[^a-z_]");
+
for (int i = 0; i < toks.length; i++) {
if (lemmas != null) {
- if (!WSDHelper.STOP_WORDS.contains(toks[i].toLowerCase()) && (index
- != i)) {
+ if (!WSDHelper.STOP_WORDS.contains(toks[i].toLowerCase()) && (index != i)) {
- String lemma = lemmas[i].toLowerCase().replaceAll("[^a-z_]", "")
- .trim();
+ String lemma = lemmas[i].toLowerCase();
+ lemma = pattern.matcher(lemma).replaceAll("").trim();
if (lemma.length() > 1) {
contextWords.add(lemma);
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/OSCCWSDContextGenerator.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/OSCCWSDContextGenerator.java
index 8c52c9d..6e13523 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/OSCCWSDContextGenerator.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/OSCCWSDContextGenerator.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.regex.Pattern;
import net.sf.extjwnl.data.Synset;
@@ -38,14 +39,16 @@ public class OSCCWSDContextGenerator implements WSDContextGenerator {
// TODO consider windowSize
ArrayList<String> contextClusters = new ArrayList<>();
+ final Pattern pattern = Pattern.compile("[^a-z_]");
+
for (int i = 0; i < toks.length; i++) {
if (lemmas != null) {
if (!WSDHelper.STOP_WORDS.contains(toks[i].toLowerCase()) && (index
!= i)) {
- String lemma = lemmas[i].toLowerCase().replaceAll("[^a-z_]", "")
- .trim();
+ String lemma = lemmas[i].toLowerCase();
+ lemma = pattern.matcher(lemma).replaceAll("").trim();
WordPOS word = new WordPOS(lemma, tags[i]);
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDDefaultParameters.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDDefaultParameters.java
index e65bccb..446b46c 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDDefaultParameters.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDDefaultParameters.java
@@ -98,8 +98,8 @@ public class WSDDefaultParameters extends WSDParameters {
this.trainingDataDirectory = trainingDataDirectory;
}
- @Override public boolean areValid() {
- // TODO recheck this pattern
+ @Override
+ public boolean areValid() {
return true;
}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDModel.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDModel.java
index a51b656..90afbbf 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDModel.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDModel.java
@@ -126,7 +126,8 @@ public class WSDModel extends BaseModel {
return true;
}
- @Override protected void validateArtifactMap() throws InvalidFormatException {
+ @Override
+ protected void validateArtifactMap() throws InvalidFormatException {
super.validateArtifactMap();
if (!(artifactMap.get(WSD_MODEL_ENTRY_NAME) instanceof AbstractModel)) {
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDSampleStream.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDSampleStream.java
index d8667d2..fc060f3 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDSampleStream.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDSampleStream.java
@@ -61,8 +61,7 @@ public class WSDSampleStream extends FilterObjectStream<String, WSDSample> {
} catch (InvalidFormatException e) {
if (LOG.isLoggable(Level.WARNING)) {
- LOG
- .warning("Error during parsing, ignoring sentence: " + sentence);
+ LOG.warning("Error during parsing, ignoring sentence: " + sentence);
}
sample = null; // new WSDSample(new String[]{}, new String[]{},0);
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorFactory.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorFactory.java
index b222f52..f75d9b7 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorFactory.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorFactory.java
@@ -50,7 +50,8 @@ public class WSDisambiguatorFactory extends BaseToolFactory {
}
}
- @Override public void validateArtifactMap() throws InvalidFormatException {
+ @Override
+ public void validateArtifactMap() throws InvalidFormatException {
// no additional artifacts
}
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorME.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorME.java
index c8aa549..b70bd42 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorME.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WSDisambiguatorME.java
@@ -115,7 +115,8 @@ public class WSDisambiguatorME extends WSDisambiguator {
return surroundingWordsModel;
}
- @Override public String disambiguate(WSDSample sample) {
+ @Override
+ public String disambiguate(WSDSample sample) {
if (WSDHelper.isRelevantPOSTag(sample.getTargetTag())) {
String wordTag = sample.getTargetWordTag();
@@ -202,6 +203,7 @@ public class WSDisambiguatorME extends WSDisambiguator {
* @param index : the index of the word to disambiguate
* @return an array of the senses of the word to disambiguate
*/
+ @Override
public String disambiguate(String[] tokenizedContext, String[] tokenTags,
String[] lemmas, int index) {
return disambiguate(
diff --git a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java
index 7ada773..5a2ff78 100644
--- a/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java
+++ b/opennlp-wsd/src/main/java/opennlp/tools/disambiguator/WordPOS.java
@@ -36,7 +36,6 @@ public class WordPOS {
private List<String> stems;
private final POS pos;
private String posTag;
- public boolean isTarget = false;
public WordPOS(String word, String tag) throws IllegalArgumentException {
if (word == null || tag == null) {
@@ -82,8 +81,7 @@ public class WordPOS {
try {
indexWord = WSDHelper.getDictionary().lookupIndexWord(pos, word);
if (indexWord == null) {
- WSDHelper
- .print("NULL synset probably a POS tagger mistake ! :: [POS] : "
+ WSDHelper.print("NULL synset probably a POS tagger mistake ! :: [POS] : "
+ pos.getLabel() + " [word] : " + word);
return null;
}
diff --git a/tf-ner-poc/src/main/java/org/apache/opennlp/ModelUtil.java b/tf-ner-poc/src/main/java/org/apache/opennlp/ModelUtil.java
index 76e5c8a..1f5b2d2 100644
--- a/tf-ner-poc/src/main/java/org/apache/opennlp/ModelUtil.java
+++ b/tf-ner-poc/src/main/java/org/apache/opennlp/ModelUtil.java
@@ -29,21 +29,21 @@ public class ModelUtil {
public static Path writeModelToTmpDir(InputStream modelIn) throws IOException {
Path tmpDir = Files.createTempDirectory("opennlp2");
- ZipInputStream zis = new ZipInputStream(modelIn);
- ZipEntry zipEntry = zis.getNextEntry();
- while(zipEntry != null){
- Path newFile = tmpDir.resolve(zipEntry.getName());
+ try (ZipInputStream zis = new ZipInputStream(modelIn)) {
+ ZipEntry zipEntry = zis.getNextEntry();
+ while(zipEntry != null){
+ Path newFile = tmpDir.resolve(zipEntry.getName());
- Files.createDirectories(newFile.getParent());
- Files.copy(zis, newFile);
+ Files.createDirectories(newFile.getParent());
+ Files.copy(zis, newFile);
- // TODO: How to delete the tmp directory after we are done loading from it ?!
- newFile.toFile().deleteOnExit();
+ // TODO: How to delete the tmp directory after we are done loading from it ?!
+ newFile.toFile().deleteOnExit();
- zipEntry = zis.getNextEntry();
+ zipEntry = zis.getNextEntry();
+ }
+ zis.closeEntry();
}
- zis.closeEntry();
- zis.close();
return tmpDir;
}
diff --git a/tf-ner-poc/src/test/java/org/apache/opennlp/namefinder/PredictTest.java b/tf-ner-poc/src/test/java/org/apache/opennlp/namefinder/PredictTest.java
index 4c7b906..0bdae56 100644
--- a/tf-ner-poc/src/test/java/org/apache/opennlp/namefinder/PredictTest.java
+++ b/tf-ner-poc/src/test/java/org/apache/opennlp/namefinder/PredictTest.java
@@ -19,20 +19,20 @@
package org.apache.opennlp.namefinder;
-import org.junit.jupiter.api.Disabled;
+import java.io.IOException;
+import java.nio.file.Path;
+
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
import opennlp.tools.util.Span;
-import java.io.IOException;
-import java.nio.file.Path;
-
class PredictTest {
- @Test
- @Disabled // TODO This test is not platform neutral and, for instance, fails with
- // "Cannot find TensorFlow native library for OS: darwin, architecture: aarch64"
- // We need JUnit 5 in the sandbox to circumvent this, so it can be run in supported environments
+ // Note: Atm, this test is not platform neutral and, for instance, fails with
+ // "Cannot find TensorFlow native library for OS: darwin, architecture: aarch64"
+ // That's why it is disabled via the architecture system property.
+ @Test @DisabledIfSystemProperty(named = "os.arch", matches = "aarch64")
void testFindTokens() throws IOException {
// can be changed to File or InputStream