You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2016/01/28 00:32:31 UTC
svn commit: r1727215 - in
/manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp:
OpenNlpExtractor.java OpenNlpExtractorConfig.java
Author: kwright
Date: Wed Jan 27 23:32:31 2016
New Revision: 1727215
URL: http://svn.apache.org/viewvc?rev=1727215&view=rev
Log:
Add model registry
Modified:
manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractor.java
manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractorConfig.java
Modified: manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractor.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractor.java?rev=1727215&r1=1727214&r2=1727215&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractor.java (original)
+++ manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractor.java Wed Jan 27 23:32:31 2016
@@ -584,9 +584,9 @@ public class OpenNlpExtractor extends Ba
try {
sentenceDetector = OpenNlpExtractorConfig.sentenceDetector(new File(fileDirectory,sp.getSModelPath()));
tokenizer = OpenNlpExtractorConfig.tokenizer(new File(fileDirectory,sp.getTModelPath()));
- peopleFinder = OpenNlpExtractorConfig.peopleFinder(new File(fileDirectory,sp.getPModelPath()));
- locationFinder = OpenNlpExtractorConfig.locationFinder(new File(fileDirectory,sp.getLModelPath()));
- organizationFinder = OpenNlpExtractorConfig.organizationFinder(new File(fileDirectory,sp.getOModelPath()));
+ peopleFinder = OpenNlpExtractorConfig.finder(new File(fileDirectory,sp.getPModelPath()));
+ locationFinder = OpenNlpExtractorConfig.finder(new File(fileDirectory,sp.getLModelPath()));
+ organizationFinder = OpenNlpExtractorConfig.finder(new File(fileDirectory,sp.getOModelPath()));
} catch (IOException e) {
throw new ManifoldCFException(e.getMessage(), e);
}
Modified: manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractorConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractorConfig.java?rev=1727215&r1=1727214&r2=1727215&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractorConfig.java (original)
+++ manifoldcf/branches/CONNECTORS-1270/connectors/opennlp/connector/src/main/java/org/apache/manifoldcf/agents/transformation/opennlp/OpenNlpExtractorConfig.java Wed Jan 27 23:32:31 2016
@@ -20,6 +20,10 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.File;
+import java.io.InputStream;
+
+import java.util.HashMap;
+import java.util.Map;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinderModel;
@@ -33,9 +37,6 @@ import opennlp.tools.util.InvalidFormatE
public class OpenNlpExtractorConfig
{
- private static enum MODEL{
- SENTENCE, TOKENIZER, PEOPLE, LOCATIONS, ORGANIZATIONS;
- }
// Specification nodes and values
public static final String NODE_SMODEL_PATH = "SModelPath";
@@ -46,56 +47,69 @@ public class OpenNlpExtractorConfig
public static final String ATTRIBUTE_VALUE = "value";
- private static SentenceModel sModel = null;
- private static TokenizerModel tModel = null;
- private static TokenNameFinderModel pModel = null;
- private static TokenNameFinderModel lModel = null;
- private static TokenNameFinderModel oModel = null;
-
- private static synchronized void initializeModel(MODEL m, File path) throws InvalidFormatException, FileNotFoundException, IOException{
- if(sModel == null && m == MODEL.SENTENCE)
- sModel = new SentenceModel(new FileInputStream(path));
- if(tModel == null && m == MODEL.TOKENIZER)
- tModel = new TokenizerModel(new FileInputStream(path));
- if(pModel == null && m == MODEL.PEOPLE)
- pModel = new TokenNameFinderModel(new FileInputStream(path));
- if(lModel == null && m == MODEL.LOCATIONS)
- lModel = new TokenNameFinderModel(new FileInputStream(path));
- if(oModel == null && m == MODEL.ORGANIZATIONS)
- oModel = new TokenNameFinderModel(new FileInputStream(path));
+ private final static Map<File,SentenceModel> sModels = new HashMap<>();
+ private final static Map<File,TokenizerModel> tModels = new HashMap<>();
+ private final static Map<File,TokenNameFinderModel> tnfModels = new HashMap<>();
+
+ protected static SentenceModel loadSModel(final File path) throws InvalidFormatException, FileNotFoundException, IOException {
+ synchronized (sModels) {
+ SentenceModel sd = sModels.get(path);
+ if (sd == null) {
+ final InputStream is = new FileInputStream(path);
+ try {
+ sd = new SentenceModel(is);
+ } finally {
+ is.close();
+ }
+ sModels.put(path, sd);
+ }
+ return sd;
+ }
+ }
+
+ protected static TokenizerModel loadTModel(final File path) throws InvalidFormatException, FileNotFoundException, IOException {
+ synchronized (tModels) {
+ TokenizerModel sd = tModels.get(path);
+ if (sd == null) {
+ final InputStream is = new FileInputStream(path);
+ try {
+ sd = new TokenizerModel(is);
+ } finally {
+ is.close();
+ }
+ tModels.put(path, sd);
+ }
+ return sd;
+ }
}
-
+
+ protected static TokenNameFinderModel loadTnfModel(final File path) throws InvalidFormatException, FileNotFoundException, IOException {
+ synchronized (tnfModels) {
+ TokenNameFinderModel sd = tnfModels.get(path);
+ if (sd == null) {
+ final InputStream is = new FileInputStream(path);
+ try {
+ sd = new TokenNameFinderModel(is);
+ } finally {
+ is.close();
+ }
+ tnfModels.put(path, sd);
+ }
+ return sd;
+ }
+ }
+
public static final SentenceDetector sentenceDetector(File path) throws InvalidFormatException, FileNotFoundException, IOException{
- if(sModel == null)
- initializeModel(MODEL.SENTENCE, path);
- return new SentenceDetectorME(sModel);
+ return new SentenceDetectorME(loadSModel(path));
}
public static final Tokenizer tokenizer(File path) throws InvalidFormatException, FileNotFoundException, IOException{
- if(tModel == null)
- initializeModel(MODEL.TOKENIZER, path);
- return new TokenizerME(tModel);
+ return new TokenizerME(loadTModel(path));
}
- public static final NameFinderME peopleFinder(File path) throws InvalidFormatException, FileNotFoundException, IOException{
- if(pModel == null)
- initializeModel(MODEL.PEOPLE, path);
- return new NameFinderME(pModel);
+ public static final NameFinderME finder(File path) throws InvalidFormatException, FileNotFoundException, IOException{
+ return new NameFinderME(loadTnfModel(path));
}
- public static final NameFinderME locationFinder(File path) throws InvalidFormatException, FileNotFoundException, IOException{
- if(lModel == null)
- initializeModel(MODEL.LOCATIONS, path);
- return new NameFinderME(lModel);
- }
-
- public static final NameFinderME organizationFinder(File path) throws InvalidFormatException, FileNotFoundException, IOException{
- if(oModel == null)
- initializeModel(MODEL.ORGANIZATIONS, path);
- return new NameFinderME(oModel);
- }
-
-
-
}