You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2012/07/17 16:30:51 UTC

svn commit: r1362511 - in /opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server: ./ namefind/ postag/ sentdetect/ tokenize/

Author: joern
Date: Tue Jul 17 14:30:50 2012
New Revision: 1362511

URL: http://svn.apache.org/viewvc?rev=1362511&view=rev
Log:
OPENNLP-480 Added initial support for tokenizer and sentence detector, updated name finder and pos tagger.

Added:
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ServiceUtil.java   (contents, props changed)
      - copied, changed from r1331314, opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ModelUtil.java
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/DefaultRawTextNameFinderFactory.java   (contents, props changed)
      - copied, changed from r1331314, opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/ModelNameFinderPreprocessFactory.java
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/RawTextNameFinderFactory.java   (contents, props changed)
      - copied, changed from r1331314, opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderPreprocessFactory.java
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/sentdetect/
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/sentdetect/SentenceDetectorResource.java   (with props)
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/tokenize/
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/tokenize/TokenizerResource.java   (with props)
Modified:
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderResource.java
    opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/postag/POSTaggerResource.java

Copied: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ServiceUtil.java (from r1331314, opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ModelUtil.java)
URL: http://svn.apache.org/viewvc/opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ServiceUtil.java?p2=opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ServiceUtil.java&p1=opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ModelUtil.java&r1=1331314&r2=1362511&rev=1362511&view=diff
==============================================================================
--- opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ModelUtil.java (original)
+++ opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ServiceUtil.java Tue Jul 17 14:30:50 2012
@@ -22,20 +22,20 @@ import org.osgi.framework.BundleContext;
 import org.osgi.framework.FrameworkUtil;
 import org.osgi.framework.ServiceReference;
 
-public class ModelUtil {
+public class ServiceUtil {
 
-  private ModelUtil() {
+  private ServiceUtil() {
   }
   
-  public static ServiceReference getService(Class<?> serviceClazz) {
-    Bundle bundle = FrameworkUtil.getBundle(ModelUtil.class);
+  public static ServiceReference getServiceReference(Class<?> serviceClazz) {
+    Bundle bundle = FrameworkUtil.getBundle(ServiceUtil.class);
     BundleContext context = bundle.getBundleContext();
 
     return context.getServiceReference(serviceClazz.getName());
   }
-  
+
   @SuppressWarnings("unchecked")
-  public static <T> T getModel(ServiceReference modelService, Class<T> modelClazz) {
+  public static <T> T getService(ServiceReference modelService, Class<T> modelClazz) {
     
     T model;
     if (modelService != null) {

Propchange: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/ServiceUtil.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Copied: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/DefaultRawTextNameFinderFactory.java (from r1331314, opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/ModelNameFinderPreprocessFactory.java)
URL: http://svn.apache.org/viewvc/opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/DefaultRawTextNameFinderFactory.java?p2=opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/DefaultRawTextNameFinderFactory.java&p1=opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/ModelNameFinderPreprocessFactory.java&r1=1331314&r2=1362511&rev=1362511&view=diff
==============================================================================
--- opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/ModelNameFinderPreprocessFactory.java (original)
+++ opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/DefaultRawTextNameFinderFactory.java Tue Jul 17 14:30:50 2012
@@ -17,6 +17,9 @@
 
 package org.apache.opennlp.tagging_server.namefind;
 
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.TokenNameFinder;
+import opennlp.tools.namefind.TokenNameFinderModel;
 import opennlp.tools.sentdetect.SentenceDetector;
 import opennlp.tools.sentdetect.SentenceDetectorME;
 import opennlp.tools.sentdetect.SentenceModel;
@@ -24,15 +27,17 @@ import opennlp.tools.tokenize.Tokenizer;
 import opennlp.tools.tokenize.TokenizerME;
 import opennlp.tools.tokenize.TokenizerModel;
 
-public class ModelNameFinderPreprocessFactory implements NameFinderPreprocessFactory {
+public class DefaultRawTextNameFinderFactory implements RawTextNameFinderFactory {
   
   private final SentenceModel sentModel;
   private final TokenizerModel tokenModel;
+  private final TokenNameFinderModel nameModel;
 
-  public ModelNameFinderPreprocessFactory(SentenceModel sentModel,
-      TokenizerModel tokenModel) {
-        this.sentModel = sentModel;
-        this.tokenModel = tokenModel;
+  public DefaultRawTextNameFinderFactory(SentenceModel sentModel,
+      TokenizerModel tokenModel, TokenNameFinderModel nameModel) {
+    this.sentModel = sentModel;
+    this.tokenModel = tokenModel;
+    this.nameModel = nameModel;
   }
 
   @Override
@@ -44,4 +49,9 @@ public class ModelNameFinderPreprocessFa
   public Tokenizer createTokenizer() {
     return new TokenizerME(tokenModel);
   }
+
+  @Override
+  public TokenNameFinder createNameFinder() {
+    return new NameFinderME(nameModel);
+  }
 }

Propchange: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/DefaultRawTextNameFinderFactory.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderResource.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderResource.java?rev=1362511&r1=1362510&r2=1362511&view=diff
==============================================================================
--- opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderResource.java (original)
+++ opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderResource.java Tue Jul 17 14:30:50 2012
@@ -18,23 +18,22 @@
 package org.apache.opennlp.tagging_server.namefind;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 import javax.ws.rs.Consumes;
-import javax.ws.rs.OPTIONS;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
 
 import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.TokenNameFinder;
 import opennlp.tools.namefind.TokenNameFinderModel;
-import opennlp.tools.tokenize.SimpleTokenizer;
+import opennlp.tools.sentdetect.SentenceDetector;
+import opennlp.tools.tokenize.Tokenizer;
 import opennlp.tools.util.Span;
 
-import org.apache.opennlp.tagging_server.ModelUtil;
+import org.apache.opennlp.tagging_server.ServiceUtil;
 import org.osgi.framework.ServiceReference;
 
 
@@ -42,10 +41,10 @@ import org.osgi.framework.ServiceReferen
 public class NameFinderResource {
 
   public static class NameFinderDocument {
-    private String document[][];
+    private List<Span[]> document;
     private List<Span[]> names;
     
-    NameFinderDocument(String document[][], List<Span[]> names) {
+    NameFinderDocument(List<Span[]> document, List<Span[]> names) {
       this.document = document;
       this.names = names;
     }
@@ -54,77 +53,88 @@ public class NameFinderResource {
       return names;
     }
     
-    public String[][] getDocument() {
+    public List<Span[]> getDocument() {
       return document;
     }
   }
   
+  private List<Span[]> find(TokenNameFinder nameFinder, String[][] document) {
+
+    List<Span[]> names = new ArrayList<Span[]>();
+
+    for (String sentence[] : document) {
+      names.add(nameFinder.find(sentence));
+    }
+
+    return names;
+  }
+  
   @POST
   @Consumes(MediaType.APPLICATION_JSON)
   @Produces(MediaType.APPLICATION_JSON)
   @Path("_find")
-  public List<Span> find(String[][] document) {
+  public List<Span[]> find(String[][] document) {
     
-    ServiceReference modelService = ModelUtil.getService(TokenNameFinderModel.class);
+    ServiceReference modelService = ServiceUtil.getServiceReference(TokenNameFinderModel.class);
     
     try {
       NameFinderME nameFinder = new NameFinderME(
-          ModelUtil.getModel(modelService, TokenNameFinderModel.class));
+          ServiceUtil.getService(modelService, TokenNameFinderModel.class));
       
-      List<Span> names = new ArrayList<Span>();
+      List<Span[]> names = new ArrayList<Span[]>();
       
       for (String sentence[] : document) {
-        names.addAll(Arrays.asList(nameFinder.find(sentence)));
+        names.add(nameFinder.find(sentence));
       }
       
       return names;
     }
     finally {
-      ModelUtil.releaseService(modelService);
+      ServiceUtil.releaseService(modelService);
     }
   }
-  
-  // Just a hack to get arround cross domain issues in my test environment!
-  // Need to investigate how this should be done!
-  @OPTIONS
-  @Path("_findRawText")
-  public Response findRawTextOptions() {
-    System.out.println("Called options ...");
-    return Response.ok()
-        .header("Access-Control-Allow-Origin", "*")
-        .header("Access-Control-Allow-Headers", "Content-Type")
-        .header("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
-        .build();
-  }
 
+  // TODO:
+  // User should pass a key for the models (e.g. default_eng)
   @POST
   @Consumes(MediaType.APPLICATION_JSON)
   @Produces(MediaType.APPLICATION_JSON)
   @Path("_findRawText")
-  public NameFinderDocument findRawText(String document) { // input could be a single string ... return contains everything!
-    
-    
-    System.out.println("Request: " + document);
-    String[][] tokenizedSentences = new String[][]{SimpleTokenizer.INSTANCE.tokenize(document)};
-    
-    // TODO: Fix this. User should be able to define this in blueprint!
+  public NameFinderDocument findRawText(String document) {
+
+    ServiceReference preprocessFactoryService = ServiceUtil.getServiceReference(RawTextNameFinderFactory.class);
     
-    ServiceReference modelService = ModelUtil.getService(TokenNameFinderModel.class);
-      
     try {
-      NameFinderME nameFinder = new NameFinderME(
-              ModelUtil.getModel(modelService, TokenNameFinderModel.class));
-
-      List<Span[]> names = new ArrayList<Span[]>();
-      
-      for (String sentence[] : tokenizedSentences) {
-        names.add(nameFinder.find(sentence));
+      // TODO: Pass a key here!
+      RawTextNameFinderFactory factory =
+              ServiceUtil.getService(preprocessFactoryService, RawTextNameFinderFactory.class);
+      
+      SentenceDetector sentDetect = factory.createSentenceDetector();
+      Tokenizer tokenizer = factory.createTokenizer();
+      
+      Span sentenceSpans[] = sentDetect.sentPosDetect(document);
+      
+      List<Span[]> tokenizedSentencesSpan = new ArrayList<Span[]>();
+      String[][] tokenizedSentences = new String[sentenceSpans.length][];
+      
+      for (int i = 0; i < sentenceSpans.length; i++) {
+        Span tokenSpans[] = tokenizer.tokenizePos(sentenceSpans[i].getCoveredText(document).toString());
+        tokenizedSentencesSpan.add(tokenSpans);
+        
+        String tokens[] = new String[tokenSpans.length];
+        for (int ti = 0; ti < tokenSpans.length; ti++) {
+          tokens[ti] = tokenSpans[ti].getCoveredText(document).toString();
+        }
+        
+        tokenizedSentences[i] = tokens;
       }
       
-      return new NameFinderDocument(tokenizedSentences, names);
+      TokenNameFinder nameFinder = factory.createNameFinder();
+      
+      return new NameFinderDocument(tokenizedSentencesSpan, find(nameFinder, tokenizedSentences));
     }
     finally {
-      ModelUtil.releaseService(modelService);
+      ServiceUtil.releaseService(preprocessFactoryService);
     }
   }
 }

Copied: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/RawTextNameFinderFactory.java (from r1331314, opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderPreprocessFactory.java)
URL: http://svn.apache.org/viewvc/opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/RawTextNameFinderFactory.java?p2=opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/RawTextNameFinderFactory.java&p1=opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderPreprocessFactory.java&r1=1331314&r2=1362511&rev=1362511&view=diff
==============================================================================
--- opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/NameFinderPreprocessFactory.java (original)
+++ opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/RawTextNameFinderFactory.java Tue Jul 17 14:30:50 2012
@@ -17,10 +17,16 @@
 
 package org.apache.opennlp.tagging_server.namefind;
 
+import opennlp.tools.namefind.TokenNameFinder;
 import opennlp.tools.sentdetect.SentenceDetector;
 import opennlp.tools.tokenize.Tokenizer;
 
-public interface NameFinderPreprocessFactory {
+/**
+ * The raw text name finder factory supports the creation of all
+ * components which are needed to process raw text with the name finder.
+ */
+public interface RawTextNameFinderFactory {
   SentenceDetector createSentenceDetector();
   Tokenizer createTokenizer();
+  TokenNameFinder createNameFinder();
 }
\ No newline at end of file

Propchange: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/namefind/RawTextNameFinderFactory.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/postag/POSTaggerResource.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/postag/POSTaggerResource.java?rev=1362511&r1=1362510&r2=1362511&view=diff
==============================================================================
--- opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/postag/POSTaggerResource.java (original)
+++ opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/postag/POSTaggerResource.java Tue Jul 17 14:30:50 2012
@@ -27,7 +27,7 @@ import opennlp.tools.postag.POSModel;
 import opennlp.tools.postag.POSTagger;
 import opennlp.tools.postag.POSTaggerME;
 
-import org.apache.opennlp.tagging_server.ModelUtil;
+import org.apache.opennlp.tagging_server.ServiceUtil;
 import org.osgi.framework.ServiceReference;
 
 @Path("/postagger")
@@ -41,10 +41,10 @@ public class POSTaggerResource {
   // @QueryParam("model") String modelName
   public String[][] tag(String document[][]) {
     
-    ServiceReference modelService = ModelUtil.getService(POSModel.class);
+    ServiceReference modelService = ServiceUtil.getServiceReference(POSModel.class);
     
     try {
-      POSTagger tagger = new POSTaggerME(ModelUtil.getModel(modelService, POSModel.class));
+      POSTagger tagger = new POSTaggerME(ServiceUtil.getService(modelService, POSModel.class));
       
       String[][] tags = new String[document.length][];
       
@@ -55,7 +55,7 @@ public class POSTaggerResource {
       return tags;
     }
     finally {
-      ModelUtil.releaseService(modelService);
+      ServiceUtil.releaseService(modelService);
     }
   }
 }

Added: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/sentdetect/SentenceDetectorResource.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/sentdetect/SentenceDetectorResource.java?rev=1362511&view=auto
==============================================================================
--- opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/sentdetect/SentenceDetectorResource.java (added)
+++ opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/sentdetect/SentenceDetectorResource.java Tue Jul 17 14:30:50 2012
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.opennlp.tagging_server.sentdetect;
+
+import java.util.Arrays;
+import java.util.List;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+
+import opennlp.tools.sentdetect.SentenceDetector;
+import opennlp.tools.sentdetect.SentenceDetectorME;
+import opennlp.tools.sentdetect.SentenceModel;
+import opennlp.tools.util.Span;
+
+import org.apache.opennlp.tagging_server.ServiceUtil;
+import org.osgi.framework.ServiceReference;
+
+@Path("/sentdetect")
+public class SentenceDetectorResource {
+
+  @POST
+  @Consumes(MediaType.APPLICATION_JSON)
+  @Produces(MediaType.APPLICATION_JSON)
+  @Path("_sentPosDetect")
+  public List<Span> sentPosDetect(String document) {
+    
+    ServiceReference modelService = ServiceUtil.getServiceReference(SentenceModel.class);
+      
+    try {
+      SentenceDetector sentDetector = new SentenceDetectorME(
+              ServiceUtil.getService(modelService, SentenceModel.class));
+      
+      return Arrays.asList(sentDetector.sentPosDetect(document));
+    }
+    finally {
+      ServiceUtil.releaseService(modelService);
+    }
+  }
+}

Propchange: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/sentdetect/SentenceDetectorResource.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/tokenize/TokenizerResource.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/tokenize/TokenizerResource.java?rev=1362511&view=auto
==============================================================================
--- opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/tokenize/TokenizerResource.java (added)
+++ opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/tokenize/TokenizerResource.java Tue Jul 17 14:30:50 2012
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.opennlp.tagging_server.tokenize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+
+import opennlp.tools.tokenize.TokenizerME;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.util.Span;
+
+import org.apache.opennlp.tagging_server.ServiceUtil;
+import org.osgi.framework.ServiceReference;
+
+@Path("/tokenize")
+public class TokenizerResource {
+
+  @POST
+  @Consumes(MediaType.APPLICATION_JSON)
+  @Produces(MediaType.APPLICATION_JSON)
+  @Path("_tokenize")
+  public List<String[]> tokenize(String[] document) {
+    ServiceReference modelService = ServiceUtil.getServiceReference(TokenizerModel.class);
+    
+    try {
+      TokenizerME tokenizer = new TokenizerME(
+          ServiceUtil.getService(modelService, TokenizerModel.class));
+      
+      List<String[]> tokenizedSentences = new ArrayList<String[]>();
+      
+      for (String sentence : document) {
+        tokenizedSentences.add(tokenizer.tokenize(sentence));
+      }
+      
+      return tokenizedSentences;
+    }
+    finally {
+      ServiceUtil.releaseService(modelService);
+    }
+  }
+  
+  @POST
+  @Consumes(MediaType.APPLICATION_JSON)
+  @Produces(MediaType.APPLICATION_JSON)
+  @Path("_tokenizePos")
+  public List<Span[]> tokenizePos(String[] document) {
+    ServiceReference modelService = ServiceUtil.getServiceReference(TokenizerModel.class);
+    
+    try {
+      TokenizerME tokenizer = new TokenizerME(
+              ServiceUtil.getService(modelService, TokenizerModel.class));
+
+      List<Span[]> tokenizedSentences = new ArrayList<Span[]>();
+      
+      for (String sentence : document) {
+        tokenizedSentences.add(tokenizer.tokenizePos(sentence));
+      }
+      
+      return tokenizedSentences;
+    }
+    finally {
+      ServiceUtil.releaseService(modelService);
+    }
+  }
+}

Propchange: opennlp/sandbox/tagging-server/src/main/java/org/apache/opennlp/tagging_server/tokenize/TokenizerResource.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain