You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by re...@apache.org on 2014/04/18 23:10:36 UTC
svn commit: r1588568 - in /uima/uimafit/trunk: ./
uimafit-core/src/main/java/org/apache/uima/fit/pipeline/
uimafit-core/src/test/java/org/apache/uima/fit/pipeline/
Author: rec
Date: Fri Apr 18 21:10:35 2014
New Revision: 1588568
URL: http://svn.apache.org/r1588568
Log:
[UIMA-3671] Resources cannot be shared between reader and analysis engines in SimplePipeline
Modified:
uima/uimafit/trunk/ (props changed)
uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/JCasIterable.java
uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/SimplePipeline.java
uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/JCasIterableTest.java
uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/SimplePipelineTest.java
Propchange: uima/uimafit/trunk/
------------------------------------------------------------------------------
Merged /uima/uimafit/branches/2.0.x:r1588567
Modified: uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/JCasIterable.java
URL: http://svn.apache.org/viewvc/uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/JCasIterable.java?rev=1588568&r1=1588567&r2=1588568&view=diff
==============================================================================
--- uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/JCasIterable.java (original)
+++ uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/JCasIterable.java Fri Apr 18 21:10:35 2014
@@ -18,20 +18,26 @@
*/
package org.apache.uima.fit.pipeline;
-import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine;
import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
-import static org.apache.uima.fit.factory.CollectionReaderFactory.createReader;
import org.apache.uima.UIMAException;
+import org.apache.uima.UIMAFramework;
+import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
+import org.apache.uima.collection.CollectionReader;
import org.apache.uima.collection.CollectionReaderDescription;
import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceManager;
/**
+ * <p>
* A class implementing iteration over a the documents of a collection. Each element in the Iterable
* is a JCas containing a single document. The documents have been loaded by the CollectionReader
* and processed by the AnalysisEngine (if any).
- *
+ * </p>
+ * <p>
+ * External resources can be shared between the reader and the analysis engines.
+ * </p>
*/
public class JCasIterable implements Iterable<JCas> {
@@ -58,8 +64,18 @@ public class JCasIterable implements Ite
public JCasIterator iterator() {
try {
- JCasIterator i = new JCasIterator(createReader(reader),
- createEngine(createEngineDescription(engines)));
+ ResourceManager resMgr = UIMAFramework.newDefaultResourceManager();
+
+ // Create the components
+ CollectionReader readerInst = UIMAFramework.produceCollectionReader(reader, resMgr, null);
+
+ // Create AAE
+ AnalysisEngineDescription aaeDesc = createEngineDescription(engines);
+
+ // Instantiate AAE
+ AnalysisEngine aaeInst = UIMAFramework.produceAnalysisEngine(aaeDesc, resMgr, null);
+
+ JCasIterator i = new JCasIterator(readerInst, aaeInst);
i.setSelfComplete(true);
i.setSelfDestroy(true);
return i;
Modified: uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/SimplePipeline.java
URL: http://svn.apache.org/viewvc/uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/SimplePipeline.java?rev=1588568&r1=1588567&r2=1588568&view=diff
==============================================================================
--- uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/SimplePipeline.java (original)
+++ uima/uimafit/trunk/uimafit-core/src/main/java/org/apache/uima/fit/pipeline/SimplePipeline.java Fri Apr 18 21:10:35 2014
@@ -31,6 +31,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.uima.UIMAException;
+import org.apache.uima.UIMAFramework;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
@@ -40,6 +41,7 @@ import org.apache.uima.collection.Collec
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.Resource;
import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.ResourceManager;
import org.apache.uima.resource.metadata.ResourceMetaData;
import org.apache.uima.util.CasCreationUtils;
@@ -52,10 +54,16 @@ public final class SimplePipeline {
}
/**
+ * <p>
* Run the CollectionReader and AnalysisEngines as a pipeline. After processing all CASes provided
* by the reader, the method calls {@link AnalysisEngine#collectionProcessComplete()
* collectionProcessComplete()} on the engines and {@link Resource#destroy() destroy()} on all
* engines.
+ * </p>
+ * <p>
+ * Note that with this method, external resources cannot be shared between the reader and the
+ * analysis engines. They can be shared amongst the analysis engines.
+ * </p>
*
* @param reader
* The CollectionReader that loads the documents into the CAS.
@@ -97,10 +105,15 @@ public final class SimplePipeline {
}
/**
+ * <p>
* Run the CollectionReader and AnalysisEngines as a pipeline. After processing all CASes provided
* by the reader, the method calls {@link AnalysisEngine#collectionProcessComplete()
* collectionProcessComplete()} on the engines, {@link CollectionReader#close() close()} on the
* reader and {@link Resource#destroy() destroy()} on the reader and all engines.
+ * </p>
+ * <p>
+ * External resources can be shared between the reader and the analysis engines.
+ * </p>
*
* @param readerDesc
* The CollectionReader that loads the documents into the CAS.
@@ -115,22 +128,47 @@ public final class SimplePipeline {
*/
public static void runPipeline(final CollectionReaderDescription readerDesc,
final AnalysisEngineDescription... descs) throws UIMAException, IOException {
+ ResourceManager resMgr = UIMAFramework.newDefaultResourceManager();
+
// Create the components
- final CollectionReader reader = createReader(readerDesc);
+ final CollectionReader reader = UIMAFramework.produceCollectionReader(readerDesc, resMgr, null);
+
+ // Create AAE
+ final AnalysisEngineDescription aaeDesc = createEngineDescription(descs);
+
+ // Instantiate AAE
+ final AnalysisEngine aae = UIMAFramework.produceAnalysisEngine(aaeDesc, resMgr, null);
+
+ // Create CAS from merged metadata
+ final CAS cas = CasCreationUtils.createCas(asList(reader.getMetaData(), aae.getMetaData()));
+ reader.typeSystemInit(cas.getTypeSystem());
try {
- // Run the pipeline
- runPipeline(reader, descs);
+ // Process
+ while (reader.hasNext()) {
+ reader.getNext(cas);
+ aae.process(cas);
+ cas.reset();
+ }
+
+ // Signal end of processing
+ aae.collectionProcessComplete();
} finally {
- close(reader);
- destroy(reader);
+ // Destroy
+ aae.destroy();
}
}
/**
+ * <p>
* Provides a simple way to run a pipeline for a given collection reader and sequence of analysis
* engines. After processing all CASes provided by the reader, the method calls
* {@link AnalysisEngine#collectionProcessComplete() collectionProcessComplete()} on the engines.
+ * </p>
+ * <p>
+ * External resources can only be shared between the reader and/or the analysis engines if the
+ * reader/engines have been previously instantiated using a shared resource manager.
+ * </p>
*
* @param reader
* a collection reader
@@ -162,8 +200,13 @@ public final class SimplePipeline {
}
/**
+ * <p>
* Run a sequence of {@link AnalysisEngine analysis engines} over a {@link JCas}. The result of
* the analysis can be read from the JCas.
+ * </p>
+ * <p>
+ * External resources can be shared between the analysis engines.
+ * </p>
*
* @param aCas
* the CAS to process
@@ -194,8 +237,13 @@ public final class SimplePipeline {
}
/**
+ * <p>
* Run a sequence of {@link AnalysisEngine analysis engines} over a {@link JCas}. The result of
* the analysis can be read from the JCas.
+ * </p>
+ * <p>
+ * External resources can be shared between the analysis engines.
+ * </p>
*
* @param jCas
* the jCas to process
@@ -212,9 +260,16 @@ public final class SimplePipeline {
}
/**
+ * <p>
* Run a sequence of {@link AnalysisEngine analysis engines} over a {@link JCas}. This method does
* not {@link AnalysisEngine#destroy() destroy} the engines or send them other events like
* {@link AnalysisEngine#collectionProcessComplete()}. This is left to the caller.
+ * </p>
+ * <p>
+ * External resources can only be shared between the analysis engines if the engines have been
+ * previously instantiated using a shared resource manager.
+ * </p>
+ *
*
* @param jCas
* the jCas to process
@@ -231,9 +286,15 @@ public final class SimplePipeline {
}
/**
+ * <p>
* Run a sequence of {@link AnalysisEngine analysis engines} over a {@link CAS}. This method does
* not {@link AnalysisEngine#destroy() destroy} the engines or send them other events like
* {@link AnalysisEngine#collectionProcessComplete()}. This is left to the caller.
+ * </p>
+ * <p>
+ * External resources can only be shared between the analysis engines if the engines have been
+ * previously instantiated using a shared resource manager.
+ * </p>
*
* @param cas
* the CAS to process
@@ -250,8 +311,13 @@ public final class SimplePipeline {
}
/**
+ * <p>
* Iterate through the {@link JCas JCases} processed by the pipeline, allowing to access each one
* after it has been processed.
+ * </p>
+ * <p>
+ * External resources can be shared between the reader and the analysis engines.
+ * </p>
*
* @param aReader
* the collection reader.
Modified: uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/JCasIterableTest.java
URL: http://svn.apache.org/viewvc/uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/JCasIterableTest.java?rev=1588568&r1=1588567&r2=1588568&view=diff
==============================================================================
--- uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/JCasIterableTest.java (original)
+++ uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/JCasIterableTest.java Fri Apr 18 21:10:35 2014
@@ -21,17 +21,27 @@ package org.apache.uima.fit.pipeline;
import static org.apache.uima.fit.pipeline.SimplePipeline.*;
import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
import static org.apache.uima.fit.factory.CollectionReaderFactory.createReaderDescription;
+import static org.apache.uima.fit.factory.ExternalResourceFactory.createExternalResourceDescription;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.component.JCasCollectionReader_ImplBase;
+import org.apache.uima.fit.component.Resource_ImplBase;
+import org.apache.uima.fit.descriptor.ExternalResource;
+import org.apache.uima.fit.pipeline.SimplePipelineTest.Annotator;
+import org.apache.uima.fit.pipeline.SimplePipelineTest.DummySharedResource;
+import org.apache.uima.fit.pipeline.SimplePipelineTest.Reader;
+import org.apache.uima.fit.pipeline.SimplePipelineTest.Writer;
import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ExternalResourceDescription;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.util.Progress;
import org.apache.uima.util.ProgressImpl;
@@ -51,7 +61,32 @@ public class JCasIterableTest {
assertTrue(GetTextAE.destroyed);
}
+ @Test
+ public void testResourceSharing() throws Exception {
+ ThreeDocsReader.resource = null;
+ GetTextAE.resource = null;
+
+ ExternalResourceDescription res = createExternalResourceDescription(DummySharedResource.class);
+ for (@SuppressWarnings("unused") JCas jcas : iteratePipeline(
+ createReaderDescription(ThreeDocsReader.class, "resource", res),
+ createEngineDescription(GetTextAE.class, "resource", res))) {
+ }
+
+ assertNotNull(ThreeDocsReader.resource);
+ assertNotNull(GetTextAE.resource);
+ assertTrue(ThreeDocsReader.resource == GetTextAE.resource);
+
+ ThreeDocsReader.resource = null;
+ GetTextAE.resource = null;
+ }
+
+ public static final class DummySharedResource extends Resource_ImplBase {
+ }
+
public static final class ThreeDocsReader extends JCasCollectionReader_ImplBase {
+ @ExternalResource(mandatory=false)
+ private static DummySharedResource resource;
+
private final int N = 3;
private int n = 0;
@@ -71,6 +106,9 @@ public class JCasIterableTest {
}
public static final class GetTextAE extends JCasAnnotator_ImplBase {
+ @ExternalResource(mandatory=false)
+ private static DummySharedResource resource;
+
public static String lastText = null;
public static boolean complete = false;
Modified: uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/SimplePipelineTest.java
URL: http://svn.apache.org/viewvc/uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/SimplePipelineTest.java?rev=1588568&r1=1588567&r2=1588568&view=diff
==============================================================================
--- uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/SimplePipelineTest.java (original)
+++ uima/uimafit/trunk/uimafit-core/src/test/java/org/apache/uima/fit/pipeline/SimplePipelineTest.java Fri Apr 18 21:10:35 2014
@@ -22,7 +22,9 @@ import static org.apache.uima.fit.factor
import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
import static org.apache.uima.fit.factory.CollectionReaderFactory.createReader;
import static org.apache.uima.fit.factory.CollectionReaderFactory.createReaderDescription;
+import static org.apache.uima.fit.factory.ExternalResourceFactory.createExternalResourceDescription;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -36,9 +38,12 @@ import org.apache.uima.cas.TypeSystem;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.component.JCasCollectionReader_ImplBase;
+import org.apache.uima.fit.component.Resource_ImplBase;
+import org.apache.uima.fit.descriptor.ExternalResource;
import org.apache.uima.fit.type.Sentence;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ExternalResourceDescription;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.util.Progress;
import org.junit.Test;
@@ -51,6 +56,9 @@ public class SimplePipelineTest {
public static class Reader extends JCasCollectionReader_ImplBase {
+ @ExternalResource(mandatory=false)
+ private static DummySharedResource resource;
+
private int size = 1;
private int current = 0;
@@ -80,6 +88,8 @@ public class SimplePipelineTest {
}
public static class Annotator extends JCasAnnotator_ImplBase {
+ @ExternalResource(mandatory=false)
+ private static DummySharedResource resource;
@Override
public void process(JCas jCas) throws AnalysisEngineProcessException {
@@ -109,6 +119,9 @@ public class SimplePipelineTest {
}
+ public static final class DummySharedResource extends Resource_ImplBase {
+ }
+
@Test
public void testWithInstances() throws Exception {
SimplePipeline.runPipeline(createReader(Reader.class),
@@ -124,4 +137,21 @@ public class SimplePipelineTest {
createEngineDescription(Writer.class));
assertEquals(Arrays.asList(SENTENCE_TEXT), Writer.SENTENCES);
}
+
+ @Test
+ public void testResourceSharing() throws Exception {
+ Reader.resource = null;
+ Annotator.resource = null;
+
+ ExternalResourceDescription res = createExternalResourceDescription(DummySharedResource.class);
+ SimplePipeline.runPipeline(createReaderDescription(Reader.class, "resource", res),
+ createEngineDescription(Annotator.class, "resource", res));
+
+ assertNotNull(Reader.resource);
+ assertNotNull(Annotator.resource);
+ assertTrue(Reader.resource == Annotator.resource);
+
+ Reader.resource = null;
+ Annotator.resource = null;
+ }
}