You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ctakes.apache.org by se...@apache.org on 2022/05/11 00:27:34 UTC
svn commit: r1900795 - in /ctakes/trunk: ctakes-assertion-res/src/main/resources/org/apache/ctakes/assertion/pipeline/ ctakes-core/src/main/java/org/apache/ctakes/core/ae/ ctakes-core/src/main/java/org/apache/ctakes/core/cr/ ctakes-core/src/main/java/o...
Author: seanfinan
Date: Wed May 11 00:27:34 2022
New Revision: 1900795
URL: http://svn.apache.org/viewvc?rev=1900795&view=rev
Log:
Add env command to PiperFileReader and PipelineBuilder
Comment in AttributeCleartkSubPipe.piper
add CommandRunner, CtakesRunner
add SystemUtil
AbstractFileTreeReader can WriteBanner
add BannerWriter
FinishedLogger logs without prefix
Added:
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CommandRunner.java
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CtakesRunner.java
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/PbjStarter.java
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/BannerWriter.java
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/external/
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/external/SystemUtil.java
Modified:
ctakes/trunk/ctakes-assertion-res/src/main/resources/org/apache/ctakes/assertion/pipeline/AttributeCleartkSubPipe.piper
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/cr/AbstractFileTreeReader.java
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PipelineBuilder.java
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PiperFileReader.java
ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/log/FinishedLogger.java
Modified: ctakes/trunk/ctakes-assertion-res/src/main/resources/org/apache/ctakes/assertion/pipeline/AttributeCleartkSubPipe.piper
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-assertion-res/src/main/resources/org/apache/ctakes/assertion/pipeline/AttributeCleartkSubPipe.piper?rev=1900795&r1=1900794&r2=1900795&view=diff
==============================================================================
--- ctakes/trunk/ctakes-assertion-res/src/main/resources/org/apache/ctakes/assertion/pipeline/AttributeCleartkSubPipe.piper (original)
+++ ctakes/trunk/ctakes-assertion-res/src/main/resources/org/apache/ctakes/assertion/pipeline/AttributeCleartkSubPipe.piper Wed May 11 00:27:34 2022
@@ -1,8 +1,10 @@
// Commands and parameters to create a default entity attributes processing sub-pipeline. This is not a full pipeline.
-// Add the Dependency parser for use by cleartk
+// Add the Dependency parser for use by SubjectCleartkAnalysisEngine.
+// If you remove the Subject annotator you might be able to remove the dependency parser.
addDescription ClearNLPDependencyParserAE
-// Add the Semantic Role Labeler parser for use by cleartk
+// Add the Semantic Role Labeler parser for use by cleartk.
+// Not necessary for assertion anymore, but it is used for temporal, coref, and others so be careful removing this.
addLogged ClearNLPSemanticRoleLabelerAE
// Add the cleartk package for cleartk class lookups
Added: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CommandRunner.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CommandRunner.java?rev=1900795&view=auto
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CommandRunner.java (added)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CommandRunner.java Wed May 11 00:27:34 2022
@@ -0,0 +1,169 @@
+package org.apache.ctakes.core.ae;
+
+import org.apache.ctakes.core.pipeline.PipeBitInfo;
+import org.apache.ctakes.core.util.external.SystemUtil;
+import org.apache.ctakes.core.util.log.DotLogger;
+import org.apache.log4j.Logger;
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
+import org.apache.uima.fit.descriptor.ConfigurationParameter;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+
+import java.io.IOException;
+
+/**
+ * @author SPF , chip-nlp
+ * @since {5/4/2022}
+ */
+@PipeBitInfo(
+ name = "CommandRunner",
+ description = "Runs an external process.",
+ role = PipeBitInfo.Role.SPECIAL
+)
+public class CommandRunner extends JCasAnnotator_ImplBase {
+
+ static private final Logger LOGGER = Logger.getLogger( "CommandRunner" );
+
+ // to add a configuration parameter, type "param" and hit tab.
+ static public final String CMD_PARAM = "Command";
+ static public final String CMD_DESC = "A full command line to be executed. Make sure to quote.";
+ @ConfigurationParameter(
+ name = CMD_PARAM,
+ description = CMD_DESC
+ )
+ private String _cmd;
+
+ static public final String DIR_PARAM = "Dir";
+ static public final String DIR_DESC = "Set a value for parameter Command's root directory.";
+ @ConfigurationParameter(
+ name = DIR_PARAM,
+ description = DIR_DESC,
+ mandatory = false
+ )
+ private String _dir;
+
+
+ static public final String PER_DOC_PARAM = "PerDoc";
+ static public final String PER_DOC_DESC = "yes to run the command once per document. Default is no.";
+ @ConfigurationParameter(
+ name = PER_DOC_PARAM,
+ description = PER_DOC_DESC,
+ defaultValue = "no",
+ mandatory = false
+ )
+ private String _perDoc;
+
+ static public final String PAUSE_PARAM = "Pause";
+ static public final String PAUSE_DESC = "Pause for some seconds after launching. Default is 0";
+ @ConfigurationParameter(
+ name = PAUSE_PARAM,
+ description = PAUSE_DESC,
+ mandatory = false
+ )
+ private int _pause = 0;
+
+ static public final String WAIT_PARAM = "Wait";
+ static public final String WAIT_DESC = "Wait for the launched command to finish. Default is no.";
+ @ConfigurationParameter(
+ name = WAIT_PARAM,
+ description = WAIT_DESC,
+ defaultValue = "no",
+ mandatory = false
+ )
+ private String _wait;
+
+ static public final String LOG_NAME_PARAM = "Log";
+ static public final String LOG_NAME_DESC = "Set a name for the CTAKES logger. Default is the Command.";
+ @ConfigurationParameter(
+ name = LOG_NAME_PARAM,
+ description = LOG_NAME_DESC,
+ mandatory = false
+ )
+ private String _logName;
+
+
+ static public final String LOG_FILE_PARAM = "LogFile";
+ static public final String LOG_FILE_DESC = "File to which the command's output should be sent. This overrides Log.";
+ @ConfigurationParameter(
+ name = LOG_FILE_PARAM,
+ description = LOG_FILE_DESC,
+ mandatory = false
+ )
+ private String _logFile;
+
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void initialize( final UimaContext context ) throws ResourceInitializationException {
+ super.initialize( context );
+ if ( _perDoc.equalsIgnoreCase( "yes" ) || _perDoc.equalsIgnoreCase( "true" ) ) {
+ return;
+ }
+ try {
+ runCommand();
+ } catch ( IOException ioE ) {
+ throw new ResourceInitializationException( ioE );
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void process( final JCas jcas ) throws AnalysisEngineProcessException {
+ if ( !_perDoc.equalsIgnoreCase( "yes" ) && !_perDoc.equalsIgnoreCase( "true" ) ) {
+ return;
+ }
+ try {
+ runCommand();
+ } catch ( IOException ioE ) {
+ throw new AnalysisEngineProcessException( ioE );
+ }
+ }
+
+
+ private void runCommand() throws IOException {
+ final SystemUtil.CommandRunner runner = new SystemUtil.CommandRunner( _cmd );
+ if ( _logFile != null && !_logFile.isEmpty() ) {
+ runner.setLogFiles( _logFile, _logFile );
+ } else {
+ final Logger logger = getRunLogger();
+ runner.setLogger( logger );
+ }
+ if ( _wait.equalsIgnoreCase( "yes" ) || _wait.equalsIgnoreCase( "true" ) ) {
+ runner.wait( true );
+ }
+ if ( _dir != null && !_dir.isEmpty() ) {
+ runner.setDirectory( _dir );
+ }
+ LOGGER.info( "Running " + _cmd + " ..." );
+ SystemUtil.run( runner );
+ if ( _pause < 1 ) {
+ return;
+ }
+ final long pause = _pause * 1000L;
+ LOGGER.info( "Pausing " + _pause + " seconds ..." );
+ try ( DotLogger dotter = new DotLogger() ) {
+ Thread.sleep( pause );
+ } catch ( IOException | InterruptedException multE ) {
+ // do nothing
+ }
+ }
+
+ private Logger getRunLogger() {
+ if ( _logName != null && !_logName.isEmpty() ) {
+ return Logger.getLogger( _logName );
+ }
+ final int spaceIndex = _cmd.indexOf( ' ' );
+ if ( spaceIndex < 0 ) {
+ return Logger.getLogger( _cmd );
+ }
+ return Logger.getLogger( _cmd.substring( 0, spaceIndex ) );
+ }
+
+
+}
Added: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CtakesRunner.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CtakesRunner.java?rev=1900795&view=auto
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CtakesRunner.java (added)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/CtakesRunner.java Wed May 11 00:27:34 2022
@@ -0,0 +1,129 @@
+package org.apache.ctakes.core.ae;
+
+import org.apache.ctakes.core.pipeline.PipeBitInfo;
+import org.apache.ctakes.core.util.external.SystemUtil;
+import org.apache.ctakes.core.util.log.DotLogger;
+import org.apache.log4j.Logger;
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_engine.AnalysisEngineDescription;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
+import org.apache.uima.fit.descriptor.ConfigurationParameter;
+import org.apache.uima.fit.factory.AnalysisEngineFactory;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+
+import java.io.IOException;
+
+/**
+ * @author SPF , chip-nlp
+ * @since {5/10/2022}
+ */
+@PipeBitInfo(
+ name = "CtakesRunner",
+ description = "Starts a new instance of cTAKES with the given piper parameters.",
+ role = PipeBitInfo.Role.SPECIAL
+)
+public class CtakesRunner extends JCasAnnotator_ImplBase {
+
+ static private final Logger LOGGER = Logger.getLogger( "CtakesRunner" );
+
+ static public final String CLI_PARAM = "Pipe";
+ static public final String CLI_DESC = "Piper parameters. Make sure to quote.";
+ @ConfigurationParameter(
+ name = CLI_PARAM,
+ description = CLI_DESC
+ )
+ private String _cli;
+
+ static public final String LOG_FILE_PARAM = "LogFile";
+ static public final String LOG_FILE_DESC = "File to which cTAKES output should be sent.";
+ @ConfigurationParameter(
+ name = LOG_FILE_PARAM,
+ description = LOG_FILE_DESC,
+ mandatory = false
+ )
+ private String _logFile;
+
+ static public final String PAUSE_PARAM = "Pause";
+ static public final String PAUSE_DESC = "Pause for some seconds after launching. Default is 0";
+ @ConfigurationParameter(
+ name = PAUSE_PARAM,
+ description = PAUSE_DESC,
+ mandatory = false
+ )
+ private int _pause = 0;
+
+ static private final String JAVA_CMD = "java -Xms512M -Xmx3g org.apache.ctakes.core.pipeline.PiperFileRunner";
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void initialize( final UimaContext context ) throws ResourceInitializationException {
+ super.initialize( context );
+ try {
+ final String piper = getPiper();
+ if ( _logFile == null || _logFile.isEmpty() ) {
+ _logFile = "ctakes_" + piper + ".log";
+ }
+ runCommand();
+ } catch ( IOException ioE ) {
+ throw new ResourceInitializationException( ioE );
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void process( final JCas jcas ) throws AnalysisEngineProcessException {
+ // Implementation of the process(..) method is mandatory, even if it does nothing.
+ }
+
+
+ private String getPiper() throws IOException {
+ final int pIndex = _cli.indexOf( "-p " );
+ if ( pIndex < 0 ) {
+ throw new IOException( "Improper Piper Runner Specification " + _cli );
+ }
+ final int spaceIndex = _cli.indexOf( ' ', pIndex + 4 );
+ if ( spaceIndex < 4 ) {
+ throw new IOException( "Improper Piper Runner Specification " + _cli );
+ }
+ String piper = _cli.substring( pIndex + 3, spaceIndex );
+ int slashIndex = piper.lastIndexOf( '/' );
+ if ( slashIndex < 0 ) {
+ slashIndex = piper.lastIndexOf( '\\' );
+ }
+ if ( slashIndex >= 0 ) {
+ piper = piper.substring( slashIndex + 1 );
+ }
+ return piper;
+ }
+
+ private void runCommand() throws IOException {
+ final SystemUtil.CommandRunner runner = new SystemUtil.CommandRunner( JAVA_CMD + " " + _cli );
+ runner.setLogFiles( _logFile, _logFile );
+ LOGGER.info( "Starting cTAKES with " + _cli + " ..." );
+ SystemUtil.run( runner );
+ if ( _pause < 1 ) {
+ return;
+ }
+ final long pause = _pause * 1000L;
+ LOGGER.info( "Pausing " + _pause + " seconds ..." );
+ try ( DotLogger dotter = new DotLogger() ) {
+ Thread.sleep( pause );
+ } catch ( IOException | InterruptedException multE ) {
+ // do nothing
+ }
+ }
+
+
+ static public AnalysisEngineDescription createEngineDescription( final String pipe )
+ throws ResourceInitializationException {
+ return AnalysisEngineFactory.createEngineDescription( CtakesRunner.class, CtakesRunner.CLI_PARAM, pipe );
+ }
+
+
+}
Added: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/PbjStarter.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/PbjStarter.java?rev=1900795&view=auto
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/PbjStarter.java (added)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/ae/PbjStarter.java Wed May 11 00:27:34 2022
@@ -0,0 +1,108 @@
+package org.apache.ctakes.core.ae;
+
+import org.apache.ctakes.core.pipeline.PipeBitInfo;
+import org.apache.ctakes.core.util.external.SystemUtil;
+import org.apache.ctakes.core.util.log.DotLogger;
+import org.apache.log4j.Logger;
+import org.apache.uima.UimaContext;
+import org.apache.uima.analysis_engine.AnalysisEngineDescription;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
+import org.apache.uima.fit.descriptor.ConfigurationParameter;
+import org.apache.uima.fit.factory.AnalysisEngineFactory;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * @author SPF , chip-nlp
+ * @since {5/10/2022}
+ */
+@PipeBitInfo(
+ name = "PbjStarter",
+ description = "Starts the Apache Artemis broker.",
+ role = PipeBitInfo.Role.ANNOTATOR
+)
+public class PbjStarter extends JCasAnnotator_ImplBase {
+
+ static private final Logger LOGGER = Logger.getLogger( "PbjStarter" );
+
+
+ static public final String LOG_FILE_PARAM = "ArtemisLog";
+ static public final String LOG_FILE_DESC = "File to which Artemis output should be sent.";
+ @ConfigurationParameter(
+ name = LOG_FILE_PARAM,
+ description = LOG_FILE_DESC,
+ mandatory = false
+ )
+ private String _logFile = "ctakes_artemis.log";
+
+ static public final String DIR_PARAM = "Artemis";
+ static public final String DIR_DESC = "Set a value for your Artemis root directory.";
+ @ConfigurationParameter(
+ name = DIR_PARAM,
+ description = DIR_DESC,
+ mandatory = false
+ )
+ private String _dir;
+
+ static public final String PAUSE_PARAM = "Pause";
+ static public final String PAUSE_DESC = "Pause for some seconds after launching. Default is 0";
+ @ConfigurationParameter(
+ name = PAUSE_PARAM,
+ description = PAUSE_DESC,
+ mandatory = false
+ )
+ private int _pause = 0;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void initialize( final UimaContext context ) throws ResourceInitializationException {
+ super.initialize( context );
+ try {
+ runCommand();
+ } catch ( IOException ioE ) {
+ throw new ResourceInitializationException( ioE );
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void process( final JCas jcas ) throws AnalysisEngineProcessException {
+ // Implementation of the process(..) method is mandatory, even if it does nothing.
+ }
+
+ private void runCommand() throws IOException {
+ final SystemUtil.CommandRunner runner
+ = new SystemUtil.CommandRunner( "bin" + File.separatorChar + "artemis run" );
+ runner.setLogFiles( _logFile, _logFile );
+ if ( _dir != null && !_dir.isEmpty() ) {
+ runner.setDirectory( _dir );
+ }
+ LOGGER.info( "Starting Apache Artemis ..." );
+ SystemUtil.run( runner );
+ if ( _pause < 1 ) {
+ return;
+ }
+ final long pause = _pause * 1000L;
+ LOGGER.info( "Pausing " + _pause + " seconds ..." );
+ try ( DotLogger dotter = new DotLogger() ) {
+ Thread.sleep( pause );
+ } catch ( IOException | InterruptedException multE ) {
+ // do nothing
+ }
+ }
+
+ static public AnalysisEngineDescription createEngineDescription( final String artemisDir )
+ throws ResourceInitializationException {
+ return AnalysisEngineFactory.createEngineDescription( PbjStarter.class, PbjStarter.DIR_PARAM, artemisDir );
+ }
+
+
+}
Modified: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/cr/AbstractFileTreeReader.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/cr/AbstractFileTreeReader.java?rev=1900795&r1=1900794&r2=1900795&view=diff
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/cr/AbstractFileTreeReader.java (original)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/cr/AbstractFileTreeReader.java Wed May 11 00:27:34 2022
@@ -4,6 +4,7 @@ import org.apache.ctakes.core.config.Con
import org.apache.ctakes.core.patient.PatientNoteStore;
import org.apache.ctakes.core.pipeline.ProgressManager;
import org.apache.ctakes.core.resource.FileLocator;
+import org.apache.ctakes.core.util.BannerWriter;
import org.apache.ctakes.core.util.NumberedSuffixComparator;
import org.apache.ctakes.core.util.doc.JCasBuilder;
import org.apache.ctakes.core.util.doc.NoteSpecs;
@@ -52,6 +53,15 @@ abstract public class AbstractFileTreeRe
static private final Logger LOGGER = Logger.getLogger( "AbstractFileTreeReader" );
+ static public final String PARAM_WRITE_BANNER = "WriteBanner";
+ @ConfigurationParameter(
+ name = PARAM_WRITE_BANNER,
+ description = "Write a large banner at each major step of the pipeline.",
+ mandatory = false,
+ defaultValue = "no"
+ )
+ private String _writeBannerChoice;
+
/**
* Name of configuration parameter that must be set to the path of
* a directory containing input files.
@@ -155,6 +165,7 @@ abstract public class AbstractFileTreeRe
static private final Pattern CR_LF = Pattern.compile( "\\r\\n" );
+ private boolean _writeBanner;
private File _rootDir;
private Collection<String> _validExtensions;
private List<File> _files;
@@ -285,6 +296,11 @@ abstract public class AbstractFileTreeRe
@Override
public void initialize( final UimaContext context ) throws ResourceInitializationException {
super.initialize( context );
+ _writeBanner = _writeBannerChoice.equalsIgnoreCase( "yes" )
+ || _writeBannerChoice.equalsIgnoreCase( "true" );
+ if ( _writeBanner ) {
+ BannerWriter.writeHello();
+ }
try {
_rootDir = FileLocator.getFile( _rootDirPath );
} catch ( FileNotFoundException fnfE ) {
@@ -549,9 +565,16 @@ abstract public class AbstractFileTreeRe
*/
@Override
public boolean hasNext() {
+ if ( _currentIndex == 0 && _writeBanner ) {
+ BannerWriter.writeProcess();
+ }
final boolean hasNext = _currentIndex < _files.size();
if ( !hasNext ) {
- ProgressManager.getInstance().updateProgress( _files.size() );
+ ProgressManager.getInstance()
+ .updateProgress( _files.size() );
+ if ( _writeBanner ) {
+ BannerWriter.writeFinished();
+ }
}
return hasNext;
}
Modified: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PipelineBuilder.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PipelineBuilder.java?rev=1900795&r1=1900794&r2=1900795&view=diff
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PipelineBuilder.java (original)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PipelineBuilder.java Wed May 11 00:27:34 2022
@@ -6,6 +6,7 @@ import org.apache.ctakes.core.cc.pretty.
import org.apache.ctakes.core.config.ConfigParameterConstants;
import org.apache.ctakes.core.cr.FileTreeReader;
import org.apache.ctakes.core.util.PropertyAeFactory;
+import org.apache.ctakes.core.util.external.SystemUtil;
import org.apache.log4j.Logger;
import org.apache.uima.UIMAException;
import org.apache.uima.analysis_component.AnalysisComponent;
@@ -95,11 +96,17 @@ final public class PipelineBuilder {
* @return this PipelineBuilder
*/
public PipelineBuilder setIfEmpty( final Object... parameters ) {
- PropertyAeFactory.getInstance().addIfEmptyParameters( parameters );
+ PropertyAeFactory.getInstance()
+ .addIfEmptyParameters( parameters );
_pipelineChanged = true;
return this;
}
+ public PipelineBuilder env( final Object... parameters ) {
+ SystemUtil.addEnvironmentVariables( parameters );
+ // Pipeline was not changed
+ return this;
+ }
/**
* Use of this method is not order-specific
Modified: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PiperFileReader.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PiperFileReader.java?rev=1900795&r1=1900794&r2=1900795&view=diff
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PiperFileReader.java (original)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/pipeline/PiperFileReader.java Wed May 11 00:27:34 2022
@@ -158,7 +158,8 @@ final public class PiperFileReader {
case "load":
return loadPipelineFile( info );
case "package":
- PipeBitLocator.getInstance().addUserPackage( info );
+ PipeBitLocator.getInstance()
+ .addUserPackage( info );
return true;
case "set":
_builder.set( splitParameters( info ) );
@@ -166,14 +167,19 @@ final public class PiperFileReader {
case "cli":
_builder.setIfEmpty( getCliParameters( info ) );
return true;
+ case "env":
+ _builder.env( splitParameters( info ) );
+ return true;
case "reader":
if ( hasParameters( info ) ) {
final String[] component_parameters = splitFromParameters( info );
final String component = component_parameters[ 0 ];
final Object[] parameters = splitParameters( component_parameters[ 1 ] );
- _builder.reader( PipeBitLocator.getInstance().getReaderClass( component ), parameters );
+ _builder.reader( PipeBitLocator.getInstance()
+ .getReaderClass( component ), parameters );
} else {
- _builder.reader( PipeBitLocator.getInstance().getReaderClass( info ) );
+ _builder.reader( PipeBitLocator.getInstance()
+ .getReaderClass( info ) );
}
return true;
case "readFiles":
Added: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/BannerWriter.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/BannerWriter.java?rev=1900795&view=auto
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/BannerWriter.java (added)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/BannerWriter.java Wed May 11 00:27:34 2022
@@ -0,0 +1,68 @@
+package org.apache.ctakes.core.util;
+
+import org.apache.log4j.Logger;
+
+/**
+ * @author SPF , chip-nlp
+ * @since {1/14/2022}
+ */
+final public class BannerWriter {
+
+ // The ProgressDone logger in our ctakes log4j configuration hides the name of the logger.
+ static private final Logger EOL_LOGGER = Logger.getLogger( "ProgressDone" );
+
+ private BannerWriter() {
+ }
+
+
+ static public void writeHello() {
+ EOL_LOGGER.info( "\n" );
+ EOL_LOGGER.info( " Welcome to" );
+ EOL_LOGGER.info( "\n" );
+ EOL_LOGGER.info( " _/_/ _/" );
+ EOL_LOGGER.info( " _/ _/ _/_/_/ _/_/_/ _/_/_/ _/_/_/ _/_/" );
+ EOL_LOGGER.info( " _/_/_/_/ _/ _/ _/ _/ _/ _/ _/ _/_/_/_/" );
+ EOL_LOGGER.info( " _/ _/ _/ _/ _/ _/ _/ _/ _/ _/" );
+ EOL_LOGGER.info( "_/ _/ _/_/_/ _/_/_/ _/_/_/ _/ _/ _/_/_/" );
+ EOL_LOGGER.info( " _/" );
+ EOL_LOGGER.info( " _/" );
+ EOL_LOGGER.info( " _/_/_/_/_/ _/_/ _/ _/ _/_/_/_/ _/_/_/" );
+ EOL_LOGGER.info( " _/_/_/ _/ _/ _/ _/ _/ _/ _/" );
+ EOL_LOGGER.info( " _/ _/ _/_/_/_/ _/_/ _/_/_/ _/_/" );
+ EOL_LOGGER.info( " _/ _/ _/ _/ _/ _/ _/ _/" );
+ EOL_LOGGER.info( " _/_/_/ _/ _/ _/ _/ _/ _/_/_/_/ _/_/_/" );
+ EOL_LOGGER.info( "\n" );
+ }
+
+ static public void writeInitialize() {
+ EOL_LOGGER.info( "\n" );
+ EOL_LOGGER.info( " _/_/_/ _/ _/ _/ _/ _/" );
+ EOL_LOGGER.info( " _/ _/_/_/ _/_/_/_/ _/_/_/ _/ _/_/_/_/ _/_/" );
+ EOL_LOGGER.info( " _/ _/ _/ _/ _/ _/ _/ _/ _/ _/ _/ _/_/_/_/" );
+ EOL_LOGGER.info( " _/ _/ _/ _/ _/ _/ _/ _/ _/ _/ _/ _/" );
+ EOL_LOGGER.info( "_/_/_/ _/ _/ _/ _/_/ _/ _/_/_/ _/ _/ _/_/_/_/ _/_/_/" );
+ EOL_LOGGER.info( "\n" );
+ }
+
+ static public void writeProcess() {
+ EOL_LOGGER.info( "\n" );
+ EOL_LOGGER.info( " _/_/_/" );
+ EOL_LOGGER.info( " _/ _/ _/ _/_/ _/_/ _/_/_/ _/_/ _/_/_/ _/_/_/" );
+ EOL_LOGGER.info( " _/_/_/ _/_/ _/ _/ _/ _/_/_/_/ _/_/ _/_/" );
+ EOL_LOGGER.info( " _/ _/ _/ _/ _/ _/ _/_/ _/_/" );
+ EOL_LOGGER.info( "_/ _/ _/_/ _/_/_/ _/_/_/ _/_/_/ _/_/_/" );
+ EOL_LOGGER.info( "\n" );
+ }
+
+ static public void writeFinished() {
+ EOL_LOGGER.info( "\n" );
+ EOL_LOGGER.info( " _/_/_/_/ _/ _/ _/ _/" );
+ EOL_LOGGER.info( " _/ _/_/_/ _/_/_/ _/_/_/ _/_/ _/_/_/" );
+ EOL_LOGGER.info( " _/_/_/ _/ _/ _/ _/ _/_/ _/ _/ _/_/_/_/ _/ _/" );
+ EOL_LOGGER.info( " _/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/ _/" );
+ EOL_LOGGER.info( "_/ _/ _/ _/ _/ _/_/_/ _/ _/ _/_/_/ _/_/_/" );
+ EOL_LOGGER.info( "\n" );
+ }
+
+
+}
Added: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/external/SystemUtil.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/external/SystemUtil.java?rev=1900795&view=auto
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/external/SystemUtil.java (added)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/external/SystemUtil.java Wed May 11 00:27:34 2022
@@ -0,0 +1,395 @@
+package org.apache.ctakes.core.util.external;
+
+import org.apache.ctakes.core.resource.FileLocator;
+import org.apache.log4j.Logger;
+
+import java.io.*;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.*;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.*;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/**
+ * @author SPF , chip-nlp
+ * @since {5/4/2022}
+ */
+final public class SystemUtil {
+
+
+ static private final Logger LOGGER = Logger.getLogger( "SystemUtil" );
+
+
+ private SystemUtil() {
+ }
+
+ static private final String ENV_VAR_PREFIX = "ctakes.env.";
+ static public final File NO_FILE = new File( "" );
+ static public final String FILE_NOT_FOUND = "FILE_NOT_FOUND";
+
+
+ /**
+ * Add key value pairs to a set of environment variables for use in external processes.
+ *
+ * @param variables ket value pairs
+ */
+ static public void addEnvironmentVariables( final Object... variables ) {
+ if ( variables.length == 0 ) {
+ LOGGER.warn( "No variables specified." );
+ return;
+ }
+ if ( variables.length % 2 != 0 ) {
+ LOGGER.error( "Odd number of variables provided. Should be key value pairs." );
+ return;
+ }
+ for ( int i = 0; i < variables.length; i += 2 ) {
+ if ( variables[ i ] instanceof String ) {
+ System.setProperty( ENV_VAR_PREFIX + variables[ i ], variables[ i + 1 ].toString() );
+ } else {
+ LOGGER.warn( "Variable" + i + " not a String, using " + variables[ i ].toString() );
+ System.setProperty( ENV_VAR_PREFIX + variables[ i ].toString(), variables[ i + 1 ].toString() );
+ }
+ }
+ }
+
+
+ public boolean copyFile( final String source, final String target ) {
+ final InputStream sourceStream = FileLocator.getStreamQuiet( source );
+ if ( sourceStream == null ) {
+ LOGGER.error( "Cannot access source " + source );
+ return false;
+ }
+ final File targetFile = FileLocator.getFileQuiet( target );
+ if ( targetFile == null ) {
+ LOGGER.error( "Cannot access target file " + target );
+ return false;
+ }
+ targetFile.getParentFile()
+ .mkdirs();
+ Path targetPath;
+ try {
+ targetPath = Paths.get( target );
+ } catch ( InvalidPathException ipE ) {
+ LOGGER.error( "Cannot access target path " + target );
+ return false;
+ }
+ return SystemUtil.copyToDisk( sourceStream, targetPath );
+ }
+
+ static public boolean copyToDisk( final InputStream source, final Path target ) {
+ try {
+ Files.copy( source, target, StandardCopyOption.REPLACE_EXISTING );
+ } catch ( IOException ioE ) {
+ LOGGER.error( ioE.getMessage() );
+ return false;
+ }
+ return true;
+ }
+
+ static public String findExecutable( final String name ) {
+ final String executable = findExecutableInCtakes( name );
+ if ( !FILE_NOT_FOUND.equals( executable ) ) {
+ return executable;
+ }
+ return findExecutableOnPath( name );
+ }
+
+ static public String findExecutableInCtakes( final String name ) {
+ final File executable = FileLocator.getFileQuiet( name );
+ if ( executable != null ) {
+ if ( executable.canExecute() ) {
+ return executable.getAbsolutePath();
+ }
+ }
+ return FILE_NOT_FOUND;
+ }
+
+ static public String findExecutableOnPath( final String name ) {
+ for ( String dirname : System.getenv( "PATH" )
+ .split( File.pathSeparator ) ) {
+ final File testFile = new File( dirname, name );
+ if ( testFile.isFile() && testFile.canExecute() ) {
+ return testFile.getAbsolutePath();
+ }
+ }
+ return FILE_NOT_FOUND;
+ }
+
+
+ static public class FileDownloader implements Callable<File> {
+
+ private final String _url;
+ private final String _tempPrefix;
+ private final String _tempSuffix;
+
+ public FileDownloader( final String url ) {
+ this( url, "Prefix", "suffix" );
+ }
+
+ public FileDownloader( final String url, final String tempPrefix, final String tempSuffix ) {
+ _url = url;
+ _tempPrefix = tempPrefix;
+ _tempSuffix = tempSuffix;
+ }
+
+ public File call() throws IOException {
+ final File tempZip = File.createTempFile( _tempPrefix, _tempSuffix );
+ tempZip.deleteOnExit();
+ URL url = new URL( _url );
+ try ( ReadableByteChannel readableByteChannel = Channels.newChannel( url.openStream() );
+ FileOutputStream fileOutputStream = new FileOutputStream( tempZip );
+ FileChannel fileChannel = fileOutputStream.getChannel() ) {
+ fileChannel.transferFrom( readableByteChannel, 0, Long.MAX_VALUE );
+ }
+ return tempZip;
+ }
+
+ }
+
+
+ static private boolean unzipit( final String zippedFile, final File unzipDir ) throws IOException {
+ final InputStream zippedStream = FileLocator.getStreamQuiet( zippedFile );
+ if ( zippedStream == null ) {
+ LOGGER.error( "Could not access " + zippedFile );
+ return false;
+ }
+ return unzipit( zippedStream, unzipDir );
+ }
+
+ static private boolean unzipit( final File zippedFile, final File unzipDir ) throws IOException {
+ return unzipit( zippedFile.getPath(), unzipDir );
+ }
+
+ static private boolean unzipit( final InputStream zippedStream, final File unzipDir ) throws IOException {
+ final byte[] buffer = new byte[ 1024 ];
+ final ZipInputStream zis = new ZipInputStream( zippedStream );
+ ZipEntry zipEntry = zis.getNextEntry();
+ while ( zipEntry != null ) {
+ if ( zipEntry.isDirectory() ) {
+ final File newUnzipDir = new File( unzipDir, zipEntry.getName() );
+ newUnzipDir.mkdirs();
+ } else {
+ final File newUnzipFile = newUnzipFile( unzipDir, zipEntry );
+ final FileOutputStream fos = new FileOutputStream( newUnzipFile );
+ int len;
+ while ( ( len = zis.read( buffer ) ) > 0 ) {
+ fos.write( buffer, 0, len );
+ }
+ fos.close();
+ }
+ zipEntry = zis.getNextEntry();
+ }
+ zis.closeEntry();
+ zis.close();
+ return true;
+ }
+
+ static public class FileUnzipper implements Callable<File> {
+
+ private final File _zip;
+ private final File _unzipDir;
+
+ public FileUnzipper( final File zip, final File unzipDir ) {
+ _zip = zip;
+ _unzipDir = unzipDir;
+ }
+
+ public File call() throws IOException {
+ unzipit( _zip, _unzipDir );
+ return _unzipDir;
+ }
+
+ }
+
+
+ static private File newUnzipFile( final File unzipDirPath, final ZipEntry zipEntry ) throws IOException {
+ final File unzippedFile = new File( unzipDirPath, zipEntry.getName() );
+
+ final String destDirPath = unzipDirPath.getCanonicalPath();
+ final String destFilePath = unzippedFile.getCanonicalPath();
+
+ if ( !destFilePath.startsWith( destDirPath + File.separator ) ) {
+ throw new IOException( "Entry is outside of the target dir: " + zipEntry.getName() );
+ }
+ return unzippedFile;
+ }
+
+
+ static public boolean run( final String command ) throws IOException {
+ final CommandRunner runner = new CommandRunner( command );
+ return run( runner );
+ }
+
+ static public boolean run( final CommandRunner runner ) throws IOException {
+ boolean ok = false;
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ final Future<Boolean> future = executor.submit( runner );
+ ok = future.get();
+ } catch ( InterruptedException | ExecutionException multE ) {
+ throw new IOException( multE );
+ }
+ return ok;
+ }
+
+ static public class CommandRunner implements Callable<Boolean> {
+
+ private final String _command;
+ private String _dir;
+ private String _outLog;
+ private String _errLog;
+ private Logger _logger;
+ private boolean _wait;
+
+ public CommandRunner( final String command ) {
+ _command = command;
+ }
+
+ public void setDirectory( final String directory ) {
+ _dir = directory;
+ }
+
+ public void setLogger( final Logger logger ) {
+ _logger = logger;
+ }
+
+ public void setLogFiles( final String outLog,
+ final String errLog ) {
+ _outLog = outLog;
+ _errLog = errLog;
+ }
+
+ public void wait( final boolean wait ) {
+ _wait = wait;
+ }
+
+ private String getDefaultLogFile() {
+ final Random randomizer = new Random();
+ final int spaceIndex = _command.indexOf( ' ' );
+ if ( spaceIndex < 0 ) {
+ return _command + ".ctakes.log." + randomizer.nextLong();
+ }
+ return _command.substring( 0, spaceIndex ) + ".ctakes.log." + randomizer.nextLong();
+ }
+
+ static private void ensureEnvironment( final ProcessBuilder processBuilder ) {
+ final Map<String, String> env = processBuilder.environment();
+ // If the user set a variable in a piper file using "env" then add that to the environment.
+ System.getProperties()
+ .stringPropertyNames()
+ .stream()
+ .filter( n -> n.startsWith( ENV_VAR_PREFIX ) )
+ .forEach( n -> env.put( n.substring( ENV_VAR_PREFIX.length() ), System.getProperty( n ) ) );
+ if ( !env.containsKey( "JAVA_HOME" ) ) {
+ env.put( "JAVA_HOME", System.getProperty( "java.home" ) );
+ }
+ if ( !env.containsKey( "CTAKES_HOME" ) ) {
+ String cTakesHome = System.getenv( "CTAKES_HOME" );
+ if ( cTakesHome == null || cTakesHome.isEmpty() ) {
+ cTakesHome = System.getProperty( "user.dir" );
+ }
+ env.put( "CTAKES_HOME", cTakesHome );
+ }
+ if ( !env.containsKey( "CLASSPATH" ) ) {
+ final String classpath = System.getProperty( "java.class.path" );
+ if ( classpath != null && !classpath.isEmpty() ) {
+ env.put( "CLASSPATH", classpath );
+ }
+ }
+ }
+
+ public Boolean call() throws IOException, InterruptedException {
+ String command = _command;
+ if ( _logger == null ) {
+ if ( _outLog != null && !_outLog.isEmpty() ) {
+ command += " > " + _outLog + " 2>&1";
+ } else {
+ command += " > " + getDefaultLogFile() + " 2>&1";
+ }
+ }
+ String cmd = "cmd.exe";
+ String cmdOpt = "/c";
+ final String os = System.getProperty( "os.name" );
+ if ( os.toLowerCase()
+ .contains( "windows" ) ) {
+ command = command.replace( '/', '\\' );
+ } else {
+ cmd = "bash";
+ cmdOpt = "-c";
+// if ( !_wait ) {
+// command += " &";
+// }
+ }
+ final ProcessBuilder processBuilder = new ProcessBuilder( cmd, cmdOpt, command );
+ if ( _dir != null && !_dir.isEmpty() ) {
+ final File dir = new File( _dir );
+ if ( !dir.exists() ) {
+ dir.mkdirs();
+ }
+ processBuilder.directory( dir );
+ }
+ ensureEnvironment( processBuilder );
+ final Process process = processBuilder.start();
+ if ( _logger != null ) {
+ final ExecutorService executors = Executors.newFixedThreadPool( 2 );
+ executors.submit( new OutputLogger( process, _logger ) );
+ executors.submit( new ErrorLogger( process, _logger ) );
+ }
+ if ( _wait ) {
+ return process.waitFor() == 0;
+ }
+ return true;
+ }
+
+ }
+
+
+ static private class OutputLogger implements Runnable {
+
+ final private InputStream _output;
+ final private Logger _logger;
+
+ private OutputLogger( final Process process, final Logger logger ) {
+ _output = process.getInputStream();
+ _logger = logger;
+ }
+
+ public void run() {
+ try ( BufferedReader reader = new BufferedReader( new InputStreamReader( _output ) ) ) {
+ reader.lines()
+ .forEach( _logger::info );
+ } catch ( IOException ioE ) {
+ _logger.error( ioE.getMessage() );
+ }
+ }
+
+ }
+
+ static private class ErrorLogger implements Runnable {
+
+ final private InputStream _error;
+ final private Logger _logger;
+
+ private ErrorLogger( final Process process, final Logger logger ) {
+ _error = process.getErrorStream();
+ _logger = logger;
+ }
+
+ public void run() {
+ try ( BufferedReader reader = new BufferedReader( new InputStreamReader( _error ) ) ) {
+ reader.lines()
+ .forEach( _logger::error );
+ } catch ( IOException ioE ) {
+ _logger.error( ioE.getMessage() );
+ }
+ }
+
+ }
+
+
+}
Modified: ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/log/FinishedLogger.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/log/FinishedLogger.java?rev=1900795&r1=1900794&r2=1900795&view=diff
==============================================================================
--- ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/log/FinishedLogger.java (original)
+++ ctakes/trunk/ctakes-core/src/main/java/org/apache/ctakes/core/util/log/FinishedLogger.java Wed May 11 00:27:34 2022
@@ -24,7 +24,8 @@ import java.util.Date;
)
final public class FinishedLogger extends JCasAnnotator_ImplBase {
- static private final Logger LOGGER = Logger.getLogger( "FinishedLogger" );
+// static private final Logger LOGGER = Logger.getLogger( "FinishedLogger" );
+static private final Logger LOGGER = Logger.getLogger( "ProgressDone" );
private long _initMillis;
private long _docCount = 0;
@@ -61,17 +62,6 @@ final public class FinishedLogger extend
LOGGER.info( "Documents Processed: " + _docCount );
final long millisPerNote = (endMillis - _initMillis) / _docCount;
LOGGER.info( String.format( "Average Seconds per Document: %.2f", (millisPerNote / 1000f) ) );
-
- final String FINISHED =
- "\n\n" +
- " ###### ###### ## ## ####### ## ####### ####### #######\n" +
- "## ## ## ## ### ### ## ## ## ## ## ## \n" +
- "## ## ## #### #### ## ## ## ## ## ## \n" +
- "## ## ## ## ### ## ####### ## ##### ## ##### \n" +
- "## ## ## ## ## ## ## ## ## ## \n" +
- "## ## ## ## ## ## ## ## ## ## ## \n" +
- " ###### ###### ## ## ## ####### ####### ## #######\n\n";
- LOGGER.info( FINISHED );
}
static private String getTime( final long millis ) {