You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2015/06/16 15:16:02 UTC
svn commit: r1685824 - in
/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core:
IndexerFactory.java impl/EntityErrorLoggerDaemon.java
impl/FinishedEntityDaemon.java impl/IndexerImpl.java
Author: rwesten
Date: Tue Jun 16 13:16:02 2015
New Revision: 1685824
URL: http://svn.apache.org/r1685824
Log:
minor: The Entityhub Indexing Tool now prints the name of the index when logging status updates. Also the created threads include the name in their names (related to STANBOL-1232)
Modified:
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityErrorLoggerDaemon.java
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java
Modified: stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java?rev=1685824&r1=1685823&r2=1685824&view=diff
==============================================================================
--- stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java (original)
+++ stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java Tue Jun 16 13:16:02 2015
@@ -99,6 +99,7 @@ public final class IndexerFactory {
config= new IndexingConfig(dir);
}
//get the mode based on the configured IndexingComponents
+ String name = config.getName();
EntityDataIterable dataIterable = config.getDataIterable();
EntityIterator idIterator = config.getEntityIdIterator();
EntityDataProvider dataProvider = config.getEntityDataProvider();
@@ -137,12 +138,12 @@ public final class IndexerFactory {
}
if(dataIterable != null && scoreProvider != null){
// iterate over data and lookup scores
- indexer = new IndexerImpl(dataIterable, scoreProvider,
+ indexer = new IndexerImpl(name, dataIterable, scoreProvider,
config.getNormaliser(),destination, processors,
config.getIndexedEntitiesIdsFile(),postProcessors);
} else if(idIterator != null && dataProvider != null){
// iterate over id and lookup data
- indexer = new IndexerImpl(idIterator,dataProvider,
+ indexer = new IndexerImpl(name, idIterator,dataProvider,
config.getNormaliser(),destination, processors,
config.getIndexedEntitiesIdsFile(),postProcessors);
} else if(dataIterable != null && idIterator != null){
@@ -152,7 +153,7 @@ public final class IndexerFactory {
"required EntityScoreProvider as needed together with the " +
"configured EntityDataIterable '{}'",
idIterator.getClass(), dataIterable.getClass());
- indexer = new IndexerImpl(dataIterable,
+ indexer = new IndexerImpl(config.getName(), dataIterable,
new EntityIneratorToScoreProviderAdapter(idIterator),
config.getNormaliser(),destination, processors,
config.getIndexedEntitiesIdsFile(),postProcessors);
@@ -167,45 +168,45 @@ public final class IndexerFactory {
return indexer;
}
- public Indexer create(EntityIterator idIterator, EntityDataProvider dataProvider,
+ public Indexer create(String name, EntityIterator idIterator, EntityDataProvider dataProvider,
ScoreNormaliser normaliser,
List<EntityProcessor> processors, IndexingDestination destination){
- return new IndexerImpl(idIterator, dataProvider, normaliser,destination, processors,null,null);
+ return new IndexerImpl(name, idIterator, dataProvider, normaliser,destination, processors,null,null);
}
- public Indexer create(EntityIterator idIterator, EntityDataProvider dataProvider,
+ public Indexer create(String name, EntityIterator idIterator, EntityDataProvider dataProvider,
ScoreNormaliser normaliser,
List<EntityProcessor> processors, List<EntityProcessor> postProcessors,
IndexingDestination destination){
File tmp;
try {
- tmp = File.createTempFile("ind-ent-ids","zip");
+ tmp = File.createTempFile("ind-ent-ids",".zip");
tmp.deleteOnExit();
} catch (IOException e) {
throw new IllegalStateException("Unable to create temporary file for storing the" +
"indexed Entity IDs",e);
}
- return new IndexerImpl(idIterator, dataProvider, normaliser,destination, processors,
+ return new IndexerImpl(name, idIterator, dataProvider, normaliser,destination, processors,
tmp,postProcessors);
}
- public Indexer create(EntityDataIterable dataIterable,EntityScoreProvider scoreProvider,
+ public Indexer create(String name, EntityDataIterable dataIterable,EntityScoreProvider scoreProvider,
ScoreNormaliser normaliser,
List<EntityProcessor> processors, IndexingDestination destination){
- return new IndexerImpl(dataIterable, scoreProvider, normaliser,destination, processors,null,null);
+ return new IndexerImpl(name, dataIterable, scoreProvider, normaliser,destination, processors,null,null);
}
- public Indexer create(EntityDataIterable dataIterable,EntityScoreProvider scoreProvider,
+ public Indexer create(String name, EntityDataIterable dataIterable,EntityScoreProvider scoreProvider,
ScoreNormaliser normaliser,
List<EntityProcessor> processors, List<EntityProcessor> postProcessors,
IndexingDestination destination){
File tmp;
try {
- tmp = File.createTempFile("ind-ent-ids","zip");
+ tmp = File.createTempFile("ind-ent-ids",".zip");
tmp.deleteOnExit();
} catch (IOException e) {
throw new IllegalStateException("Unable to create temporary file for storing the" +
"indexed Entity IDs",e);
}
- return new IndexerImpl(dataIterable, scoreProvider, normaliser,destination, processors,
+ return new IndexerImpl(name, dataIterable, scoreProvider, normaliser,destination, processors,
tmp,postProcessors);
}
}
Modified: stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityErrorLoggerDaemon.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityErrorLoggerDaemon.java?rev=1685824&r1=1685823&r2=1685824&view=diff
==============================================================================
--- stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityErrorLoggerDaemon.java (original)
+++ stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityErrorLoggerDaemon.java Tue Jun 16 13:16:02 2015
@@ -23,10 +23,9 @@ import org.slf4j.Logger;
public class EntityErrorLoggerDaemon extends IndexingDaemon<IndexingError,Object> {
private final Logger err;
- public EntityErrorLoggerDaemon(BlockingQueue<QueueItem<IndexingError>> consume,
- Logger err) {
- super("Indexer: Entity Error Logging Daemon",
- IndexerConstants.SEQUENCE_NUMBER_ERROR_HANDLING_DAEMON,
+ public EntityErrorLoggerDaemon(String name,
+ BlockingQueue<QueueItem<IndexingError>> consume, Logger err) {
+ super(name, IndexerConstants.SEQUENCE_NUMBER_ERROR_HANDLING_DAEMON,
consume, null, null);
this.err = err;
}
Modified: stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java?rev=1685824&r1=1685823&r2=1685824&view=diff
==============================================================================
--- stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java (original)
+++ stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java Tue Jun 16 13:16:02 2015
@@ -72,12 +72,9 @@ public class FinishedEntityDaemon extend
private static final Charset UTF8 = Charset.forName("UTF-8");
- public FinishedEntityDaemon(BlockingQueue<QueueItem<Representation>> consume,
- int majorInterval,
- Logger out,
- OutputStream idOut) {
- super("Indexing: Finished Entity Logger Deamon",
- IndexerConstants.SEQUENCE_NUMBER_FINISHED_DAEMON,
+ public FinishedEntityDaemon(String name, BlockingQueue<QueueItem<Representation>> consume,
+ int majorInterval, Logger out, OutputStream idOut) {
+ super(name, IndexerConstants.SEQUENCE_NUMBER_FINISHED_DAEMON,
consume, null, null);
this.out = out;
if(majorInterval > 0){
Modified: stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java?rev=1685824&r1=1685823&r2=1685824&view=diff
==============================================================================
--- stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java (original)
+++ stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java Tue Jun 16 13:16:02 2015
@@ -129,15 +129,20 @@ public class IndexerImpl implements Inde
private List<EntityProcessor> entityPostProcessors;
private OutputStream indexedEntityIdOutputStream;
+ /**
+ * The name of the index this indexer creates (used for logging)
+ */
+ private String name;
- public IndexerImpl(EntityIterator entityIterator,
+ public IndexerImpl(String name,
+ EntityIterator entityIterator,
EntityDataProvider dataProvider,
ScoreNormaliser normaliser,
IndexingDestination indexingDestination,
List<EntityProcessor> entityProcessors,
File indexedEntityIdFile,
List<EntityProcessor> entityPostProcessors){
- this(normaliser,indexingDestination,entityProcessors,indexedEntityIdFile,entityPostProcessors);
+ this(name, normaliser,indexingDestination,entityProcessors,indexedEntityIdFile,entityPostProcessors);
//set entityMode interfaces
if(entityIterator == null){
throw new IllegalArgumentException("The EntityIterator MUST NOT be NULL!");
@@ -148,14 +153,15 @@ public class IndexerImpl implements Inde
this.indexingComponents.add(entityIterator);
this.indexingComponents.add(dataProvider);
}
- public IndexerImpl(EntityDataIterable dataIterable,
+ public IndexerImpl(String name,
+ EntityDataIterable dataIterable,
EntityScoreProvider scoreProvider,
ScoreNormaliser normaliser,
IndexingDestination indexingDestination,
List<EntityProcessor> entityProcessors,
File indexedEntityIdFile,
List<EntityProcessor> entityPostProcessors){
- this(normaliser,indexingDestination,entityProcessors,indexedEntityIdFile,entityPostProcessors);
+ this(name, normaliser,indexingDestination,entityProcessors,indexedEntityIdFile,entityPostProcessors);
//deactivate entityMode interfaces
this.entityIterator = null;
if(scoreProvider == null){
@@ -168,11 +174,13 @@ public class IndexerImpl implements Inde
this.indexingComponents.add(dataIterable);
}
- protected IndexerImpl(ScoreNormaliser normaliser,
+ protected IndexerImpl(String name,
+ ScoreNormaliser normaliser,
IndexingDestination indexingDestination,
List<EntityProcessor> entityProcessors,
File indexedEntityIdFile,
List<EntityProcessor> entityPostProcessors){
+ this.name = name;
if(indexingDestination == null){
throw new IllegalArgumentException("The Yard MUST NOT be NULL!");
}
@@ -269,7 +277,7 @@ public class IndexerImpl implements Inde
return;
}
setState(State.INITIALISING);
- log.info("Initialisation started ...");
+ log.info("{}: initialisation started ...", name);
}
//add all IndexingSources that need to be initialised to a set
final Collection<IndexingComponent> toInitialise = new HashSet<IndexingComponent>();
@@ -331,7 +339,7 @@ public class IndexerImpl implements Inde
}
}
- log.info("Initialisation completed");
+ log.info("initialisation completed for {}", name);
setState(State.INITIALISED);
}
/* (non-Javadoc)
@@ -349,9 +357,9 @@ public class IndexerImpl implements Inde
"Calling this Method is not supported while in State %s! Supported States are ",
state,supportedStates));
}
- log.info("start initialisation ...");
+ log.info("{}: start initialisation ...", name);
initialiseIndexing();
- log.info(" ... initialisation completed");
+ log.info(" ... initialisation completed for {}", name);
//if now the state is an unsupported one it indicates that
//initialiseIndexingSources() was called by an other thread before this one!
state = getState();
@@ -360,15 +368,15 @@ public class IndexerImpl implements Inde
"Calling this Method is not supported while in State %s! Supported States are ",
state,supportedStates));
}
- log.info("start indexing ...");
+ log.info("{}: start indexing ...", name);
indexEntities();
- log.info(" ... indexing completed");
- log.info("start post-processing ...");
+ log.info(" ... indexing completed for {}", name);
+ log.info("{}: start post-processing ...", name);
postProcessEntities();
- log.info(" ... post-processing finished ...");
- log.info("start finalisation....");
+ log.info(" ... post-processing finished for {}",name);
+ log.info("{}: start finalisation....", name);
finaliseIndexing();
- log.info(" ...finalisation completed");
+ log.info(" ... finalisation completed for {}", name);
}
@Override
public void skipPostProcessEntities() {
@@ -400,7 +408,7 @@ public class IndexerImpl implements Inde
return; // ignore this call
}
setState(State.POSTPROCESSING);
- log.info("PostProcessing started ...");
+ log.info("{}: PostProcessing started ...", name);
}
if(entityPostProcessors == null || entityPostProcessors.isEmpty()){
setState(State.POSTPROCESSED);
@@ -450,7 +458,7 @@ public class IndexerImpl implements Inde
//TODO: Here we would need to create multiple instances in case
// one would e.g. like to use several threads for processing entities
//(1) the daemon reading from the IndexingSources
- String entitySourceReaderName = "Post-processing: Entity Reader Deamon";
+ String entitySourceReaderName = name + ": post-processing: Entity Reader Deamon";
activeIndexingDeamons.add(
new EntityIdBasedIndexingDaemon(
entitySourceReaderName,
@@ -462,7 +470,7 @@ public class IndexerImpl implements Inde
//(2) The daemon for post-processing the entities
activeIndexingDeamons.add(
new EntityProcessorRunnable(
- "Post-processing: Entity Processor Deamon",
+ name +": post-processing: Entity Processor Deamon",
indexedEntityQueue, //it consumes indexed Entities
processedEntityQueue, //it produces processed Entities
errorEntityQueue,
@@ -473,7 +481,7 @@ public class IndexerImpl implements Inde
//(3) The daemon for persisting the entities
activeIndexingDeamons.add(
new EntityPersisterRunnable(
- "Indexing: Entity Perstisting Deamon",
+ name + ": Entity Perstisting Deamon",
processedEntityQueue, //it consumes processed Entities
finishedEntityQueue, //it produces finished Entities
errorEntityQueue,
@@ -481,12 +489,14 @@ public class IndexerImpl implements Inde
//(4) The daemon for logging finished entities
activeIndexingDeamons.add(
new FinishedEntityDaemon(
+ name + ": Finished Entity Logger Deamon",
finishedEntityQueue, -1, log,
null)); //we have already all entity ids!
//(5) The daemon for logging errors
activeIndexingDeamons.add(
new EntityErrorLoggerDaemon(
- errorEntityQueue, log));
+ name +": Entity Error Logging Daemon",
+ errorEntityQueue, log));
//start post-processing and wait until it has finished
startAndWait(activeIndexingDeamons);
//close all post processors
@@ -551,7 +561,7 @@ public class IndexerImpl implements Inde
return; // ignore this call
}
setState(State.FINALISING);
- log.info("finalisation started ...");
+ log.info("{}: finalisation started ...",name);
}
indexingDestination.finalise();
//close the source and the destination
@@ -600,7 +610,7 @@ public class IndexerImpl implements Inde
return; // ignore this call
}
setState(State.INDEXING);
- log.info("Indexing started ...");
+ log.info("{}: indexing started ...",name);
}
//init the queues
int queueSize = Math.max(MIN_QUEUE_SIZE, chunkSize*2);
@@ -620,7 +630,7 @@ public class IndexerImpl implements Inde
//TODO: Here we would need to create multiple instances in case
// one would e.g. like to use several threads for processing entities
//(1) the daemon reading from the IndexingSources
- String entitySourceReaderName = "Indexing: Entity Source Reader Deamon";
+ String entitySourceReaderName = name +": Entity Source Reader Deamon";
if(entityIterator != null){
activeIndexingDeamons.add(
new EntityIdBasedIndexingDaemon(
@@ -643,7 +653,7 @@ public class IndexerImpl implements Inde
//(2) The daemon for processing the entities
activeIndexingDeamons.add(
new EntityProcessorRunnable(
- "Indexing: Entity Processor Deamon",
+ name +": Entity Processor Deamon",
indexedEntityQueue, //it consumes indexed Entities
processedEntityQueue, //it produces processed Entities
errorEntityQueue,
@@ -652,7 +662,7 @@ public class IndexerImpl implements Inde
//(3) The daemon for persisting the entities
activeIndexingDeamons.add(
new EntityPersisterRunnable(
- "Indexing: Entity Perstisting Deamon",
+ name + ": Entity Perstisting Deamon",
processedEntityQueue, //it consumes processed Entities
finishedEntityQueue, //it produces finished Entities
errorEntityQueue,
@@ -660,11 +670,13 @@ public class IndexerImpl implements Inde
//(4) The daemon for logging finished entities
activeIndexingDeamons.add(
new FinishedEntityDaemon(
+ name + ": Finished Entity Logger Deamon",
finishedEntityQueue, -1, log, indexedEntityIdOutputStream));
//(5) The daemon for logging errors
activeIndexingDeamons.add(
new EntityErrorLoggerDaemon(
- errorEntityQueue, log));
+ name +": Entity Error Logging Daemon",
+ errorEntityQueue, log));
//start indexing and wait until it has finished
startAndWait(activeIndexingDeamons);
//close the stream with IDs