You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2011/04/29 11:20:38 UTC
svn commit: r1097740 [1/10] - in /incubator/stanbol/trunk: entityhub/
entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/
entityhub/generic/servicesapi/src/main/java/org/apache/stanbol/entityhub/servicesapi/mapping/
entityhu...
Author: rwesten
Date: Fri Apr 29 09:20:31 2011
New Revision: 1097740
URL: http://svn.apache.org/viewvc?rev=1097740&view=rev
Log:
STANBOL-187: First commit of the new Indexing infrastructure
This includes the core infrastructure and implementations for
- Indexing RDF data (based on jena TDB)
- Storing indexed Entities to the SolrYard
Missing:
- The old indexing utils are not yet moved to this infrastructure
- Readme and documentations of the configuration
I will write a comment for STANBOL-187 providing more details
In addition:
- added some utilities to the FieldMappingUtils
- FieldMappings now support comments
- The SolrDirectoryManager now checks if the solr.xml is present and not only if the configured directory exist
Added:
incubator/stanbol/trunk/entityhub/indexing/core/
incubator/stanbol/trunk/entityhub/indexing/core/pom.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterable.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterator.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataProvider.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityIterator.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityProcessor.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityScoreProvider.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/Indexer.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingComponent.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConstants.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/event/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/event/IndexingEvent.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/event/IndexingListener.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/AbstractEntityIndexingDaemon.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityDataBasedIndexingDaemon.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityErrorLoggerDaemon.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityIdBasedIndexingDaemon.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityPersisterRunnable.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityProcessorRunnable.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerConstants.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingDaemon.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingError.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingSourceInitialiser.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/QueueItem.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/normaliser/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/normaliser/DefaultNormaliser.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/normaliser/MinScoreNormalizer.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/normaliser/NaturalLogNormaliser.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/normaliser/RangeNormaliser.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/normaliser/ScoreNormaliser.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/EmptyProcessor.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/FiledMapperProcessor.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/EntityFieldScoreProvider.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/EntityIneratorToScoreProviderAdapter.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/LineBasedEntityIterator.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/MapEntityScoreProvider.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/ResourceImporter.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/ResourceLoader.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/ResourceState.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/ConfigTest.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/DummyEntityDataSource.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/DummyEntityIdSource.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/DummyEntityScoreSource.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/DummyIndexingDestination.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/IndexerTest.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/ResourceLoaderTest.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/indexing/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/indexing/config/indexerTestMappings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/indexing/config/minIncomming.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/indexing/config/range.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/dataIterating/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/indexing/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/indexing/config/indexerTestMappings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/indexing/config/minIncomming.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/indexing/config/range.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/indexerTests/idIterating/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/singleFileTest.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/archiveInFolder.zip (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/archiveWithError.zip (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/archiveWithIgnore.zip (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/errorFileInFolder.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/fileInFolder.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/ignoreFileInFolder.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/resourceLoaderTest/testFolder/otherFileInFolder.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/missingconfig/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/missingconfig/indexing/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/missingconfigfolder/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/missingconfigfolder/indexing/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/missingconfigfolder/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbPediaMappings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/minIncomming.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/range.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/pom.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestination.java (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/stanbol/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/stanbol/entityhub/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/stanbol/entityhub/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/stanbol/entityhub/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/stanbol/entityhub/indexing/destination/solryard/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestinationTest.java (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/config/fieldBoosts.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/dist/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/invalidBoostConfig/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/dist/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingBoostConfig/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/dist/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingDefaultSolrConf/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/dist/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/missingSolrConf/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/dbPediaMappings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/fieldBoosts.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/minIncomming.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/range.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/dist/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/dbPediaMappings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/fieldBoosts.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/minIncomming.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/range.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/admin-extra.html (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/elevate.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/mapping-ISOLatin1Accent.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/protwords.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/schema.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/scripts.conf
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/solrconfig.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/spellings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/stopwords.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/synonyms.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/xslt/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/xslt/example.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/xslt/example_atom.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/xslt/example_rss.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/simple/conf/xslt/luke.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/destination/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/dist/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/parent/
incubator/stanbol/trunk/entityhub/indexing/parent/pom.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/pom.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/source/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/pom.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/entityhub/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/entityhub/indexing/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/entityhub/indexing/source/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/entityhub/indexing/source/jenatdb/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/entityhub/indexing/source/jenatdb/RdfIndexingSource.java (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/entityhub/indexing/source/jenatdb/RdfResourceImporter.java (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/stanbol/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/stanbol/entityhub/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/stanbol/entityhub/indexing/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/stanbol/entityhub/indexing/source/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/stanbol/entityhub/indexing/source/jenatdb/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/stanbol/entityhub/indexing/source/jenatdb/RdfIndexingSourceTest.java (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testData/Provence-Alpes-Côte_d'Azur.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testData/Rhône-Alpes.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testData/france.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testEntityIds.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testData/Provence-Alpes-Côte_d'Azur.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testData/Rhône-Alpes.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testData/france.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testEntityScore.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/config/indexFieldConfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testData/Provence-Alpes-Côte_d'Azur.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testData/Rhône-Alpes.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testData/france.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testEntityIds.txt (with props)
Modified:
incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java
incubator/stanbol/trunk/entityhub/generic/servicesapi/src/main/java/org/apache/stanbol/entityhub/servicesapi/mapping/FieldMapping.java
incubator/stanbol/trunk/entityhub/indexing/dblp/pom.xml
incubator/stanbol/trunk/entityhub/indexing/dbpedia/pom.xml
incubator/stanbol/trunk/entityhub/indexing/genericrdf/pom.xml
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/java/org/apache/stanbol/entityhub/indexing/rdf/RdfIndexer.java
incubator/stanbol/trunk/entityhub/indexing/geonames/pom.xml
incubator/stanbol/trunk/entityhub/pom.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ConfigUtils.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/DefaultSolrDirectoryManager.java
incubator/stanbol/trunk/parent/pom.xml
Modified: incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java?rev=1097740&r1=1097739&r2=1097740&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java (original)
+++ incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java Fri Apr 29 09:20:31 2011
@@ -128,7 +128,13 @@ public final class FieldMappingUtils {
* String can not be parsed.
*/
public static FieldMapping parseFieldMapping(String mapping){
- if(mapping == null || mapping.isEmpty()){
+ if(mapping == null){
+ return null;
+ }
+ if(mapping.isEmpty()){
+ return null;
+ }
+ if(mapping.charAt(0) == '#'){ //commend
return null;
}
final boolean ignore = mapping.charAt(0) == '!';
@@ -156,11 +162,69 @@ public final class FieldMappingUtils {
}
}
if(ignore && filter != null){
- log.warn(String.format("Filters are not supported for '!<fieldPatter>' type field mappings! Filter %s ignored",filter));
+ log.warn("Filters are not supported for '!<fieldPatter>' type field mappings! Filter {} ignored",filter);
filter = null;
}
return new FieldMapping(fieldPattern, filter, mappedTo.toArray(new String[mappedTo.size()]));
}
+ /**
+ * Parses FieldMappings from the parsed strings
+ * @param mappings the mappings to parse
+ * @return the parsed mappings
+ */
+ public static List<FieldMapping> parseFieldMappings(Iterator<String> mappings) {
+ List<FieldMapping> fieldMappings = new ArrayList<FieldMapping>();
+ log.debug("Parse FieldMappings");
+ while(mappings.hasNext()){
+ String mappingString = mappings.next();
+ log.debug(mappingString);
+ if(mappingString != null &&
+ !mappingString.isEmpty() && //not an empty line
+ !(mappingString.charAt(0) == FieldMapping.COMMENT_CHAR)){ //not an comment
+ FieldMapping fieldMapping = FieldMappingUtils.parseFieldMapping(mappingString.toString());
+ if(fieldMapping != null){
+ fieldMappings.add(fieldMapping);
+ } else {
+ log.warn("Unable to parse FieldMapping for '{}'",mappingString);
+ }
+ }
+ }
+ return fieldMappings;
+ }
+ /**
+ * Creates an FieldMapper instance by using the {@link DefaultFieldMapperImpl}
+ * and the default instance if the {@link ValueConverterFactory} and configure
+ * it with {@link FieldMapping}s parsed form the list of string parsed as
+ * argument to this method.
+ * @param mappings The mappings or <code>null</code> if none
+ * @return A new and configured FieldMapper instance.
+ */
+ public static FieldMapper createDefaultFieldMapper(Iterator<String> mappings){
+ FieldMapper mapper = new DefaultFieldMapperImpl(ValueConverterFactory.getDefaultInstance());
+ if(mappings != null){
+ for(FieldMapping mapping : parseFieldMappings(mappings)){
+ mapper.addMapping(mapping);
+ }
+ }
+ return mapper;
+ }
+ /**
+ * Creates an FieldMapper instance by using the {@link DefaultFieldMapperImpl}
+ * and the default instance if the {@link ValueConverterFactory} and configure
+ * it with {@link FieldMapping}s
+ * @param mappings The mappings or <code>null</code> if none
+ * @return A new and configured FieldMapper instance.
+ */
+ public static FieldMapper createDefaultFieldMapper(Iterable<FieldMapping> mappings){
+ FieldMapper mapper = new DefaultFieldMapperImpl(ValueConverterFactory.getDefaultInstance());
+ if(mappings != null){
+ for(FieldMapping mapping : mappings){
+ mapper.addMapping(mapping);
+ }
+ }
+ return mapper;
+ }
+
//moved to NamespaceEnum
// private static String getFullUri(String value){
// int index = value.indexOf(':');
Modified: incubator/stanbol/trunk/entityhub/generic/servicesapi/src/main/java/org/apache/stanbol/entityhub/servicesapi/mapping/FieldMapping.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/generic/servicesapi/src/main/java/org/apache/stanbol/entityhub/servicesapi/mapping/FieldMapping.java?rev=1097740&r1=1097739&r2=1097740&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/generic/servicesapi/src/main/java/org/apache/stanbol/entityhub/servicesapi/mapping/FieldMapping.java (original)
+++ incubator/stanbol/trunk/entityhub/generic/servicesapi/src/main/java/org/apache/stanbol/entityhub/servicesapi/mapping/FieldMapping.java Fri Apr 29 09:20:31 2011
@@ -44,6 +44,10 @@ import org.apache.stanbol.entityhub.serv
*/
public class FieldMapping {
+ /**
+ * The '#' char is used for comments
+ */
+ public static final char COMMENT_CHAR = '#';
private final String pattern;
private final Pattern regex;
private final boolean usesWildcard;
Added: incubator/stanbol/trunk/entityhub/indexing/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/pom.xml?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/pom.xml (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/pom.xml Fri Apr 29 09:20:31 2011
@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.indexing.parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>../parent</relativePath>
+ </parent>
+
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.indexing.core</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Stanbol Entityhub Indexing Core</name>
+ <description>Defines the indexing api and default implementations
+ </description>
+ <scm>
+ <connection>
+ scm:svn:http://svn.apache.org/repos/asf/incubator/stanbol/trunk/entityhub/indexing/core
+ </connection>
+ <developerConnection>
+ scm:svn:https://svn.apache.org/repos/asf/incubator/stanbol/trunk/entityhub/indexing/core
+ </developerConnection>
+ <url>http://incubator.apache.org/stanbol</url>
+ </scm>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Export-Package>
+ org.apache.stanbol.entityhub.indexing.core;version=${pom.version},
+ org.apache.stanbol.entityhub.indexing.core.normaliser;version=${pom.version},
+ org.apache.stanbol.entityhub.indexing.core.processor;version=${pom.version}
+ </Export-Package>
+ <!-- <Private-Package>
+ org.apache.stanbol.entityhub.indexing.core.impl;version=${pom.version}
+ </Private-Package> -->
+ </instructions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.servicesapi</artifactId>
+ <version>${stanbol-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.core</artifactId>
+ <version>${stanbol-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <!-- For Testing -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/pom.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterable.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterable.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterable.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterable.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,15 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+/**
+ * Interface used to create an instance of an {@link EntityDataIterator}
+ * @author Rupert Westenthaler
+ *
+ */
+public interface EntityDataIterable extends IndexingComponent {
+ /**
+ * Returns an iterator over all Representations of the Entities.
+ * @return A new instance of an {@link EntityDataIterator}
+ */
+ EntityDataIterator entityDataIterator();
+
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterable.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterator.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterator.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterator.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterator.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,34 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import java.util.Iterator;
+
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+
+/**
+ * Interface used to iterate over all entities.
+ * By calling {@link #next()} one can iterate over the IDs of the Entities.
+ * The data ({@link Representation}) of the current entity are available by
+ * calling {@link #getRepresentation()}.<p>
+ * This interface is intended for data source that prefer to read entity
+ * information as a stream (e.g. from an tab separated text file) and therefore
+ * can not provide an implementation of the {@link EntityDataProvider} interface.
+ * @see EntityDataProvider
+ * @author Rupert Westenthaler
+ *
+ */
+public interface EntityDataIterator extends Iterator<String>{
+
+ /**
+ * Getter for the Representation of the current active Element. This is the
+ * Representation of the Entity with the ID returned for the previous
+ * call to {@link #next()}. This method does not change the current element
+ * of the iteration.
+ * @return the Representation for the entity with the ID returned by
+ * {@link #next()}
+ */
+ Representation getRepresentation();
+ /**
+ * Close the iteration.
+ */
+ void close();
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataIterator.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataProvider.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataProvider.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataProvider.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataProvider.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,18 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+
+/**
+ * Interface used to get the representation (data) for an entity based on the
+ * id. This Interface is used for indexing in cases, where the list of entities
+ * to index is known in advance and the data source provides the possibility to
+ * retrieve the entity data based on the ID (e.g. a RDF triple store).
+ * @see {@link EntityDataIterator}
+ * @author Rupert Westenthaler
+ *
+ */
+public interface EntityDataProvider extends IndexingComponent {
+
+ Representation getEntityData(String id);
+
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityDataProvider.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityIterator.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityIterator.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityIterator.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityIterator.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,53 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import java.util.Iterator;
+
+/**
+ * Interface used to iterate over Entities and optionally there score.
+ * Typically Entities are by decreasing score values however this is no
+ * requirement by this implementation. Scores < 0 indicate that entities
+ * should not be indexed. If no score is available the score MUST BE
+ * <code>null</code>
+ * @author Rupert Westenthaler
+ *
+ */
+public interface EntityIterator extends IndexingComponent, Iterator<EntityIterator.EntityScore> {
+
+ /**
+ * Struct like class providing access to the Id and optionally the score of
+ * an entity. A score of <code>0</code> indicated that no score is available.
+ * Scores <code>< 0</code> indicate that entities should be ignored
+ * (unless the indexer is configured otherwise)
+ * Allows direct access to the final id and score values
+ * @author Rupert Westenthaler
+ *
+ */
+ public class EntityScore{
+ /**
+ * The ID of the Entity (read only)
+ */
+ public final String id;
+ /**
+ * The score for the Entity. Entities with scores < 0 should be
+ * ignored for indexing. <code>null</code> indicates that no score is
+ * available
+ */
+ public final Float score;
+ public EntityScore(String id,Float score){
+ this.id = id;
+ this.score = score;
+ }
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof EntityScore && id.equals(((EntityScore)obj).id);
+ }
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+ @Override
+ public String toString() {
+ return String.format("EntityScore[id=%s|score=%s]", id,score);
+ }
+ }
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityIterator.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityProcessor.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityProcessor.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityProcessor.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityProcessor.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,17 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import org.apache.stanbol.entityhub.indexing.core.impl.IndexerImpl;
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+import org.apache.stanbol.entityhub.servicesapi.yard.Yard;
+
+public interface EntityProcessor extends IndexingComponent {
+ /**
+ * Processes the source representation based on some processing rules.
+ * This interface is used by the {@link IndexerImpl} to process Entities
+ * retrieved from the Source before they are stored to the {@link Yard}
+ * @param source the source
+ * @return the processed/mapped representation
+ */
+ Representation process(Representation source);
+
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityProcessor.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityScoreProvider.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityScoreProvider.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityScoreProvider.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityScoreProvider.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,47 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+
+/**
+ * Interface used be the {@link RdfIndexer} to check if an entity should be
+ * indexes or not.
+ * @author Rupert Westenthaler
+ *
+ */
+public interface EntityScoreProvider extends IndexingComponent {
+
+ /**
+ * Indicates if the data of the Entity are needed for calculating/providing
+ * the score for Entities. If an implementation returns <code>true</code>
+ * {@link #process(Representation)} will be called. If <code>false</code>
+ * is returned the {@link #process(String)} is called. <p>
+ * Implementors should consider that supporting the String variant will
+ * improve indexing speed because there is no need to get the actual
+ * Representation for Entities that need not to be indexed.
+ * @return if the indexer needs to parse the {@link Representation} for the
+ * entity.
+ */
+ boolean needsData();
+ /**
+ * Method called to evaluate an entity in case {@link #needsData()} returns
+ * <code>false</code>
+ * @param id the ID of the entity
+ * @return The score or <code>null</code> if no score is available for the
+ * parsed Entity. Values <code>< 0</code> indicate that the entity should
+ * not be indexed.
+ * @throws UnsupportedOperationException if <code>{@link #needsData()}</code>
+ * returns <code>true</code>
+ */
+ Float process(String id) throws UnsupportedOperationException;
+ /**
+ * Method called to evaluate an entity in case {@link #needsData()} returns
+ * <code>true</code>
+ * @param entity the {@link Representation} of the entity
+ * @return The score or <code>null</code> if no score is available for the
+ * parsed Entity. Values <code>< 0</code> indicate that the entity should
+ * not be indexed.
+ * @throws UnsupportedOperationException if <code>{@link #needsData()}</code>
+ * returns <code>false</code>
+ */
+ Float process(Representation entity) throws UnsupportedOperationException;
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/EntityScoreProvider.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/Indexer.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/Indexer.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/Indexer.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/Indexer.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,146 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import org.apache.stanbol.entityhub.servicesapi.yard.Yard;
+
+/**
+ * The main indexing interface.
+ * @author Rupert Westenthaler
+ *
+ */
+public interface Indexer {
+ /**
+ * States of the Indexer
+ * @author Rupert Westenthaler
+ *
+ */
+ enum State {
+ /**
+ * After construction of the an instance
+ */
+ UNINITIALISED,
+ /**
+ * Indicates that the initialising of the {@link IndexingSource}s is on
+ * the way.
+ */
+ INITIALISING,
+ /**
+ * All {@link IndexingSource}s are initialised, but the actual indexing
+ * of the entities has not yet started.
+ * This is the last opportunity to call {@link Indexer#setChunkSize(int)}
+ * and {@link Indexer#setIndexAllEntitiesState(boolean)}
+ */
+ INITIALISED,
+ /**
+ * While the indexing of the entities is on the way.
+ */
+ INDEXING,
+ /**
+ * All Entities provided by the {@link IndexingSource}s are processed
+ * and stored to the {@link IndexingTarget}.
+ */
+ INDEXED,
+ /**
+ * While the {@link IndexingTarget} is finalising the indexing process.
+ */
+ FINALISING,
+ /**
+ * Indicates that the indexing has finished and the {@link IndexingTarget}
+ * has completed the finalisation.
+ */
+ FINISHED};
+
+ /**
+ * Setter for the chunk size. parsing values <= 0 results in the
+ * chunk size to be set to {@link #DEFAULT_CHUNK_SIZE}.
+ * @param chunkSize the chunkSize to set
+ * @throws IllegalStateException if {@link #getState()} >
+ * {@link State#INITIALISED}
+ */
+ void setChunkSize(int chunkSize) throws IllegalStateException;
+
+ /**
+ * Getter for the chunk size (the number of entities that are indexed before
+ * they are store to the {@link Yard}.
+ * @return the chunkSize
+ */
+ int getChunkSize();
+
+ /**
+ * Getter for the Yard used to store the indexed entities.
+ * @return the yard
+ */
+ Yard getYard();
+ /**
+ * Brings this indexer in the {@link State#INITIALISED} by initialising all
+ * {@link IndexingComponent}s.This method blocks until the whole process is
+ * completed.Calls to this method are ignored if the indexer is not in the
+ * {@link State#UNINITIALISED} state.
+ * <p>
+ * This Method is intended to be used by caller that need more control over
+ * the indexing process as simple to call {@link #index()}.
+ */
+ void initialiseIndexingSources();
+ /**
+ * Brings this indexer in the {@link State#INDEXED} by indexing all Entities
+ * provided by the {@link IndexingComponent}s. This method blocks until the
+ * whole process is completed. Calls to this method are ignored if the
+ * indexer is in a state greater than {@link State#INITIALISED}.
+ * <p>
+ * This Method is intended to be used by caller that need more control over
+ * the indexing process as simple to call {@link #index()}.
+ * @throws IllegalStateException if {@link #getState()} <
+ * {@link State#INITIALISED}
+ */
+ void indexAllEntities() throws IllegalStateException;
+ /**
+ * {@link State#FINISHED Finalises} the indexing process by calling finalise
+ * on the {@link IndexingDestination}. This method blocks until the
+ * whole process is completed. Calls to this method are ignored if the
+ * indexer is in a state greater than {@link State#INDEXED}.
+ * <p>
+ * This Method is intended to be used by caller that need more control over
+ * the indexing process as simple to call {@link #index()}.
+ * @throws IllegalStateException if {@link #getState()} <
+ * {@link State#INDEXED}
+ */
+ void finaliseIndexingTarget() throws IllegalStateException;
+ /**
+ * Initialise the {@link IndexingComponent}s, indexes all entities and
+ * finalises the {@link IndexingDestination}. <p>
+ * Calls to this method do have the same result as subsequent calls to
+ * {@link #initialiseIndexingSources()}, {@link #indexAllEntities()},
+ * {@link #finaliseIndexingTarget()}. This method can also be used if any of
+ * the mentioned three methods was already called to this indexer instance.
+ * <p>
+ * This method blocks until the whole process is completed. Ideal if the
+ * called does not need/want any further control over the indexing process.
+ * <p>
+ * To perform the indexing in the background one need to execute this
+ * Method in an own {@link Thread}.
+ */
+ void index();
+
+ /**
+ * This allows to set the Indexer in an state that it also indexes entities
+ * with an negative score. Typically Entities with a negative score are
+ * considered to be marked as not to be indexed. However setting this state
+ * to true allows to index also such entities.
+ * @param indexAllEntitiesState the indexAllEntitiesState to set
+ * @throws IllegalStateException if {@link #getState()} > {@link State#INITIALISED}
+ */
+ void setIndexAllEntitiesState(boolean indexAllEntitiesState) throws IllegalStateException;
+
+ /**
+ * Getter for the state if entities with a negative score are indexed or
+ * not. The default is to exclude such entities.
+ * @return the indexAllEntitiesState
+ */
+ boolean isIndexAllEntitiesState();
+
+ /**
+ * The current state of the indexing process
+ * @return the state
+ */
+ public abstract State getState();
+
+}
\ No newline at end of file
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/Indexer.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,130 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import org.apache.stanbol.entityhub.indexing.core.config.IndexingConfig;
+import org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants;
+import org.apache.stanbol.entityhub.indexing.core.impl.IndexerImpl;
+import org.apache.stanbol.entityhub.indexing.core.normaliser.ScoreNormaliser;
+import org.apache.stanbol.entityhub.indexing.core.source.EntityIneratorToScoreProviderAdapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Factory used to create {@link Indexer} instances
+ * @author Rupert Westenthaler
+ *
+ */
+public class IndexerFactory {
+ private static final Logger log = LoggerFactory.getLogger(IndexerFactory.class);
+ /**
+ * singleton instance
+ */
+ private static IndexerFactory instance = null;
+
+ /**
+ * Singleton constructor
+ */
+ private IndexerFactory(){
+ //Singleton pattern
+ }
+ /**
+ * Getter for the singleton of this factory
+ * @return the singleton
+ */
+ public static IndexerFactory getInstance(){
+ if(instance == null){
+ instance = new IndexerFactory();
+ }
+ return instance;
+ }
+ /**
+ * Creates an {@link Indexer} instance based on the configuration
+ * relative to the working directory.<p>
+ * The configuration is expected within "{workingdir}/indexing"
+ * @return The configured Indexer or an Exception when the configuration was
+ * not found or is not valid
+ */
+ public Indexer create() {
+ return create(null);
+ }
+ /**
+ * Creates an {@link Indexer} instance based on the configuration. The
+ * configuration is expected within the "/indexing" directory of the parsed
+ * location.<p>
+ * In case a relative path is parsed the current working directory is used
+ * as context. That means that the configuration is expected within
+ * folder "{workingDir}/{parsedPath}/indexing". For absolute paths the
+ * configuration is expected at "{parsedPath}/indexing".
+ * @return The configured Indexer or an Exception when the configuration was
+ * not found or is not valid
+ */
+ public Indexer create(String dir){
+ Indexer indexer;
+ IndexingConfig config = new IndexingConfig(dir);
+ //get the mode based on the configured IndexingComponents
+ EntityDataIterable dataIterable = config.getDataInterable();
+ EntityIterator idIterator = config.getEntityIdIterator();
+ EntityDataProvider dataProvider = config.getEntityDataProvider();
+ EntityScoreProvider scoreProvider = config.getEntityScoreProvider();
+
+
+ IndexingDestination destination = config.getIndexingDestination();
+ if(destination == null){
+ log.error("The indexing configuration does not provide an " +
+ "indexing destination. This needs to be configured by the key " +
+ "'{}' in the indexing.properties within the directory {}",
+ IndexingConstants.KEY_INDEXING_DESTINATION,config.getConfigFolder());
+ throw new IllegalArgumentException("No IndexingDestination present");
+ }
+ EntityProcessor processor = config.getEntityProcessor();
+ if(processor == null){
+ log.error("The indexing configuration does not provide an " +
+ "entity processor. This needs to be configured by the key " +
+ "'{}' in the indexing.properties within the directory {}",
+ IndexingConstants.KEY_ENTITY_PROCESSOR,config.getConfigFolder());
+ }
+ log.info("Present Source Configuration:");
+ log.info(" - EntityDataIterable: {}",dataIterable);
+ log.info(" - EntityIterator: {}",idIterator);
+ log.info(" - EntityDataProvider: {}",dataProvider);
+ log.info(" - EntityScoreProvider: {}",scoreProvider);
+ if(dataIterable != null && scoreProvider != null){
+ // iterate over data and lookup scores
+ indexer = new IndexerImpl(dataIterable, scoreProvider,
+ config.getNormaliser(),destination, processor);
+ } else if(idIterator != null && dataProvider != null){
+ // iterate over id and lookup data
+ indexer = new IndexerImpl(idIterator,dataProvider,
+ config.getNormaliser(),destination, processor);
+ } else if(dataIterable != null && idIterator != null){
+ // create an EntityIterator to EntityScoreProvider adapter
+ log.info(
+ "Create Adapter from the configured EntityIterator '{}' to the " +
+ "required EntityScoreProvider as needed together with the " +
+ "configured EntityDataIterable '{}'",
+ idIterator.getClass(), dataIterable.getClass());
+ indexer = new IndexerImpl(dataIterable,
+ new EntityIneratorToScoreProviderAdapter(idIterator),
+ config.getNormaliser(),destination, processor);
+ } else {
+ log.error("Invalid Indexing Source configuration: ");
+ log.error(" - To iterate over the data and lookup scores one need to " +
+ "configure an EntityDataIterable and an EntityScoreProvider ");
+ log.error(" - To iterate over the Id and and lookup data one need to " +
+ "configure an EntityIterator and an EntityDataProvider");
+ throw new IllegalArgumentException("Invalid Indexing Source configuration");
+ }
+ return indexer;
+ }
+
+ public Indexer create(EntityIterator idIterator, EntityDataProvider dataProvider,
+ ScoreNormaliser normaliser,
+ EntityProcessor processor, IndexingDestination destination){
+ return new IndexerImpl(idIterator, dataProvider, normaliser,destination, processor);
+ }
+
+ public Indexer create(EntityDataIterable dataIterable,EntityScoreProvider scoreProvider,
+ ScoreNormaliser normaliser,
+ EntityProcessor processor, IndexingDestination destination){
+ return new IndexerImpl(dataIterable, scoreProvider, normaliser,destination, processor);
+ }
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexerFactory.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingComponent.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingComponent.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingComponent.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingComponent.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,40 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import java.util.Map;
+
+import org.apache.stanbol.entityhub.indexing.core.impl.IndexerImpl;
+
+/**
+ * Parent Interface defining the indexing work flow methods for all kinds of
+ * data sources used for the Indexer.
+ * @author Rupert Westenthaler
+ *
+ */
+public interface IndexingComponent {
+
+ /**
+ * Setter for the configuration
+ * @param config the configuration
+ */
+ public void setConfiguration(Map<String,Object> config);
+ /**
+ * Used by the {@link IndexerImpl} to check if this source needs to be
+ * {@link #initialise()}.
+ * @return If <code>true</code> is returned the {@link IndexerImpl} will call
+ * {@link #initialise()} during the initialisation phase of the indexing
+ * process.
+ */
+ public boolean needsInitialisation();
+ /**
+ * Initialise the IndexingSource. This should be used to perform
+ * time consuming initialisations.
+ */
+ public void initialise();
+
+ /**
+ * Called by the {@link IndexerImpl} as soon as this source is no longer needed
+ * for the indexing process.
+ */
+ public void close();
+
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingComponent.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java?rev=1097740&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java Fri Apr 29 09:20:31 2011
@@ -0,0 +1,24 @@
+package org.apache.stanbol.entityhub.indexing.core;
+
+import org.apache.stanbol.entityhub.servicesapi.yard.Yard;
+
+
+/**
+ * Interface that defines the target for indexing.
+ * @author Rupert Westenthaler
+ *
+ */
+public interface IndexingDestination extends IndexingComponent {
+
+ /**
+ * Getter for the Yard to store the indexed Entities
+ * @return the yard
+ */
+ Yard getYard();
+
+ /**
+ * Called after the indexing is completed to allow some post processing and
+ * packaging of the stored data.
+ */
+ void finalise();
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java
------------------------------------------------------------------------------
svn:mime-type = text/plain