You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2011/06/02 20:59:07 UTC
svn commit: r1130732 [1/5] - in /incubator/stanbol/trunk:
data/sites/dblp/src/main/resources/org/apache/stanbol/data/site/dblp/
data/sites/dbpedia/src/main/resources/org/apache/stanbol/data/site/dbpedia/
data/sites/geonames/src/main/resources/org/apach...
Author: rwesten
Date: Thu Jun 2 18:59:04 2011
New Revision: 1130732
URL: http://svn.apache.org/viewvc?rev=1130732&view=rev
Log:
Resolves STANBOL-208:
* The SolrYard now uses Solr index archives for its configuration.
* Added a parameter to ReferencedSite that allows to load such a configuration based on it's name.
* Configurations are loaded by using the Stanbol DataFileProvider
* Added a SolrYard specific DataFileProvider that searches configurations via the classpath ("{name}.solrindex.zip" is used as file name)
* This DataFileProvider also works outside an OSGI environment (needed because the Yards are also used outside OSGI e.g. by the Indexing tools)
* The default configuration is reduced to an "solr.xml" file with no configured cores. All cores are initialized on demand. This allows to delete all the (duplicated) index configurations.
* The entityhubYard uses a named configuration (entityhub.solrindex.zip)
* The old functionality (copying directories) is still supported by the ConfigUtils but no longer used within the SolrYard. Currently it is still used by the Indexing Utilities because there it allows users to directly change SolrYard configurations without creating Archives after every change.
* Adapted existing configurations (in "data/site/", default configurations of the launchers)
STANBOL-187:
New Feature:
* The Indexing Utilities now create the Bundle needed to add and configure a ReferencedSite for the indexed data!
* Added a "genericrdf" Indexer configuration. This configuration is intended to be used as base for indexing any kind of RDF data. See the README.md file for details.
Changes:
* Several changes related to STANBOL-208
* remanned most of the files form mixed case to lower case
* removed some configurations that are no longer needed
* added additional RDF data for the Jena TDB unit tests
* deleted Solr index configuration of DBLP. This can now use a named configuration (default)
* deleted an unused Solr index configuration directly under the in the dblp directory
Open:
* Currently the creation of the OSGI configurations and the Bundle is implemented in a Utility class and created within the SolrYard destination. This is sub-optimal because each IndexingDestionation implementation would need to provide the same feature. This should be refactored to an independent component of the indexing process that cares about the creation of such configurations.
Other Changes:
* Corrected a bug in the ReferencedSiteManager not forwarding Requests to ReferencedSites that have not defined and EntityPrefixes (such site should get all requests).
* Improved error handling in the FieldMappings utility. Now a single illegal formatted mapping does no longer cause all mappings to be ignored.
Added:
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/destination/
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/destination/OsgiConfigurationUtil.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbpediamappings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexfieldconfig.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/minscore.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/fieldboosts.properties (props changed)
- copied unchanged from r1130447, incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/fieldBoosts.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/fieldboosts.properties (props changed)
- copied unchanged from r1130447, incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/fieldBoosts.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/iditerator.properties (props changed)
- copied unchanged from r1130447, incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/idIterator.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/scorerange.properties (props changed)
- copied unchanged from r1130447, incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/scoreRange.properties
incubator/stanbol/trunk/entityhub/indexing/genericrdf/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/README.md
incubator/stanbol/trunk/entityhub/indexing/genericrdf/pom.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/assembly/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/assembly/assembly.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/java/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/fieldboosts.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/iditerator.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/indexing.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/mappings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/minscore.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/scorerange.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/log4j.properties (with props)
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/rdfdata/
- copied from r1130447, incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/rdfdata/genre.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/rdfdata/
- copied from r1130447, incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/rdfdata/genre.rdf
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/rdfdata/
- copied from r1130447, incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/rdfdata/genre.rdf
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ClassPathSolrIndexConfigProvider.java (with props)
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/default.solrindex.zip (with props)
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/entityhub.solrindex.zip (with props)
Removed:
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbPediaMappings.txt
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexFieldConfig.txt
incubator/stanbol/trunk/entityhub/indexing/dblp/solrConf/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/admin-extra.html
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/elevate.xml
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/mapping-ISOLatin1Accent.txt
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/protwords.txt
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/schema.xml
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/scripts.conf
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/solrconfig.xml
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/spellings.txt
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/stopwords.txt
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/synonyms.txt
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/xslt/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/fieldBoosts.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/fieldBoosts.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/idIterator.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/scoreRange.properties
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testData/
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/admin-extra.html
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/elevate.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/mapping-ISOLatin1Accent.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/protwords.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/schema.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/scripts.conf
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/solrconfig.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/spellings.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/stopwords.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/synonyms.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/cache/conf/xslt/
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/admin-extra.html
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/elevate.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/mapping-ISOLatin1Accent.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/protwords.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/schema.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/scripts.conf
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/solrconfig.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/spellings.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/stopwords.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/synonyms.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/entityhub/conf/xslt/
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/admin-extra.html
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/elevate.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/mapping-ISOLatin1Accent.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/protwords.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/schema.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/scripts.conf
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/solrconfig.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/spellings.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/stopwords.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/synonyms.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/core/conf/xslt/
Modified:
incubator/stanbol/trunk/data/sites/dblp/src/main/resources/org/apache/stanbol/data/site/dblp/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-dblpIndex.config
incubator/stanbol/trunk/data/sites/dbpedia/src/main/resources/org/apache/stanbol/data/site/dbpedia/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-DBpediaIndex.config
incubator/stanbol/trunk/data/sites/geonames/src/main/resources/org/apache/stanbol/data/site/geonames/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-geonamesIndex.config
incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/DefaultSiteConfiguration.java
incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/ReferenceManagerImpl.java
incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java
incubator/stanbol/trunk/entityhub/indexing/core/pom.xml
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityIdBasedIndexingDaemon.java
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java
incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/ConfigTest.java
incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/test/java/org/apache/stanbol/entityhub/indexing/dbpedia/ConfigTest.java
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestination.java
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestinationTest.java
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/simple/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/test/resources/testConfigs/withSolrConf/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/pom.xml
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/main/java/org/apache/stanbol/entityhub/indexing/source/jenatdb/RdfIndexingSource.java
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/java/org/apache/stanbol/entityhub/indexing/source/jenatdb/RdfIndexingSourceTest.java
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/existing/indexing/resources/testEntityIds.txt
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/iterable/indexing/resources/testEntityScore.txt
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/config/indexing.properties
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/src/test/resources/testConfigs/provider/indexing/resources/testEntityIds.txt
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManager.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ConfigUtils.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/DefaultSolrDirectoryManager.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYard.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYardConfig.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/resources/solr/conf/solr.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/test/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManagerTest.java
incubator/stanbol/trunk/entityhub/yard/solr/src/test/java/org/apache/stanbol/entityhub/yard/solr/SolrYardTest.java
incubator/stanbol/trunk/launchers/full/src/main/resources/resources/config/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-DBpediaCache.config
Modified: incubator/stanbol/trunk/data/sites/dblp/src/main/resources/org/apache/stanbol/data/site/dblp/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-dblpIndex.config
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/data/sites/dblp/src/main/resources/org/apache/stanbol/data/site/dblp/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-dblpIndex.config?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/data/sites/dblp/src/main/resources/org/apache/stanbol/data/site/dblp/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-dblpIndex.config (original)
+++ incubator/stanbol/trunk/data/sites/dblp/src/main/resources/org/apache/stanbol/data/site/dblp/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-dblpIndex.config Thu Jun 2 18:59:04 2011
@@ -4,6 +4,5 @@ org.apache.stanbol.entityhub.yard.descri
org.apache.stanbol.entityhub.yard.solr.maxBooleanClauses=I"1024"
org.apache.stanbol.entityhub.yard.solr.solrUri="dblp"
org.apache.stanbol.entityhub.yard.solr.multiYardIndexLayout="false"
-org.apache.stanbol.entityhub.yard.solr.allowDefaultConfig="false"
org.apache.stanbol.entityhub.yard.id="dblpIndex"
org.apache.stanbol.entityhub.yard.name="DPLP\ Yard"
Modified: incubator/stanbol/trunk/data/sites/dbpedia/src/main/resources/org/apache/stanbol/data/site/dbpedia/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-DBpediaIndex.config
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/data/sites/dbpedia/src/main/resources/org/apache/stanbol/data/site/dbpedia/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-DBpediaIndex.config?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/data/sites/dbpedia/src/main/resources/org/apache/stanbol/data/site/dbpedia/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-DBpediaIndex.config (original)
+++ incubator/stanbol/trunk/data/sites/dbpedia/src/main/resources/org/apache/stanbol/data/site/dbpedia/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-DBpediaIndex.config Thu Jun 2 18:59:04 2011
@@ -4,6 +4,5 @@ org.apache.stanbol.entityhub.yard.descri
org.apache.stanbol.entityhub.yard.solr.maxBooleanClauses=I"1024"
org.apache.stanbol.entityhub.yard.solr.solrUri="dbPedia"
org.apache.stanbol.entityhub.yard.solr.multiYardIndexLayout="false"
-org.apache.stanbol.entityhub.yard.solr.allowDefaultConfig="false"
org.apache.stanbol.entityhub.yard.id="dbPediaIndex"
org.apache.stanbol.entityhub.yard.name="dbPedia\ Cache"
Modified: incubator/stanbol/trunk/data/sites/geonames/src/main/resources/org/apache/stanbol/data/site/geonames/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-geonamesIndex.config
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/data/sites/geonames/src/main/resources/org/apache/stanbol/data/site/geonames/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-geonamesIndex.config?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/data/sites/geonames/src/main/resources/org/apache/stanbol/data/site/geonames/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-geonamesIndex.config (original)
+++ incubator/stanbol/trunk/data/sites/geonames/src/main/resources/org/apache/stanbol/data/site/geonames/org.apache.stanbol.entityhub.yard.solr.impl.SolrYard-geonamesIndex.config Thu Jun 2 18:59:04 2011
@@ -2,7 +2,6 @@ org.apache.stanbol.entityhub.yard.descri
org.apache.stanbol.entityhub.yard.solr.solrUri="geonames"
org.apache.stanbol.entityhub.yard.defaultQueryResultNumber=I"-1"
org.apache.stanbol.entityhub.yard.solr.multiYardIndexLayout="false"
-org.apache.stanbol.entityhub.yard.solr.allowDefaultConfig="false"
org.apache.stanbol.entityhub.yard.id="geonamesIndex"
org.apache.stanbol.entityhub.yard.name="geonames.org\ Cache"
org.apache.stanbol.entityhub.yard.maxQueryResultNumber=I"-1"
Modified: incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/DefaultSiteConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/DefaultSiteConfiguration.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/DefaultSiteConfiguration.java (original)
+++ incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/DefaultSiteConfiguration.java Thu Jun 2 18:59:04 2011
@@ -498,7 +498,7 @@ public class DefaultSiteConfiguration im
return licenseList.isEmpty()?null:licenseList.toArray(new License[licenseList.size()]);
}
public final void setLicenses(License[] licenses){
-
+ //TODO: implement
}
Modified: incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/ReferenceManagerImpl.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/ReferenceManagerImpl.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/ReferenceManagerImpl.java (original)
+++ incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/impl/ReferenceManagerImpl.java Thu Jun 2 18:59:04 2011
@@ -95,6 +95,8 @@ public class ReferenceManagerImpl implem
* both a Map and an sorted List over the keys!
*/
private final List<String> prefixList = new ArrayList<String>();
+ private final Set<ReferencedSite> noPrefixSites = Collections.synchronizedSet(
+ new HashSet<ReferencedSite>());
@Activate
protected void activate(ComponentContext context) {
@@ -106,6 +108,7 @@ public class ReferenceManagerImpl implem
synchronized (prefixMap) {
this.prefixList.clear();
this.prefixMap.clear();
+ this.noPrefixSites.clear();
}
this.idMap.clear();
}
@@ -133,21 +136,28 @@ public class ReferenceManagerImpl implem
* @param referencedSite
*/
private void addEntityPrefixes(ReferencedSite referencedSite) {
- for(String prefix : referencedSite.getConfiguration().getEntityPrefixes()){
+ String[] prefixes = referencedSite.getConfiguration().getEntityPrefixes();
+ if(prefixes == null || prefixes.length < 1){
synchronized (prefixMap) {
- Collection<ReferencedSite> sites = prefixMap.get(prefix);
- if(sites == null){
- sites = new CopyOnWriteArrayList<ReferencedSite>();
- prefixMap.put(prefix, sites);
- //this also means that the prefix is not part of the prefixList
- int pos = Collections.binarySearch(prefixList, prefix);
- if(pos<0){
- prefixList.add(Math.abs(pos)-1,prefix);
+ noPrefixSites.add(referencedSite);
+ }
+ } else {
+ for(String prefix : prefixes){
+ synchronized (prefixMap) {
+ Collection<ReferencedSite> sites = prefixMap.get(prefix);
+ if(sites == null){
+ sites = new CopyOnWriteArrayList<ReferencedSite>();
+ prefixMap.put(prefix, sites);
+ //this also means that the prefix is not part of the prefixList
+ int pos = Collections.binarySearch(prefixList, prefix);
+ if(pos<0){
+ prefixList.add(Math.abs(pos)-1,prefix);
+ }
+ prefixList.add(Collections.binarySearch(prefixList, prefix)+1,prefix);
}
- prefixList.add(Collections.binarySearch(prefixList, prefix)+1,prefix);
+ //TODO: Sort the referencedSites based on the ServiceRanking!
+ sites.add(referencedSite);
}
- //TODO: Sort the referencedSites based on the ServiceRanking!
- sites.add(referencedSite);
}
}
}
@@ -156,16 +166,23 @@ public class ReferenceManagerImpl implem
* @param referencedSite
*/
private void removeEntityPrefixes(ReferencedSite referencedSite) {
- for(String prefix : referencedSite.getConfiguration().getEntityPrefixes()){
+ String[] prefixes = referencedSite.getConfiguration().getEntityPrefixes();
+ if(prefixes == null || prefixes.length < 1){
synchronized (prefixMap) {
- Collection<ReferencedSite> sites = prefixMap.get(prefix);
- if(sites != null){
- sites.remove(referencedSite);
- if(sites.isEmpty()){
- //remove key from the Map
- prefixMap.remove(prefix);
- //remove also the prefix from the List
- prefixList.remove(prefix);
+ noPrefixSites.remove(referencedSite);
+ }
+ } else {
+ for(String prefix : prefixes){
+ synchronized (prefixMap) {
+ Collection<ReferencedSite> sites = prefixMap.get(prefix);
+ if(sites != null){
+ sites.remove(referencedSite);
+ if(sites.isEmpty()){
+ //remove key from the Map
+ prefixMap.remove(prefix);
+ //remove also the prefix from the List
+ prefixList.remove(prefix);
+ }
}
}
}
@@ -212,18 +229,24 @@ public class ReferenceManagerImpl implem
prefixPos = pos; //entityUri found in list
}
if(prefixPos<0){
- return Collections.emptySet();
+ return Collections.unmodifiableCollection(noPrefixSites);
} else {
String prefix = prefixList.get(prefixPos);
if(entityUri.startsWith(prefix)){
log.debug("Found prefix {} for Entity {}",prefix,entityUri);
- return prefixMap.get(prefix);
+ Collection<ReferencedSite> prefixSites = prefixMap.get(prefix);
+ Collection<ReferencedSite> sites =
+ new ArrayList<ReferencedSite>(noPrefixSites.size()+prefixSites.size());
+ sites.addAll(prefixSites);
+ sites.addAll(noPrefixSites);
+ return Collections.unmodifiableCollection(sites);
} //else the parsed entityPrefix does not start with the found prefix
// this may only happen, when the prefixPos == prefixList.size()
}
}
- log.info("No registered prefix for entity {}",entityUri);
- return Collections.emptySet();
+ log.debug("No registered prefix found for entity {} " +
+ "-> return sites that accept all entities",entityUri);
+ return Collections.unmodifiableCollection(noPrefixSites);
}
@Override
public QueryResultList<String> findIds(FieldQuery query) {
Modified: incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java (original)
+++ incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java Thu Jun 2 18:59:04 2011
@@ -165,7 +165,13 @@ public final class FieldMappingUtils {
log.warn("Filters are not supported for '!<fieldPatter>' type field mappings! Filter {} ignored",filter);
filter = null;
}
- return new FieldMapping(fieldPattern, filter, mappedTo.toArray(new String[mappedTo.size()]));
+ try {
+ return new FieldMapping(fieldPattern, filter, mappedTo.toArray(new String[mappedTo.size()]));
+ }catch (RuntimeException e) {
+ log.warn(String.format("Unable to parse FieldMapping from Line '%s'",
+ mapping),e);
+ return null;
+ }
}
/**
* Parses FieldMappings from the parsed strings
Modified: incubator/stanbol/trunk/entityhub/indexing/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/pom.xml?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/pom.xml (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/pom.xml Thu Jun 2 18:59:04 2011
@@ -88,6 +88,17 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+
+ <dependency> <!-- for writing OSGI configuration files -->
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.configadmin</artifactId>
+ <version>1.2.8</version>
+ </dependency>
+ <dependency>
+ <groupId>biz.aQute</groupId> <!-- used to generate bundles -->
+ <artifactId>bndlib</artifactId>
+ <version>1.43.0</version>
+ </dependency>
<!-- For Testing -->
<dependency>
<groupId>junit</groupId>
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/IndexingDestination.java Thu Jun 2 18:59:04 2011
@@ -1,5 +1,7 @@
package org.apache.stanbol.entityhub.indexing.core;
+import java.util.Dictionary;
+
import org.apache.stanbol.entityhub.servicesapi.yard.Yard;
@@ -18,7 +20,8 @@ public interface IndexingDestination ext
/**
* Called after the indexing is completed to allow some post processing and
- * packaging of the stored data.
+ * packaging the stored data and writing of the OSGI configuration used to
+ * initialise the Yard.
*/
void finalise();
}
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java Thu Jun 2 18:59:04 2011
@@ -43,6 +43,7 @@ import org.apache.stanbol.entityhub.inde
import org.apache.stanbol.entityhub.indexing.core.normaliser.ScoreNormaliser;
import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapper;
import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapping;
+import org.apache.stanbol.entityhub.servicesapi.site.SiteConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -360,7 +361,7 @@ public class IndexingConfig {
*/
private File getResource(File root, String fileName) {
File resource = new File(root,fileName);
- log.info("reauest for Resource {} (folder: {})",fileName,root);
+ log.info("request for Resource {} (folder: {})",fileName,root);
if(resource.getAbsoluteFile().exists()){
log.info(" > rquested Resource present");
} else if(copyFromClasspath(resource)){
@@ -961,4 +962,15 @@ public class IndexingConfig {
public Map<String,Object> getConfig(String name,boolean required) throws IllegalArgumentException {
return loadConfig(name, configDir, required);
}
+ /**
+ * Getter for configured properties directly by the key. Typically used
+ * to get Properties as defined by the {@link SiteConfiguration} interface
+ * @param key the key of the property
+ * @return the value or <code>null</code> if not present. Might also return
+ * <code>null</code> in case the value <code>null</code> is set for the
+ * requested property.
+ */
+ public Object getProperty(String key){
+ return configuration.get(key);
+ }
}
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/destination/OsgiConfigurationUtil.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/destination/OsgiConfigurationUtil.java?rev=1130732&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/destination/OsgiConfigurationUtil.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/destination/OsgiConfigurationUtil.java Thu Jun 2 18:59:04 2011
@@ -0,0 +1,259 @@
+package org.apache.stanbol.entityhub.indexing.core.destination;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Dictionary;
+import java.util.Hashtable;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.felix.cm.file.ConfigurationHandler;
+import org.apache.stanbol.entityhub.core.mapping.FieldMappingUtils;
+import org.apache.stanbol.entityhub.indexing.core.config.IndexingConfig;
+import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapper;
+import org.apache.stanbol.entityhub.servicesapi.site.ReferencedSite;
+import org.apache.stanbol.entityhub.servicesapi.site.SiteConfiguration;
+import org.apache.stanbol.entityhub.servicesapi.yard.Cache;
+import org.apache.stanbol.entityhub.servicesapi.yard.CacheStrategy;
+import org.apache.stanbol.entityhub.servicesapi.yard.Yard;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import aQute.bnd.service.BndListener;
+import aQute.lib.osgi.Builder;
+import aQute.lib.osgi.Jar;
+
+public final class OsgiConfigurationUtil {
+
+ private OsgiConfigurationUtil() {/* Do not create instances of Util classes*/}
+
+ private static final Logger log = LoggerFactory.getLogger(OsgiConfigurationUtil.class);
+
+ public static final String DEFAULT_MAPPING_STATE = "proposed";
+ public static final String DEFAULT_IMPORTED_ENTTIY_STATE = "proposed";
+ public static final Object DEFAULT_EXPIRE_DURATION = Integer.valueOf(0);
+
+ public static final String REFERENCED_SITE_COMPONENT_ID = "org.apache.stanbol.entityhub.site.referencedSite";
+ public static final String CACHE_COMPONENT_ID = "org.apache.stanbol.entityhub.core.site.CacheImpl";
+
+ private static final String CONFIG_ROOT = "config";
+ private static final String[] CONFIG_PATH_ELEMENTS = new String[]{"org","apache","stanbol","data","site"};
+ private static final String CONFIG_PATH;
+ private static final String CONFIG_PACKAGE;
+ static {
+ StringBuilder path = new StringBuilder();
+ StringBuilder java = new StringBuilder();
+ for(String part : CONFIG_PATH_ELEMENTS){
+ path.append(part);
+ java.append(part);
+ path.append(File.separatorChar);
+ java.append('.');
+ }
+ CONFIG_PATH = path.toString();
+ CONFIG_PACKAGE = java.toString();
+ }
+
+ public static void writeOsgiConfig(IndexingConfig indexingConfig,String name, Dictionary<String,Object> config) throws IOException{
+ if(indexingConfig == null){
+ throw new IllegalArgumentException("The parsed IndexingConfiguration MUST NOT be NULL");
+ }
+ if(name == null){
+ throw new IllegalArgumentException("The parsed file name MUST NOT be NULL");
+ }
+ if(config == null){
+ throw new IllegalArgumentException("The parsed configuration MUST NOT be NULL");
+ }
+ if(config.isEmpty()){
+ throw new IllegalArgumentException("The parsed configuration MUST NOT be empty");
+ }
+ File configFile = new File(getConfigDirectory(indexingConfig),name);
+ ConfigurationHandler.write(FileUtils.openOutputStream(configFile), config);
+ }
+
+ private static Dictionary<String,Object> createSiteConfig(IndexingConfig indexingConfig){
+ Dictionary<String,Object> config = new Hashtable<String,Object>();
+ //basic properties
+ //we use the name as ID
+ config.put(SiteConfiguration.ID, indexingConfig.getName());
+ //do not set a name, because the ID is used for the name if no name is defined
+ //config.put(SiteConfiguration.NAME, indexingConfig.getName());
+ if(indexingConfig.getDescription() != null && !indexingConfig.getDescription().isEmpty()){
+ config.put(SiteConfiguration.DESCRIPTION, indexingConfig.getDescription());
+ }
+ //the cache
+ //name the Cache is the same as for the Yard.
+ config.put(SiteConfiguration.CACHE_ID, getYardID(indexingConfig));
+ config.put(SiteConfiguration.CACHE_STRATEGY, CacheStrategy.all);
+ //Entity Dereferencer (optional)
+ if(addProperty(SiteConfiguration.ACCESS_URI, config, indexingConfig)){
+ addProperty(SiteConfiguration.ENTITY_DEREFERENCER_TYPE, config, indexingConfig,
+ "Referenced Site for " + indexingConfig.getName() +
+ " (including a full local Cache)");
+ }
+ //Entity Searcher (optional)
+ if(addProperty(SiteConfiguration.QUERY_URI, config, indexingConfig)){
+ addProperty(SiteConfiguration.ENTITY_SEARCHER_TYPE, config, indexingConfig);
+ }
+ //General Properties
+ addProperty(SiteConfiguration.DEFAULT_EXPIRE_DURATION, config, indexingConfig,DEFAULT_EXPIRE_DURATION);
+ addProperty(SiteConfiguration.DEFAULT_MAPPING_STATE, config, indexingConfig,DEFAULT_MAPPING_STATE);
+ addProperty(SiteConfiguration.DEFAULT_SYMBOL_STATE, config, indexingConfig,DEFAULT_IMPORTED_ENTTIY_STATE);
+ //the entity prefix is optional and may be an array
+ addPropertyValues(SiteConfiguration.ENTITY_PREFIX, config, indexingConfig);
+ //add the Field Mappings when entities of this Site are imported to the
+ //entityhub. This may be the same mappings as used for the Cache however
+ //they may be also different.
+ Object value = indexingConfig.getProperty(SiteConfiguration.SITE_FIELD_MAPPINGS);
+ if(value != null){
+ File fieldMappingConfig = indexingConfig.getConfigFile(value.toString());
+ if(fieldMappingConfig != null){
+ try {
+ config.put(SiteConfiguration.SITE_FIELD_MAPPINGS,
+ FileUtils.readLines(fieldMappingConfig, "UTF-8"));
+ } catch (IOException e) {
+ log.warn(String.format("Unable to read Field Mappings for Referenced Site " +
+ "configuration"),e);
+ }
+ } else {
+ log.warn("Unable to load configured Field Mappings for Reference Site " +
+ "{}={}",SiteConfiguration.SITE_FIELD_MAPPINGS,value);
+ }
+ }
+ //set other optional properties
+ addProperty(SiteConfiguration.SITE_ATTRIBUTION, config, indexingConfig);
+ addProperty(SiteConfiguration.SITE_ATTRIBUTION_URL, config, indexingConfig);
+ addPropertyValues(SiteConfiguration.SITE_LICENCE_NAME, config, indexingConfig);
+ addPropertyValues(SiteConfiguration.SITE_LICENCE_TEXT, config, indexingConfig);
+ addPropertyValues(SiteConfiguration.SITE_LICENCE_URL, config, indexingConfig);
+ return config;
+ }
+
+ private static Dictionary<String,Object> createCacheConfig(IndexingConfig indexingConfig){
+ Dictionary<String,Object> config = new Hashtable<String,Object>();
+ //a cache needs to provide the ID of the Yard
+ String yardId = getYardID(indexingConfig);
+ config.put(Cache.ID, yardId);
+ config.put(Cache.NAME, indexingConfig.getName()+" Cache");
+ config.put(Cache.DESCRIPTION, "Cache for the "+indexingConfig.getName()+
+ " Referenced Site using the "+yardId+".");
+ config.put(Cache.CACHE_YARD, getYardID(indexingConfig));
+ //additinal Mappings:
+ // This can be used to define what information are store to the cache
+ // if an Entity is updated from a remote site.
+ // If not present the mappings used by the Yard are used. This default
+ // is sufficient for full indexes as created by the indexing utils
+ // therefore we need not to deal with additional mappings here
+ return config;
+ }
+ /**
+ * Adds the configurations as defined by the Yard Interface. Configurations
+ * of specific Yard implementations might need to add additional
+ * parameters. <p>
+ * This also ensures that the ID of the Yard is the same as referenced by the
+ * configurations for the referenced site and the cache.
+ * @param indexingConfig
+ * @return
+ */
+ public static Dictionary<String,Object> createYardConfig(IndexingConfig indexingConfig){
+ Dictionary<String,Object> config = new Hashtable<String,Object>();
+ config.put(Yard.ID, getYardID(indexingConfig));
+ config.put(Yard.NAME, indexingConfig.getName()+" Index");
+ config.put(Yard.DESCRIPTION,"Full local index for the Referenced Site \""+indexingConfig.getName()+"\".");
+ return config;
+ }
+ public static void writeSiteConfiguration(IndexingConfig indexingConfig) throws IOException {
+ String siteConfigFileName = REFERENCED_SITE_COMPONENT_ID +
+ "-" + indexingConfig.getName()+".config";
+ writeOsgiConfig(indexingConfig,siteConfigFileName, createSiteConfig(indexingConfig));
+ }
+
+ public static void writeCacheConfiguration(IndexingConfig indexingConfig) throws IOException {
+ String cacheFileName = CACHE_COMPONENT_ID +
+ "-" + indexingConfig.getName()+".config";
+ writeOsgiConfig(indexingConfig,cacheFileName, createCacheConfig(indexingConfig));
+ }
+
+ /**
+ * Getter for default ID of the yard based on the value of
+ * {@link IndexingConfig#getName()}
+ * @param config the IndexingConfig
+ * @return the default ID of the yard based on the value of
+ * {@link IndexingConfig#getName()}
+ */
+ public static String getYardID(IndexingConfig config){
+ return config.getName()+"Index";
+ }
+
+ private static boolean addPropertyValues(String key, Dictionary<String,Object> config, IndexingConfig indexingConfig){
+ Object value = indexingConfig.getProperty(key);
+ if(value != null && !value.toString().isEmpty()) {
+ config.put(key, value.toString().split(";"));
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private static boolean addProperty(String key, Dictionary<String,Object> config, IndexingConfig indexingConfig){
+ return addProperty(key, config, indexingConfig, null);
+ }
+ private static boolean addProperty(String key, Dictionary<String,Object> config, IndexingConfig indexingConfig,Object defaultValue){
+ Object value = indexingConfig.getProperty(key);
+ if(value != null || defaultValue != null){
+ config.put(key, value != null ? value : defaultValue);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ /**
+ * Getter for the Directory that need to contain all Files to be included
+ * in the OSGI Bundle.
+ * @param config the indexing configuration
+ * @return the directory (created if not already existing)
+ * @throws IOException If the directory could not be created
+ */
+ public static File getConfigDirectory(IndexingConfig config) throws IOException{
+ File configRoot = new File(config.getDestinationFolder(),CONFIG_ROOT);
+ File siteConfigDir = new File(configRoot,CONFIG_PATH+config.getName().toLowerCase());
+ if(!siteConfigDir.isDirectory()){
+ if(!siteConfigDir.mkdirs()){
+ throw new IOException("Unable to create config Directory "+siteConfigDir);
+ }
+ }
+ return siteConfigDir;
+ }
+
+ public static void createBundle(IndexingConfig config){
+ Builder builder = new Builder();
+ builder.setProperty("Install-Path",CONFIG_PATH+config.getName().toLowerCase());
+ builder.setProperty(Builder.EXPORT_PACKAGE,CONFIG_PACKAGE+config.getName().toLowerCase());
+ builder.setProperty(Builder.BUNDLE_CATEGORY, "Stanbol Data");
+ builder.setProperty(Builder.BUNDLE_NAME, "Apache Stanbol Data: "+config.getName());
+ builder.setProperty(Builder.CREATED_BY, "Apache Stanbol Entityhub Indexing Utils");
+ builder.setProperty(Builder.BUNDLE_VENDOR, "Apache Stanbol (Incubating)");//TODO make configureable
+ builder.setProperty(Builder.BUNDLE_VERSION, "1.0.0");
+ builder.setProperty(Builder.BUNDLE_DESCRIPTION, "Bundle created for import of the referenced site "
+ + config.getName() +" into the Apache Stanbol Entityhub");
+ builder.setProperty(Builder.BUNDLE_SYMBOLICNAME, CONFIG_PACKAGE+config.getName().toLowerCase());
+ try {
+ builder.addClasspath(new File(config.getDestinationFolder(),CONFIG_ROOT));
+ } catch (IOException e) {
+ log.warn("Unable to build OSGI Bundle for Indexed Referenced Site "+config.getName(),e);
+ return;
+ }
+ Jar jar;
+ try {
+ jar = builder.build();
+ } catch (Exception e) {
+ log.warn("Unable to build OSGI Bundle for Indexed Referenced Site "+config.getName(),e);
+ return;
+ }
+ try {
+ jar.write(new File(config.getDistributionFolder(),
+ CONFIG_PACKAGE+config.getName()+"-1.0.0.jar"));
+ } catch (Exception e) {
+ log.warn("Unable to write OSGI Bundle for Indexed Referenced Site "+config.getName(),e);
+ }
+ }
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/destination/OsgiConfigurationUtil.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityIdBasedIndexingDaemon.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityIdBasedIndexingDaemon.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityIdBasedIndexingDaemon.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/EntityIdBasedIndexingDaemon.java Thu Jun 2 18:59:04 2011
@@ -50,7 +50,7 @@ public class EntityIdBasedIndexingDaemon
score.compareTo(ScoreNormaliser.ZERO) >= 0){ //score >= 0
Representation rep = dataProvider.getEntityData(entityScore.id);
if(rep == null){
- log.info("unable to get Data for Entity {} (score=norm:{}|orig:{})",
+ log.debug("unable to get Data for Entity {} (score=norm:{}|orig:{})",
new Object[]{entityScore.id,score,entityScore.score});
}
produce(rep,score,start);
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexerImpl.java Thu Jun 2 18:59:04 2011
@@ -22,6 +22,7 @@ import org.apache.stanbol.entityhub.inde
import org.apache.stanbol.entityhub.indexing.core.Indexer;
import org.apache.stanbol.entityhub.indexing.core.IndexingComponent;
import org.apache.stanbol.entityhub.indexing.core.IndexingDestination;
+import org.apache.stanbol.entityhub.indexing.core.destination.OsgiConfigurationUtil;
import org.apache.stanbol.entityhub.indexing.core.event.IndexingEvent;
import org.apache.stanbol.entityhub.indexing.core.event.IndexingListener;
import org.apache.stanbol.entityhub.indexing.core.impl.IndexingDaemon.IndexingDaemonEventObject;
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/ConfigTest.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/ConfigTest.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/ConfigTest.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/test/java/org/apache/stanbol/entityhub/indexing/core/ConfigTest.java Thu Jun 2 18:59:04 2011
@@ -114,7 +114,7 @@ public class ConfigTest {
assertTrue("Config File for the RangeNormaliser not copied",
new File(config.getConfigFolder(),"range.properties").isFile());
assertTrue("Config File for the MinScoreNormalizer not copied",
- new File(config.getConfigFolder(),"minincoming.properties").isFile());
+ new File(config.getConfigFolder(),"minscore.properties").isFile());
//now test if the configuration was parsed correctly
ScoreNormaliser testNormaliser = normaliser;
assertNotNull(testNormaliser);
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbpediamappings.txt
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbpediamappings.txt?rev=1130732&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbpediamappings.txt (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbpediamappings.txt Thu Jun 2 18:59:04 2011
@@ -0,0 +1,61 @@
+# --- Define the Languages for all fields ---
+| @=null;en;de;fr;it
+
+# --- RDF, RDFS and OWL Mappings ---
+rdfs:label
+rdfs:comment
+rdf:type | d=entityhub:ref
+# used by LOD to link to URIs used to identify the same Entity
+owl:sameAs | d=entityhub:ref
+
+# --- Dublin Core (dc terms and dc elements) ---
+dc:*
+# all DC Elements (one could also define the mappings to the DC Terms counterparts here
+dc-elements:*
+
+# --- Spatial Things ---
+geo:lat | d=xsd:double
+geo:long | d=xsd:double
+geo:alt | d=xsd:int;xsd:float
+# one can also copy the valued from the DBpedia properties
+#dbp-prop:latitude | d=xsd:decimal > geo:lat
+#dbp-prop:longitude | d=xsd:decimal > geo:long
+
+# --- Thesaurus (via SKOS) ---
+#SKOS can be used to define hierarchical terminologies
+skos:*
+skos:broader | d=entityhub:ref
+skos:narrower | d=entityhub:ref
+skos:related | d=entityhub:ref
+skos:member | d=entityhub:ref
+skos:subject | d=entityhub:ref
+skos:inScheme | d=entityhub:ref
+skos:hasTopConcept | d=entityhub:ref
+skos:topConceptOf | d=entityhub:ref
+
+# --- Social Networks (via foaf) ---
+#The Friend of a Friend schema often used to describe social relations between people
+foaf:*
+foaf:knows | d=entityhub:ref
+foaf:made | d=entityhub:ref
+foaf:maker | d=entityhub:ref
+foaf:member | d=entityhub:ref
+foaf:homepage | d=xsd:anyURI
+# also use the DBpedia property website for oaf:homepage!
+dbp-prop:website | d=xsd:anyURI > foaf:homepage
+foaf:depiction | d=xsd:anyURI
+# also use the DBpedia thumbnail as oaf:depiction
+dbp-ont:thumbnail | d=xsd:anyURI > foaf:depiction
+foaf:img | d=xsd:anyURI
+foaf:logo | d=xsd:anyURI
+# Documents about the entity
+foaf:page | d=xsd:anyURI
+
+# --- dbpedia specific
+# the "dbp-ont" defines knowledge mapped to the DBPedia ontology
+dbp-ont:*
+# the "DBpedia properties are all key values pairs extracted from the info boxes
+# on the right hand side of Wikipedia pages.
+#dbp-prop:*
+# Copy only population for now (one could add additional if necessary)!
+dbp-prop:population | d=xsd:integer
\ No newline at end of file
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/dbpediamappings.txt
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexfieldconfig.txt
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexfieldconfig.txt?rev=1130732&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexfieldconfig.txt (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexfieldconfig.txt Thu Jun 2 18:59:04 2011
@@ -0,0 +1,28 @@
+#This is a similar version of the dbPediaMappings.txt but excludes mapping rules
+# --- Define the Languages for all fields ---
+| @=null;en;de;fr;it
+
+# --- RDF, RDFS and OWL Mappings ---
+rdfs:label
+rdfs:comment
+rdf:type
+# used by LOD to link to URIs used to identify the same Entity
+owl:sameAs
+
+# --- Other Namespaces to include---
+dc:*
+dc-elements:*
+geo:*
+skos:*
+foaf:*
+
+# --- dbpedia specific
+# the "dbp-ont" defines knowledge mapped to the DBPedia ontology
+dbp-ont:*
+
+# the "DBpedia properties are all key values pairs extracted from the info boxes
+# on the right hand side of Wikipedia pages.
+# uncomment the next line to include all
+#dbp-prop:*
+# Currently only the population is added
+dbp-prop:population
\ No newline at end of file
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexfieldconfig.txt
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexing.properties
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexing.properties?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexing.properties (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/indexing.properties Thu Jun 2 18:59:04 2011
@@ -1,9 +1,9 @@
name=simple
description=Simple Configuration
-scoreNormalizer=org.apache.stanbol.entityhub.indexing.core.normaliser.RangeNormaliser,config:range;org.apache.stanbol.entityhub.indexing.core.normaliser.NaturalLogNormaliser;org.apache.stanbol.entityhub.indexing.core.normaliser.MinScoreNormalizer,config:minincoming
+scoreNormalizer=org.apache.stanbol.entityhub.indexing.core.normaliser.RangeNormaliser,config:range;org.apache.stanbol.entityhub.indexing.core.normaliser.NaturalLogNormaliser;org.apache.stanbol.entityhub.indexing.core.normaliser.MinScoreNormalizer,config:minscore
entityIdIterator=org.apache.stanbol.entityhub.indexing.core.source.LineBasedEntityIterator,source:testEntityScore.txt,charset:UTF-8,encodeIds:false
-entityProcessor=org.apache.stanbol.entityhub.indexing.core.processor.FiledMapperProcessor,mappings:dbPediaMappings.txt
-fieldConfiguration=indexFieldConfig.txt
\ No newline at end of file
+entityProcessor=org.apache.stanbol.entityhub.indexing.core.processor.FiledMapperProcessor,mappings:dbpediamappings.txt
+fieldConfiguration=indexfieldconfig.txt
\ No newline at end of file
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/minscore.properties
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/minscore.properties?rev=1130732&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/minscore.properties (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/minscore.properties Thu Jun 2 18:59:04 2011
@@ -0,0 +1,2 @@
+inclusive=true
+min-score=2
\ No newline at end of file
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/test/resources/testConfigs/simple/indexing/config/minscore.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Propchange: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/fieldboosts.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/indexing.properties
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/indexing.properties?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/indexing.properties (original)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/indexing.properties Thu Jun 2 18:59:04 2011
@@ -19,5 +19,24 @@ entityProcessor=org.apache.stanbol.entit
fieldConfiguration=mappings.txt
# Finally we need a destination to store the data
-# Note also the configured fieldBoosts.properties
-indexingDestination=org.apache.stanbol.entityhub.indexing.destination.solryard.SolrYardIndexingDestination,solrConf,boosts:fieldBoosts
\ No newline at end of file
+# Note also the configured fieldboosts.properties
+# For the SolrYard configuration we can use the default
+indexingDestination=org.apache.stanbol.entityhub.indexing.destination.solryard.SolrYardIndexingDestination,boosts:fieldboosts
+
+# Configuration for the remote DBpedia Server
+# NOTE: this is optional and can be commented. Local only mode can also be
+# enforced by starting Stanbol in offline mode!
+
+# Use the Cool URI dereferencer
+org.apache.stanbol.entityhub.site.entityPrefix=http://dblp.l3s.de/d2r/resource
+
+org.apache.stanbol.entityhub.site.accessUri=http://dblp.l3s.de/d2r/resource/
+org.apache.stanbol.entityhub.site.dereferencerType=org.apache.stanbol.entityhub.site.CoolUriDereferencer
+# The SPARQL endpoint is a d2r server so use a standard sparql server
+org.apache.stanbol.entityhub.site.queryUri=http://dblp.l3s.de/d2r/sparql
+org.apache.stanbol.entityhub.site.searcherType=org.apache.stanbol.entityhub.site.SparqlSearcher
+
+# The mappings used when importing an entity form this site to the Entityhub (optional)
+# The value need to point to the file with the mappings within the config directory
+# here we use the same values as for indexing
+org.apache.stanbol.entityhub.site.fieldMappings=mappings.txt
Propchange: incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/fieldboosts.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Propchange: incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/iditerator.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/indexing.properties
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/indexing.properties?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/indexing.properties (original)
+++ incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/indexing.properties Thu Jun 2 18:59:04 2011
@@ -10,8 +10,8 @@ description=DBpedia.org
# The dump of the RDF data needs to be located in the "indexing/resource/rdfdata"
entityDataProvider=org.apache.stanbol.entityhub.indexing.source.jenatdb.RdfIndexingSource,source:rdfdata
# Iterate over the entity ids&scores
-# the configuration is located wihtin the idIterator.properties
-entityIdIterator=org.apache.stanbol.entityhub.indexing.core.source.LineBasedEntityIterator,config:idIterator
+# the configuration is located wihtin the iditerator.properties
+entityIdIterator=org.apache.stanbol.entityhub.indexing.core.source.LineBasedEntityIterator,config:iditerator
# Entity Scores are calculated
# - based on the incomming links (provided by the entityIdIterator)
@@ -21,8 +21,8 @@ entityIdIterator=org.apache.stanbol.enti
# For this we need three Score
# - MinScoreNormalizer: for configuration see "minincoming.properties"
# - NaturalLogNormaliser: no configuration required
-# - RangeNormaliser: for configuration see "scoreRange.properties"
-scoreNormalizer=org.apache.stanbol.entityhub.indexing.core.normaliser.RangeNormaliser,config:scoreRange;org.apache.stanbol.entityhub.indexing.core.normaliser.NaturalLogNormaliser;org.apache.stanbol.entityhub.indexing.core.normaliser.MinScoreNormalizer,config:minincoming
+# - RangeNormaliser: for configuration see "scorerange.properties"
+scoreNormalizer=org.apache.stanbol.entityhub.indexing.core.normaliser.RangeNormaliser,config:scorerange;org.apache.stanbol.entityhub.indexing.core.normaliser.NaturalLogNormaliser;org.apache.stanbol.entityhub.indexing.core.normaliser.MinScoreNormalizer,config:minincoming
# Use the specified field mappings (in the mappings.txt file) to process entities
# It automatically uses the value configured for "fieldConfiguration"
@@ -32,5 +32,39 @@ entityProcessor=org.apache.stanbol.entit
fieldConfiguration=mappings.txt
# Finally we need a destination to store the data
-# Note also the configured fieldBoosts.properties
-indexingDestination=org.apache.stanbol.entityhub.indexing.destination.solryard.SolrYardIndexingDestination,solrConf,boosts:fieldBoosts
+# Note also the configured fieldboosts.properties
+indexingDestination=org.apache.stanbol.entityhub.indexing.destination.solryard.SolrYardIndexingDestination,solrConf,boosts:fieldboosts
+
+# Additional Properties directly added to the configuration of the
+# Referenced Site
+
+# Prefixes for Indexed Entities:
+# currently resources as well as the ontology are accepted
+org.apache.stanbol.entityhub.site.entityPrefix=http://dbpedia.org/resource/;http://dbpedia.org/ontology/
+
+# Configuration for the remote DBpedia Server
+# NOTE: this is optional and can be commented. Local only mode can also be
+# enforced by starting Stanbol in offline mode!
+
+# Dereferencer should use SPARQL because Cool URI will omit statements for popular
+# Entities.
+org.apache.stanbol.entityhub.site.accessUri=http://dbpedia.org/sparql/
+org.apache.stanbol.entityhub.site.dereferencerType=org.apache.stanbol.entityhub.site.SparqlDereferencer
+# The SPARQL endpoint of DBpedia supports Virtuoso specific extensions
+org.apache.stanbol.entityhub.site.searcherType=org.apache.stanbol.entityhub.site.VirtuosoSearcher
+org.apache.stanbol.entityhub.site.queryUri=http://dbpedia.org/sparql
+
+# The mappings used when importing an entity form this site to the Entityhub (optional)
+# The value need to point to the file with the mappings within the config directory
+# here we use the same values as for indexing
+org.apache.stanbol.entityhub.site.fieldMappings=mappings.txt
+
+# License(s)
+# DBpedia uses dual licenses
+# NOTE: licenseName and licenseUrl MUST use the same ordering!
+org.apache.stanbol.entityhub.site.licenseName=Creative Commons Attribution-ShareAlike 3.0;GNU Free Documentation License
+org.apache.stanbol.entityhub.site.licenseUrl=http://en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License;http://en.wikipedia.org/wiki/Wikipedia:Text_of_the_GNU_Free_Documentation_License
+# Attribution
+# The DBpedia License requires Attribution!
+org.apache.stanbol.entityhub.site.attributionUrl=http://wiki.dbpedia.org/About
+org.apache.stanbol.entityhub.site.attribution=DBpedia.org
Propchange: incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/scorerange.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/test/java/org/apache/stanbol/entityhub/indexing/dbpedia/ConfigTest.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/test/java/org/apache/stanbol/entityhub/indexing/dbpedia/ConfigTest.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/test/java/org/apache/stanbol/entityhub/indexing/dbpedia/ConfigTest.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/dbpedia/src/test/java/org/apache/stanbol/entityhub/indexing/dbpedia/ConfigTest.java Thu Jun 2 18:59:04 2011
@@ -72,7 +72,7 @@ public class ConfigTest {
assertNotNull(normalisedScore);
float nScore = normalisedScore.floatValue();
assertTrue(nScore <= lastNormalisedScore);
- if(score < 2){ //the value of "min-score" in minIncomming
+ if(score < 2){ //the value of "min-score" in minincoming
log.info("score="+score+" nScore="+nScore);
assertTrue(nScore < 0);
return;
Modified: incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestination.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestination.java?rev=1130732&r1=1130731&r2=1130732&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestination.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestination.java Thu Jun 2 18:59:04 2011
@@ -5,6 +5,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collection;
+import java.util.Dictionary;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@@ -18,6 +19,7 @@ import org.apache.stanbol.entityhub.core
import org.apache.stanbol.entityhub.core.site.CacheUtils;
import org.apache.stanbol.entityhub.indexing.core.IndexingDestination;
import org.apache.stanbol.entityhub.indexing.core.config.IndexingConfig;
+import org.apache.stanbol.entityhub.indexing.core.destination.OsgiConfigurationUtil;
import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapper;
import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapping;
import org.apache.stanbol.entityhub.servicesapi.model.rdf.RdfResourceEnum;
@@ -94,6 +96,10 @@ public class SolrYardIndexingDestination
* The extension of the solrIndex reference file
*/
public static final String SOLR_INDEX_ARCHIVE_REF_EXTENSION = ".solrindex.ref";
+ /**
+ * The ID of the OSGI component used by the SolrYard implementation
+ */
+ public static final String SOLR_YARD_COMPONENT_ID = "org.apache.stanbol.entityhub.yard.solr.impl.SolrYard";
/**
* The location of the SolrIndex. This MUST BE an absolute Path in case it
@@ -103,12 +109,6 @@ public class SolrYardIndexingDestination
* {@link SolrYardConfig#getSolrServerLocation()} on {@link #solrIndexConfig}
*/
private File solrIndexLocation;
- /**
- * The directory used in case an embedded Solr server is used. This value
- * is also available by the System property with the name
- * {@link SolrDirectoryManager#MANAGED_SOLR_DIR_PROPERTY}
- */
- private File managedSolrDir;
/**
* Directory holding the specialised Solr configuration or <code>null</code>
@@ -139,6 +139,8 @@ public class SolrYardIndexingDestination
* created SolrIndex.
*/
private Collection<FieldMapping> indexFieldConfiguration;
+
+ private IndexingConfig indexingConfig;
/**
* This Constructor relays on a subsequent call to
@@ -293,10 +295,8 @@ public class SolrYardIndexingDestination
"Directory"+distDirectory.getAbsolutePath());
}
}
- solrIndexArchive = new File(distDirectory,
- solrIndexLocation.getName()+SOLR_INDEX_ARCHIVE_EXTENSION);
- solrIndexArchiveRef = new File(distDirectory,
- solrIndexLocation.getName()+SOLR_INDEX_ARCHIVE_REF_EXTENSION);
+ solrIndexArchive = new File(solrIndexLocation.getName()+SOLR_INDEX_ARCHIVE_EXTENSION);
+ solrIndexArchiveRef = new File(solrIndexLocation.getName()+SOLR_INDEX_ARCHIVE_REF_EXTENSION);
}
return new File[]{solrIndexLocation,solrConfigLocation,
solrIndexArchive,solrIndexArchiveRef};
@@ -304,7 +304,7 @@ public class SolrYardIndexingDestination
@Override
public void setConfiguration(Map<String,Object> config) {
- IndexingConfig indexingConfig = (IndexingConfig)config.get(IndexingConfig.KEY_INDEXING_CONFIG);
+ indexingConfig = (IndexingConfig)config.get(IndexingConfig.KEY_INDEXING_CONFIG);
String yardName;
//read the Yard name configuration
Object value = config.get(PARAM_YARD_NAME);
@@ -334,7 +334,8 @@ public class SolrYardIndexingDestination
}
//get the directors holding the solr configuration
String solrConfig;
- if(!config.containsKey(PARAM_SOLR_CONFIG)){ //not present -> allow default
+ if(!config.containsKey(PARAM_SOLR_CONFIG)){ //not present
+ // -> use the default config
File configDir = indexingConfig.getConfigFile(indexName);
if(!configDir.isDirectory()){
log.info("use default Solr index configuration for index "+indexName);
@@ -415,6 +416,7 @@ public class SolrYardIndexingDestination
@Override
public void initialise() {
+ log.info("initialise {}",getClass().getSimpleName());
//The constructors and the setConfiguration(..) only validate the parsed
//parameters and initialise the member variables. This method performs
//the the actual initialisation of the SolrYard!
@@ -422,6 +424,7 @@ public class SolrYardIndexingDestination
if(solrIndexConfig != null){ //can only be != null if also solrIndexLocation
//copy the configuration
try {
+ log.info(" ... copy Solr Configuration form {} to {}",solrIndexConfig,solrIndexLocation);
FileUtils.copyDirectory(solrIndexConfig, solrIndexLocation);
} catch (IOException e) {
throw new IllegalStateException(String.format(
@@ -432,9 +435,11 @@ public class SolrYardIndexingDestination
solrYardConfig.setDefaultInitialisation(Boolean.FALSE);
} else {
//allow the default initialisation
+ log.info(" ... use default Solr Configuration");
solrYardConfig.setDefaultInitialisation(Boolean.TRUE);
}
try {
+ log.info(" ... create SolrYard");
this.solrYard = new SolrYard(solrYardConfig);
} catch (YardException e) {
throw new IllegalStateException("Unable to initialise SolrYard "+
@@ -463,56 +468,126 @@ public class SolrYardIndexingDestination
solrYard.close();
//zip the index and copy it over to distribution
if(solrArchive != null){
- //we need to get the length of the parent to calc the entry names for
- //the archvie
- int parentPathLength = solrIndexLocation.getAbsolutePath().length();
- if(solrIndexLocation.getAbsolutePath().charAt(parentPathLength-1) != File.separatorChar){
- parentPathLength++; //add the missing '/'
- }
try {
- //Moved over to use java.util.zip because Apache commons compression
- //seams not support files > 2Gb
- ZipOutputStream out = new ZipOutputStream(new FileOutputStream(solrArchive));
- for(File file : (Collection<File>)FileUtils.listFiles(solrIndexLocation, null, true)){
- if(!file.isHidden()){
- String name = file.getAbsolutePath().substring(parentPathLength);
- log.info("add "+name);
- out.putNextEntry(new ZipEntry(name));
- if(!file.isDirectory()){
- FileInputStream fileIn = new FileInputStream(file);
- IOUtils.copyLarge(fileIn,out);
- out.closeEntry();
- IOUtils.closeQuietly(fileIn);
- }
- }
- }
- out.finish();
- IOUtils.closeQuietly(out);
+ writeSolrIndexArchive();
}catch (IOException e) {
log.error("Error while creating Solr Archive "+solrArchive.getAbsolutePath()+
"! The archive will not be created!",e);
log.error("As a Workaround you can manually create the Solr Archive " +
- "by creating a ZIP archive with the contents of the Folder " +
- solrIndexLocation+"!");
+ "by creating a ZIP archive with the contents of the Folder " +
+ solrIndexLocation+"!");
}
}
if(solrArchiveRef != null){
- Properties properties = new Properties();
- properties.setProperty("Index-Archive", solrArchive.getName());
- properties.setProperty("Name", solrYardConfig.getName());
- if(solrYardConfig.getDescription() != null){
- properties.setProperty("Description", solrYardConfig.getDescription());
- }
try {
- properties.store(new FileOutputStream(solrArchiveRef), null);
+ writeSolrIndexReference();
} catch (IOException e) {
log.error("Error while creating Solr Archive Reference "+
solrArchiveRef.getAbsolutePath()+
"! The file will not be created!",e);
- log.error("As a Workaround you can manually create this text file " +
- "and adding \"Index-Archive="+solrArchive.getName()+"\"!");
}
}
+ //finally create the Osgi Configuration
+ try {
+ OsgiConfigurationUtil.writeSiteConfiguration(indexingConfig);
+ } catch (IOException e) {
+ log.error("Unable to write OSGI configuration file for the referenced site",e);
+ }
+ try {
+ OsgiConfigurationUtil.writeCacheConfiguration(indexingConfig);
+ } catch (IOException e) {
+ log.error("Unable to write OSGI configuration file for the Cache",e);
+ }
+ //create the SolrYard configuration
+ try {
+ writeSolrYardConfiguration();
+ } catch (IOException e) {
+ log.error("Unable to write OSGI configuration file for the SolrYard",e);
+ }
+ //create the bundle
+ OsgiConfigurationUtil.createBundle(indexingConfig);
+ }
+ /**
+ *
+ */
+ private void writeSolrIndexReference() throws IOException {
+ Properties properties = new Properties();
+ properties.setProperty("Index-Archive", solrArchive.getName());
+ properties.setProperty("Name", solrYardConfig.getName());
+ if(solrYardConfig.getDescription() != null){
+ properties.setProperty("Description", solrYardConfig.getDescription());
+ }
+ File solrArchiveFile = new File(
+ OsgiConfigurationUtil.getConfigDirectory(indexingConfig),solrArchiveRef.getName());
+ properties.store(new FileOutputStream(solrArchiveFile), null);
+ }
+ /**
+ *
+ */
+ private void writeSolrIndexArchive() throws IOException{
+ //we need to get the length of the parent to calc the entry names for
+ //the archvie
+ //Note that the Archive need to include the name of the index,
+ //therefore we need use the parent dir as context
+ int parentPathLength = solrIndexLocation.getParentFile().getAbsolutePath().length();
+ if(solrIndexLocation.getAbsolutePath().charAt(parentPathLength-1) != File.separatorChar){
+ parentPathLength++; //add the missing '/'
+ }
+ //Moved over to use java.util.zip because Apache commons compression
+ //seams not support files > 2Gb
+ File solrArchiveFile = new File(indexingConfig.getDistributionFolder(),solrArchive.getName());
+ ZipOutputStream out = new ZipOutputStream(new FileOutputStream(solrArchiveFile));
+ for(File file : (Collection<File>)FileUtils.listFiles(solrIndexLocation, null, true)){
+ if(!file.isHidden()){
+ String name = file.getAbsolutePath().substring(parentPathLength);
+ log.info("add "+name);
+ out.putNextEntry(new ZipEntry(name));
+ if(!file.isDirectory()){
+ FileInputStream fileIn = new FileInputStream(file);
+ IOUtils.copyLarge(fileIn,out);
+ out.closeEntry();
+ IOUtils.closeQuietly(fileIn);
+ }
+ }
+ }
+ out.finish();
+ IOUtils.closeQuietly(out);
+ }
+ /**
+ * @throws IOException
+ *
+ */
+ private void writeSolrYardConfiguration() throws IOException {
+ Dictionary<String,Object> yardConfig = OsgiConfigurationUtil.createYardConfig(indexingConfig);
+ //we need now add the solrYard specific parameters
+ String fieldBoostName = solrYardConfig.getDocumentBoostFieldName();
+ if(fieldBoostName != null){
+ yardConfig.put(SolrYard.DOCUMENT_BOOST_FIELD, fieldBoostName);
+ }
+ //TODO: fieldBoosts are currently not supported by the SolrYard Config
+ //solrYardConfig.getFieldBoosts();
+
+ //The default values for the following parameters are OK
+ //solrYardConfig.getMaxBooleanClauses();
+ //solrYardConfig.getMaxQueryResultNumber();
+
+ yardConfig.put(SolrYard.SOLR_SERVER_LOCATION, solrYardConfig.getSolrServerLocation());
+ //the server type needs not to be set. It is automatically detected by
+ //the value of the server location
+ //solrYardConfig.getSolrServerType();
+
+ //deactivate default initialisation!
+ yardConfig.put(SolrYard.SOLR_INDEX_DEFAULT_CONFIG, Boolean.FALSE);
+
+ //for immediate commit use the default value (optionally one could also
+ //fore TRUE)
+ //yardConfig.put(SolrYard.IMMEDIATE_COMMIT, Boolean.TRUE);
+
+ //deactivate multi yard layout!
+ yardConfig.put(SolrYard.MULTI_YARD_INDEX_LAYOUT, Boolean.FALSE);
+
+ String solrYardConfigFileName = SOLR_YARD_COMPONENT_ID+'-'+indexingConfig.getName()+".config";
+ OsgiConfigurationUtil.writeOsgiConfig(indexingConfig,solrYardConfigFileName, yardConfig);
}
@Override