You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2010/12/05 19:00:20 UTC
svn commit: r1042390 [1/2] - in /incubator/stanbol/trunk/rick: ./
generic/core/ generic/core/src/main/java/eu/iksproject/rick/core/impl/
generic/core/src/main/java/eu/iksproject/rick/core/mapping/
generic/servicesapi/ generic/servicesapi/src/main/java/...
Author: rwesten
Date: Sun Dec 5 18:00:18 2010
New Revision: 1042390
URL: http://svn.apache.org/viewvc?rev=1042390&view=rev
Log:
RICK ServicesAPI
- added a utility method to the NamespaceEnum that converts short URIs (prefix:localName) to the fullURI (namespace+localName)
- FieldMapper now returns the target representation to allow chaining calls.
- Added support for Sign-Ranking: This is the ranking of an Entity within an referenced site (e.g. based on the incoming links to that Entity)
- ReferencedSites now have a method to get the preferred query factory
RICK Core
- implementation of the API changes
- corrected a bug in ReferencedSiteImpl where the EntitySearcher was called after a successful Query on the local Cache (CacheStrategy.all)
- merged http://code.google.com/p/iks-project/source/detail?r=1121)
SolrYard
- added support for parsing the score for query results (result scores are parsed by using the field "http://www.iks-project.eu/ontology/rick/query/score"
- added support for document boost and field boosts. Currently the field used to read the Document boost is configurable by the SolrYardConfig. Future versions might hard wire this feature to "http://www.iks-project.eu/ontology/rick/model/signRank" (the according field defined in the RICK schema). Field Boost values can be defined by parsing a map with the field as key and the float boost as value.
Generic RDF Indexer:
First version of a generic RDF Indexer. The configuration is currently done by a Dictionary. Supported keys are defines as static fields.
Future versions will add specific configuration classes for typical use cases. The dbPedia indexer provide a good usage example
Features:
- reads multiple RDF files from a directory
- supports compressed RDF files
- supports several RDF formats
- uses a Jena TDB (file based triple store)
- supports skipping of RDF data (e.g. to use an existing Triple store)
Performance depends extremely on the hard disc.
dbPedia Indexer:
This indexer allows to create a FULL Index of dbPedia within a SolrYard.
It has support for EntityRanks based on incoming links within DB-Pedia. A precalculated file is available at http://public.dev.nuxeo.com/~ogrisel/incoming-counts.tsv.gz
geonames Indexer
- changed artifactID form eu.iksproject.indexing.geonames to eu.iksproject.rick.indexing.geonames
Added:
incubator/stanbol/trunk/rick/indexing/dbPedia/
incubator/stanbol/trunk/rick/indexing/dbPedia/pom.xml (with props)
incubator/stanbol/trunk/rick/indexing/dbPedia/src/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java (with props)
Removed:
incubator/stanbol/trunk/rick/Rick.launch
Modified:
incubator/stanbol/trunk/rick/generic/core/ (props changed)
incubator/stanbol/trunk/rick/generic/core/pom.xml
incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/impl/ReferencedSiteImpl.java
incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/DefaultFieldMapperImpl.java
incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/FieldMappingUtils.java
incubator/stanbol/trunk/rick/generic/servicesapi/ (props changed)
incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/defaults/NamespaceEnum.java
incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/mapping/FieldMapper.java
incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/model/rdf/RdfResourceEnum.java
incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/query/QueryResultList.java
incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/site/ReferencedSite.java
incubator/stanbol/trunk/rick/generic/test/ (props changed)
incubator/stanbol/trunk/rick/indexing/genericRdf/ (props changed)
incubator/stanbol/trunk/rick/indexing/genericRdf/pom.xml
incubator/stanbol/trunk/rick/indexing/genericRdf/src/main/java/eu/iksproject/rick/indexing/rdf/RdfIndexer.java
incubator/stanbol/trunk/rick/indexing/geonames/ (props changed)
incubator/stanbol/trunk/rick/indexing/geonames/pom.xml
incubator/stanbol/trunk/rick/jersey/ (props changed)
incubator/stanbol/trunk/rick/jersey/pom.xml
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JerseyEndpoint.java
incubator/stanbol/trunk/rick/launchers/sling/ (props changed)
incubator/stanbol/trunk/rick/model/clerezza/ (props changed)
incubator/stanbol/trunk/rick/pom.xml
incubator/stanbol/trunk/rick/query/clerezza/ (props changed)
incubator/stanbol/trunk/rick/site/linkedData/ (props changed)
incubator/stanbol/trunk/rick/yard/clerezza/ (props changed)
incubator/stanbol/trunk/rick/yard/solr/ (props changed)
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/defaults/SolrConst.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrFieldMapper.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrQueryFactory.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYard.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYardConfig.java
Propchange: incubator/stanbol/trunk/rick/generic/core/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sun Dec 5 18:00:18 2010
@@ -0,0 +1,7 @@
+target
+
+.settings
+
+.project
+
+.classpath
Modified: incubator/stanbol/trunk/rick/generic/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/core/pom.xml?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/core/pom.xml (original)
+++ incubator/stanbol/trunk/rick/generic/core/pom.xml Sun Dec 5 18:00:18 2010
@@ -89,6 +89,7 @@
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>1.6.2</version>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
Modified: incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/impl/ReferencedSiteImpl.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/impl/ReferencedSiteImpl.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/impl/ReferencedSiteImpl.java (original)
+++ incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/impl/ReferencedSiteImpl.java Sun Dec 5 18:00:18 2010
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
import eu.iksproject.rick.core.mapping.DefaultFieldMapperImpl;
import eu.iksproject.rick.core.mapping.FieldMappingUtils;
import eu.iksproject.rick.core.mapping.ValueConverterFactory;
+import eu.iksproject.rick.core.query.DefaultQueryFactory;
import eu.iksproject.rick.core.query.QueryResultListImpl;
import eu.iksproject.rick.core.site.AbstractEntityDereferencer;
import eu.iksproject.rick.core.utils.ModelUtils;
@@ -45,6 +46,7 @@ import eu.iksproject.rick.servicesapi.mo
import eu.iksproject.rick.servicesapi.model.Sign;
import eu.iksproject.rick.servicesapi.model.Symbol;
import eu.iksproject.rick.servicesapi.query.FieldQuery;
+import eu.iksproject.rick.servicesapi.query.FieldQueryFactory;
import eu.iksproject.rick.servicesapi.query.QueryResultList;
import eu.iksproject.rick.servicesapi.site.ConfiguredSite;
import eu.iksproject.rick.servicesapi.site.EntityDereferencer;
@@ -221,7 +223,8 @@ public class ReferencedSiteImpl implemen
protected Dictionary<String,?> properties;
protected FieldMapper fieldMappings;
- private boolean dereferencerEqualsEntitySearcherComponent;
+ private final Object searcherAndDereferencerLock = new Object();
+ private Boolean dereferencerEqualsEntitySearcherComponent;
private ComponentFactoryListener dereferencerComponentFactoryListener;
private ComponentFactoryListener searcherComponentFactoryListener;
@@ -368,6 +371,7 @@ public class ReferencedSiteImpl implemen
for(Representation result : representations){
results.add(ModelUtils.createSign(result, getId()));
}
+ return new QueryResultListImpl<Sign>(query, results, Sign.class);
} catch (YardException e) {
if(entitySearcherComponentName==null){
throw new ReferencedSiteException("Unable to execute query on Cache "+cacheId,e);
@@ -382,7 +386,7 @@ public class ReferencedSiteImpl implemen
log.warn(String.format("Cache %s currently not active will query remote Site %s as fallback",cacheId,queryUri));
}
}
- }
+ }
QueryResultList<String> entityIds;
try {
entityIds = entitySearcher.findEntities(query);
@@ -580,6 +584,25 @@ public class ReferencedSiteImpl implemen
}
/**
+ * In case {@link CacheStrategy#all} this Method returns the
+ * query factory of the Cache.
+ * Otherwise it returns {@link DefaultQueryFactory#getInstance()}.
+ */
+ @Override
+ public FieldQueryFactory getQueryFactory() {
+ FieldQueryFactory factory = null;
+ if(cacheStrategy == CacheStrategy.all){
+ Cache cache = getCache();
+ if(cache != null){
+ factory = cache.getQueryFactory();
+ }
+ }
+ if(factory == null){
+ factory = DefaultQueryFactory.getInstance();
+ }
+ return factory;
+ }
+ /**
* Internally used to get the Cache for this site. If
* {@link CacheStrategy#none}, this methods always returns <code>null</code>,
* otherwise it returns the Cache for the configured Yard or <code>null</code>
@@ -739,9 +762,9 @@ public class ReferencedSiteImpl implemen
*/
@SuppressWarnings("unchecked")
protected void createEntitySearcherComponent(ComponentFactory factory){
- //both create*** methods sync on the cacheId to avoid
+ //both create*** methods sync on the searcherAndDereferencerLock to avoid
//multiple component instances because of concurrent calls
- synchronized (this.cacheId) {
+ synchronized (this.searcherAndDereferencerLock ) {
if(entitySearcherComponentInstace == null){
this.entitySearcherComponentInstace = factory.newInstance(OsgiUtils.copyConfig(context.getProperties()));
this.entitySearcher = (EntitySearcher)entitySearcherComponentInstace.getInstance();
@@ -759,9 +782,9 @@ public class ReferencedSiteImpl implemen
*/
@SuppressWarnings("unchecked")
protected void createDereferencerComponent(ComponentFactory factory){
- //both create*** methods sync on cacheId to avoid
+ //both create*** methods sync on searcherAndDereferencerLock to avoid
//multiple component instances because of concurrent calls
- synchronized (this.cacheId) {
+ synchronized (this.searcherAndDereferencerLock) {
if(dereferencerComponentInstance == null){
dereferencerComponentInstance=factory.newInstance(OsgiUtils.copyConfig(context.getProperties()));
this.dereferencer = (EntityDereferencer)dereferencerComponentInstance.getInstance();
Modified: incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/DefaultFieldMapperImpl.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/DefaultFieldMapperImpl.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/DefaultFieldMapperImpl.java (original)
+++ incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/DefaultFieldMapperImpl.java Sun Dec 5 18:00:18 2010
@@ -186,7 +186,7 @@ public class DefaultFieldMapperImpl impl
/* (non-Javadoc)
* @see eu.iksproject.rick.servicesapi.mapping.FieldMapper#applyMappings(eu.iksproject.rick.servicesapi.model.Representation, eu.iksproject.rick.servicesapi.model.Representation)
*/
- public void applyMappings(Representation source, Representation target) {
+ public Representation applyMappings(Representation source, Representation target) {
Collection<String> fields = new HashSet<String>();
for(Iterator<String> fieldIt = source.getFieldNames();fieldIt.hasNext();){
fields.add(fieldIt.next());
@@ -267,6 +267,7 @@ public class DefaultFieldMapperImpl impl
* All mapping activities should be documented and stored with the
* MappedEntity as MappingActivity!
*/
+ return target;
}
/**
*
@@ -303,7 +304,6 @@ public class DefaultFieldMapperImpl impl
mapping.getFilter().getType(),mapping.getFilter()));
break;
}
- log.warn("TODO: Filters are currently not supported by this Implementation");
/*
* TODO: add general purpose functionality to apply Constraints.
* Currently this is done by the specific Query Implementations :(
Modified: incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/FieldMappingUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/FieldMappingUtils.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/FieldMappingUtils.java (original)
+++ incubator/stanbol/trunk/rick/generic/core/src/main/java/eu/iksproject/rick/core/mapping/FieldMappingUtils.java Sun Dec 5 18:00:18 2010
@@ -129,7 +129,7 @@ public class FieldMappingUtils {
String[] parts = mapping.split(" "); //TODO: maybe we should not use the spaces here
List<String> mappedTo = Collections.emptyList();
- String fieldPattern = getFullUri(parts[0]);
+ String fieldPattern = NamespaceEnum.getFullName(parts[0]);
Constraint filter = null;
for(int i=1;i<parts.length;i++){
if("|".equals(parts[i]) && parts.length > i+1){
@@ -145,21 +145,22 @@ public class FieldMappingUtils {
}
return new FieldMapping(fieldPattern, filter, mappedTo.toArray(new String[mappedTo.size()]));
}
- private static String getFullUri(String value){
- int index = value.indexOf(':');
- if(index>0){
- NamespaceEnum namespace = NamespaceEnum.forPrefix(value.substring(0, index));
- if(namespace!= null){
- value = namespace.getNamespace()+value.substring(index+1);
- }
- }
- return value;
- }
+//moved to NamespaceEnum
+// private static String getFullUri(String value){
+// int index = value.indexOf(':');
+// if(index>0){
+// NamespaceEnum namespace = NamespaceEnum.forPrefix(value.substring(0, index));
+// if(namespace!= null){
+// value = namespace.getNamespace()+value.substring(index+1);
+// }
+// }
+// return value;
+// }
private static List<String> parseMappings(String[] parts, int start) {
ArrayList<String> mappings = new ArrayList<String>(parts.length-start);
for(int i=start;i<parts.length;i++){
- String act = getFullUri(parts[i]);
+ String act = NamespaceEnum.getFullName(parts[i]);
if(!act.isEmpty()){ //needed to remove two spaces in a row
mappings.add(act);
}
Propchange: incubator/stanbol/trunk/rick/generic/servicesapi/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sun Dec 5 18:00:18 2010
@@ -0,0 +1,7 @@
+target
+
+.classpath
+
+.project
+
+.settings
Modified: incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/defaults/NamespaceEnum.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/defaults/NamespaceEnum.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/defaults/NamespaceEnum.java (original)
+++ incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/defaults/NamespaceEnum.java Sun Dec 5 18:00:18 2010
@@ -119,9 +119,24 @@ public enum NamespaceEnum {
public static NamespaceEnum forPrefix(String prefix){
return prefix2Namespace.get(prefix);
}
-// public static void main(String[] args) {
-// for(NamespaceEnum entry : NamespaceEnum.values()){
-// System.out.println(String.format("forNamespace(%s)=%s == %s",entry.getNamespace(),forNamespace(entry.getNamespace()).getPrefix(),entry.getPrefix()));
-// }
-// }
+ /**
+ * Lookup if the parsed URI uses one of the registered prefixes of this
+ * Enumeration. If this is the case, the prefix is replaced by the namespace
+ * and the full URI is returned. If no prefix is returned, the
+ * parsed URI is returned
+ * @param shortUri the short URI
+ * @return the full URI if the parsed shortUri uses a prefix defined by this
+ * Enumeration. Otherwise the parsed value.
+ */
+ public static String getFullName(String shortUri){
+ if(shortUri == null) return null;
+ int index = shortUri.indexOf(':');
+ if(index>0){
+ NamespaceEnum namespace = NamespaceEnum.forPrefix(shortUri.substring(0, index));
+ if(namespace!= null){
+ shortUri = namespace.getNamespace()+shortUri.substring(index+1);
+ }
+ }
+ return shortUri;
+ }
}
Modified: incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/mapping/FieldMapper.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/mapping/FieldMapper.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/mapping/FieldMapper.java (original)
+++ incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/mapping/FieldMapper.java Sun Dec 5 18:00:18 2010
@@ -30,11 +30,12 @@ public interface FieldMapper extends Clo
* are NOT used as source fields for further mappings.
* @param source the source for the mapping process
* @param target the target for the processed mappings (can be the same as source)
+ * @return the {@link Representation} parsed as target.
* TODO: This Method should return a MappingReport, that can be stored with
* the {@link EntityMapping}. However the MappingActivity functionality is
* not yet designed/implemented!
*/
- void applyMappings(Representation source, Representation target);
+ Representation applyMappings(Representation source, Representation target);
/**
* Getter for the unmodifiable collection of all mappings
* @return the configured mappings
Modified: incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/model/rdf/RdfResourceEnum.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/model/rdf/RdfResourceEnum.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/model/rdf/RdfResourceEnum.java (original)
+++ incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/model/rdf/RdfResourceEnum.java Sun Dec 5 18:00:18 2010
@@ -20,6 +20,14 @@ public enum RdfResourceEnum {
*/
signSite,
/**
+ * The ranking of the entity by this site in the value range of [0..1]
+ * A sign with the rank 1 would be (one of) the most important entities
+ * managed by this Site. A sign with rank 0 has no relevance. Even that this
+ * is still within the value range one could wonder why this site does
+ * even manage a representation about that entity.
+ */
+ signRank,
+ /**
* The representation of the Sign (domain=Sign, range=Representation).
*/
signRepresentation,
@@ -121,10 +129,9 @@ public enum RdfResourceEnum {
*/
queryResult(NamespaceEnum.rickQuery),
/**
- * The ranking of the result, optimally the page rank or if not available the
- * number of incoming relations
+ * The score of the result in respect to the parsed query.
*/
- resultRank(NamespaceEnum.rickQuery),
+ resultScore(NamespaceEnum.rickQuery,"score"),
/**
* The id of the site the result was found
*/
Modified: incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/query/QueryResultList.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/query/QueryResultList.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/query/QueryResultList.java (original)
+++ incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/query/QueryResultList.java Sun Dec 5 18:00:18 2010
@@ -38,5 +38,4 @@ public interface QueryResultList<T> exte
* @return the type
*/
Class<T> getType();
-
}
\ No newline at end of file
Modified: incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/site/ReferencedSite.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/site/ReferencedSite.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/site/ReferencedSite.java (original)
+++ incubator/stanbol/trunk/rick/generic/servicesapi/src/main/java/eu/iksproject/rick/servicesapi/site/ReferencedSite.java Sun Dec 5 18:00:18 2010
@@ -8,6 +8,7 @@ import eu.iksproject.rick.servicesapi.mo
import eu.iksproject.rick.servicesapi.model.Sign;
import eu.iksproject.rick.servicesapi.query.Constraint;
import eu.iksproject.rick.servicesapi.query.FieldQuery;
+import eu.iksproject.rick.servicesapi.query.FieldQueryFactory;
import eu.iksproject.rick.servicesapi.query.QueryResultList;
import eu.iksproject.rick.servicesapi.yard.Cache;
@@ -84,5 +85,16 @@ public interface ReferencedSite extends
* @return The {@link FieldMapping} present for this Site.
*/
FieldMapper getFieldMapper();
+ /**
+ * Getter for the QueryFactory implementation preferable used with this Site.
+ * Note that Site MUST support query instances regardless of there specific
+ * implementation. However specific implementations might have performance
+ * advantages for query processing and may be even execution. Therefore
+ * if one creates queries that are specifically executed on this specific
+ * site, that it is best practice to use the instance provided by this
+ * method.
+ * @return The query factory of this site.
+ */
+ FieldQueryFactory getQueryFactory();
}
Propchange: incubator/stanbol/trunk/rick/generic/test/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sun Dec 5 18:00:18 2010
@@ -0,0 +1,7 @@
+target
+
+.settings
+
+.project
+
+.classpath
Added: incubator/stanbol/trunk/rick/indexing/dbPedia/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/indexing/dbPedia/pom.xml?rev=1042390&view=auto
==============================================================================
--- incubator/stanbol/trunk/rick/indexing/dbPedia/pom.xml (added)
+++ incubator/stanbol/trunk/rick/indexing/dbPedia/pom.xml Sun Dec 5 18:00:18 2010
@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ IKS FISE software is licensed under the Apache License, Version 2.0,
+ see http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+ applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the
+ License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+<!--
+ <parent>
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.rick.parent</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+-->
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.rick.indexing.dbPedia</artifactId>
+ <packaging>jar</packaging>
+ <version>0.1-SNAPSHOT</version>
+ <name>IKS RICK Indexing for geonames.org</name>
+ <description>This uses the RDF dump of dbPedia.org to create a Full Yard for dbPedia.org </description>
+ <dependencies>
+ <!-- for logging -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.5.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.5.8</version>
+ <scope>runtime</scope>
+ </dependency>
+ <!-- RICK dependencies -->
+ <dependency>
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.rick.servicesapi</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.rick.core</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.rick.yard.solr</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.rick.core</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>eu.iksproject</groupId>
+ <artifactId>eu.iksproject.rick.indexing.genericRdf</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </dependency>
+ <!-- for the main -->
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.0</version>
+ </dependency>
+ <!-- for testing -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.7</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <archive>
+ <manifest>
+ <mainClass>eu.iksproject.rick.indexing.dbPedia.cli.CommandLineRunner</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
Propchange: incubator/stanbol/trunk/rick/indexing/dbPedia/pom.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java?rev=1042390&view=auto
==============================================================================
--- incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java (added)
+++ incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java Sun Dec 5 18:00:18 2010
@@ -0,0 +1,299 @@
+package eu.iksproject.rick.indexing.dbPedia.cli;
+
+import static eu.iksproject.rick.indexing.rdf.RdfIndexer.*;
+
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.iksproject.rick.indexing.rdf.RdfIndexer;
+import eu.iksproject.rick.servicesapi.defaults.NamespaceEnum;
+import eu.iksproject.rick.servicesapi.model.rdf.RdfResourceEnum;
+import eu.iksproject.rick.servicesapi.yard.YardException;
+import eu.iksproject.rick.yard.solr.impl.SolrYard;
+import eu.iksproject.rick.yard.solr.impl.SolrYardConfig;
+
+public class CommandLineRunner {
+ public static final String[] defaultFieldMappings = new String [] {
+ // --- Define the Languages for all fields ---
+ //NOTE: the leading space is required for the global filter!
+ // --- RDF, RDFS and OWL Mappings ---
+ "rdfs:label", //rdf:label
+ "rdfs:comment",//rdf:comment
+ "rdf:type | d=rick:ref",//The types
+ "owl:sameAs | d=rick:ref",//used by LOD to link to URIs used to identify the same Entity
+ // --- Dublin Core ---
+ "dc:*", //all DC Terms properties
+ "dc-elements:*", //all DC Elements (one could also define the mappings to the DC Terms counterparts here
+ // --- Spatial Things ---
+ "geo:lat | d=xsd:double",
+ "geo:long | d=xsd:double",
+ "geo:alt | d=xsd:int;xsd:float", //also allow floating point if one needs to use fractions of meters
+ // --- Thesaurus (via SKOS) ---
+ //SKOS can be used to define hierarchical terminologies
+ "skos:*",
+ "skos:broader | d=rick:ref",
+ "skos:narrower | d=rick:ref",
+ "skos:related | d=rick:ref",
+ "skos:member | d=rick:ref",
+ "skos:subject | d=rick:ref",
+ "skos:inScheme | d=rick:ref",
+ "skos:hasTopConcept | d=rick:ref",
+ "skos:topConceptOf | d=rick:ref",
+ // --- Social Networks (via foaf) ---
+ "foaf:*", //The Friend of a Friend schema often used to describe social relations between people
+ "foaf:knows | d=rick:ref",
+ "foaf:made | d=rick:ref",
+ "foaf:maker | d=rick:ref",
+ "foaf:member | d=rick:ref",
+ "foaf:homepage | d=xsd:anyURI",
+ "foaf:depiction | d=xsd:anyURI",
+ "foaf:img | d=xsd:anyURI",
+ "foaf:logo | d=xsd:anyURI",
+ "foaf:page | d=xsd:anyURI", //page about the entity
+ // --- dbPedia specific
+ "dbp-ont:*",
+ "dbp-ont:thumbnail | d=xsd:anyURI > foaf:depiction",
+// "dbp-prop:latitude | d=xsd:decimal > geo:lat",
+// "dbp-prop:longitude | d=xsd:decimal > geo:long",
+ "dbp-prop:population | d=xsd:integer",
+ "dbp-prop:website | d=xsd:anyURI > foaf:homepage"
+ };
+ public static final Map<String,Float> fieldBoosts;
+ static {
+ Map<String,Float> boosts = new HashMap<String, Float>();
+ boosts.put(NamespaceEnum.rdfs+"label", 3f);
+ boosts.put(NamespaceEnum.dcTerms+"title", 3f);
+ boosts.put(NamespaceEnum.dcElements+"title", 3f);
+ boosts.put(NamespaceEnum.foaf+"name", 3f);
+ boosts.put(NamespaceEnum.skos+"prefLabel", 3f);
+ boosts.put(NamespaceEnum.skos+"altLabel", 2f);
+ fieldBoosts = Collections.unmodifiableMap(boosts);
+ }
+
+ protected static final Logger log = LoggerFactory.getLogger(CommandLineRunner.class);
+
+ private static final String header;
+ static {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Description:\nThis Utility creates a full Yard for dbPedia.org by using the SolrYard implementation.\n");
+ builder.append("\nParameter:\n");
+ builder.append(" - \"-Xmx\": This implementation does not need much memory. RDF data are loaded into the file based Jena TDB store. Indexing is done in chunks of 1000 (default). In case of OutOfMemory errors you need to increase this value!");
+ builder.append(" - solrServerUri : The URL of the Solr Server used to index the data. Make sure to use the schema.xml as needed by the SolrYard!\n");
+ builder.append(" - dbPediaDumpDir: The relative or absolute path to the Dir with the source RDF files to be used for indexing. You can direcly use the compressed archives. All files in that directory are used to create the index\n");
+ builder.append("\nOptions:\n");
+ header = builder.toString();
+ builder = null;
+ }
+ private static final Options options;
+ static {
+ options = new Options();
+ options.addOption("h", "help", false, "display this help and exit");
+ options.addOption("d", "debug", false, "show debug stacktrace upon error");
+ //options.addOption("yt","yardtype",false, "the type of the yard used as target 'solr' or 'rdf' (default:'solr')");
+ //options.addOption("i","index",true, "Base URI of the used Solr Server used to index the data");
+ options.addOption("n", "name", true, "the id and name used for the Yard (default: 'dbPedia')");
+ options.addOption("m","mappings",true, "file with the fieldMappings used for indexing (this will replace the defaults)");
+ options.addOption("c","chunksize",true, "the number of documents stored in one chunk (default: 1000");
+ options.addOption("s","skipRdf",false, "this options allow to skip the loading of the RDF Data (e.g. if alredy loaded to the Triple Store)");
+ options.addOption("i","incomming",true,"the file with the incomming links for Entities (id tab num, highest num needs to be the first line!)");
+ options.addOption("ri","requiredIncomming",true,"the minimum number of incomming lins for Entities to be indexed");
+ options.addOption("r","resume",true,"resume a previous canceled indexing session (usually used with -s)");
+ }
+ private static final String footer;
+ static {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Default Field Mappings:\n");
+ for(String mapping: defaultFieldMappings){
+ builder.append(String.format("\t%s",mapping));
+ }
+ footer = builder.toString();
+ builder = null;
+ }
+
+ private static float minRequiredRanking;
+
+ public static void main(String[] args) throws IOException, ParseException, YardException {
+ CommandLineParser parser = new PosixParser();
+ CommandLine line = parser.parse(options, args);
+ args = line.getArgs();
+
+ if (line.getArgs().length < 2 || line.hasOption("h")) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(
+ "java -Xmx1024M -jar eu.iksproject.indexing.dbPedia-0.1-SNAPSHOT-jar-with-dependencies.jar [options] solrServerUri dbPediaDataDumpDir",
+ header,
+ options,
+ footer);
+ System.exit(0);
+ }
+ String yardName = line.getOptionValue("n");
+ if(yardName == null){
+ yardName = "dbPedia";
+ }
+ Dictionary<String, Object> indexingConfig = new Hashtable<String, Object>();
+ //first the SolrServer used to store the index
+ URL solrServer = new URL(line.getArgs()[0]);
+ SolrYardConfig yardConfig = new SolrYardConfig(yardName, solrServer);
+ //use the signRank as default for document Boosts
+ yardConfig.setDocumentBoostFieldName(RdfResourceEnum.signRank.getUri());
+ //increase the boost for fields that are usually used as labels
+ yardConfig.setFieldBoosts(fieldBoosts);
+ SolrYard yard = new SolrYard(yardConfig);
+ indexingConfig.put(KEY_YARD, yard);
+ //now the other properties
+ File dataDir = new File(line.getArgs()[1]);
+ if(!dataDir.exists()){
+ log.error("Parsed Data Directory "+dataDir+" does not Exist on the File System");
+ System.exit(0);
+ }
+ if(!dataDir.isDirectory()){
+ log.error("Parsed Data Directory "+dataDir+" exists, but is not a Directory!");
+ System.exit(0);
+ }
+ if(!dataDir.canRead()){
+ log.error("Unable to read Data Directory "+dataDir+"!");
+ System.exit(0);
+ }
+ File[] files = dataDir.listFiles();
+ indexingConfig.put(KEY_RDF_FILES, Arrays.asList(files));
+ indexingConfig.put(KEY_RDF_STORE_DIR, "dbPedia-rdf-data");
+ Integer chunkSize;
+ try {
+ chunkSize = Integer.valueOf(line.getOptionValue("c", "1000"));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Value for option \"chunkSize\" need to be a valid Integer");
+ }
+ if(chunkSize<0){
+ log.warn("Negative number parsed for option \"chunkSize\". Use '1000' as default.");
+ chunkSize = 1000;
+ }
+ if(!line.hasOption("m")){
+ indexingConfig.put(KEY_FIELD_MAPPINGS, Arrays.asList(defaultFieldMappings));
+ } else {
+ File fieldMappingFile = new File(line.getOptionValue("m"));
+ if(fieldMappingFile.exists() && fieldMappingFile.isFile() && fieldMappingFile.canRead()){
+ String[] fieldMappings = IOUtils.toString(new FileInputStream(fieldMappingFile)).split("\n");
+ indexingConfig.put(KEY_FIELD_MAPPINGS, Arrays.asList(fieldMappings));
+ } else {
+ log.error(String.format("Invalid fieldMapping File (exist: %s | isFile: %s | canRead: %s)",
+ fieldMappingFile.exists(),fieldMappingFile.isFile(),fieldMappingFile.canRead()));
+ System.exit(0);
+ }
+
+ }
+ if(line.hasOption("s")){
+ indexingConfig.put(KEY_SKIP_READ, Boolean.TRUE);
+ }
+ indexingConfig.put(KEY_CHUNK_SIZE, chunkSize);
+ indexingConfig.put(KEY_MODEL_NAME, "indexingModel-49e176b9-0138-dd4c-2b87-89af85b89a57");
+ //entityRank related properties
+ if(line.hasOption("i")){
+ File tsvScoreFile = new File(line.getOptionValue("i"));
+ if(tsvScoreFile.exists() && tsvScoreFile.isFile() && tsvScoreFile.canRead()){
+ int minIncommings = -1;
+ try {
+ minIncommings = Integer.parseInt(line.getOptionValue("ri", "-1"));
+ } catch (Exception e) {
+ log.error("Value of option --minIncomming/-mi MUST BE a valid integer");
+ System.exit(0);
+ }
+ Map<String,Float> entityRankings = clacEntityRanks(tsvScoreFile,minIncommings);
+ indexingConfig.put(KEY_ENTITY_RANKINGS, entityRankings);
+ log.info(String.format(" ... set min required score to %s (represents %s incomming links",minRequiredRanking,minIncommings));
+ indexingConfig.put(KEY_REQUIRED_ENTITY_RANKING, minRequiredRanking);
+ } else {
+ log.error(String.format("Parsed File with the incommung links is invalid (esists:%s,isFile:%s,canRead:%s)",
+ tsvScoreFile.exists(), tsvScoreFile.isFile(), tsvScoreFile.canRead()));
+ }
+ } else {
+ if(line.hasOption("ri")){
+ log.warn("Option --requiredIncomming/-ri is only supported of Option --incomming/-i is active!");
+ }
+ }
+ //THis mode uses the id of the entity rnking map as main lookup for
+ //entities to index. This is faster than the usual mode if less than
+ //50% of the entities are indexed!
+ if(line.hasOption("r")){
+ //resume makes only really sense with the RANKING BASED MODE
+ indexingConfig.put(KEY_ENTITY_RANKING_BASED_INDEXING_MODE, Boolean.TRUE);
+ //set the RESUME MODE
+ indexingConfig.put(KEY_RESUME_MODE, Boolean.TRUE);
+ }
+ RdfIndexer indexer = new RdfIndexer(indexingConfig);
+ indexer.index();
+ }
+ @SuppressWarnings("unchecked")
+ private static Map<String,Float> clacEntityRanks(File tsvScoreFile,int minIncommings) throws IOException {
+ TreeMap<String,Float> entityRankings = new TreeMap<String, Float>();
+ final Iterator<String> lines = IOUtils.lineIterator(
+ new FileInputStream(tsvScoreFile), "utf-8");
+ long lineNumber = 0;
+ int maxIncommung = 0;
+ float maxScore = 0;
+ long filtered = 0;
+ log.info(" ... init Entity Ranks based on "+tsvScoreFile);
+ while (lines.hasNext()) {
+ String line = lines.next();
+ String[] parts = line.split("\t");
+ if (parts.length != 2) {
+ log.warn(String.format("skipping line: '%s'", line));
+ }
+ int incomming = Integer.parseInt(parts[1].trim());
+ // take the log to avoid over popular entities to
+ // dominate the results (attenuate the Zipf law of
+ // culturally generated distribution)
+ float score = (float)Math.log1p(incomming);
+ if (lineNumber == 0 && score > 0) {
+ maxIncommung = incomming;
+ maxScore = score;
+ if(minIncommings <= 0){
+ minRequiredRanking = -1f; //deactivate
+ } else {
+ float min = (float)Math.log1p(minIncommings);
+ minRequiredRanking = min/maxScore;
+ if(minRequiredRanking > 1){
+ log.error("Parsed minimum required incomming links is bigger than the highest number of incomming links for any entity!");
+ System.exit(0);
+ }
+ }
+ }
+ score = score/maxScore;
+ if(score > 1){
+ log.error("Found Entity wiht more incomming links than the entity in the first line");
+ log.error("current:"+line);
+ System.exit(0);
+ }
+ if(score >= minRequiredRanking){
+ entityRankings.put(parts[0], score);
+ } else {
+ filtered ++;
+ }
+ lineNumber++;
+ }
+ log.info(String.format(" ... processed %s entities (%s with ranking > required | %s filtered",
+ lineNumber,lineNumber-filtered,filtered));
+ return entityRankings;
+ }
+
+}
Propchange: incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Propchange: incubator/stanbol/trunk/rick/indexing/genericRdf/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sun Dec 5 18:00:18 2010
@@ -0,0 +1,7 @@
+target
+
+.settings
+
+.project
+
+.classpath
Modified: incubator/stanbol/trunk/rick/indexing/genericRdf/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/indexing/genericRdf/pom.xml?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/indexing/genericRdf/pom.xml (original)
+++ incubator/stanbol/trunk/rick/indexing/genericRdf/pom.xml Sun Dec 5 18:00:18 2010
@@ -21,7 +21,7 @@
</parent>
<groupId>eu.iksproject</groupId>
- <artifactId>eu.iksproject.indexing.genericRdf</artifactId>
+ <artifactId>eu.iksproject.rick.indexing.genericRdf</artifactId>
<packaging>jar</packaging>
<version>0.1-SNAPSHOT</version>
<name>IKS RICK Indexing for rdf Resource</name>
Modified: incubator/stanbol/trunk/rick/indexing/genericRdf/src/main/java/eu/iksproject/rick/indexing/rdf/RdfIndexer.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/indexing/genericRdf/src/main/java/eu/iksproject/rick/indexing/rdf/RdfIndexer.java?rev=1042390&r1=1042389&r2=1042390&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/indexing/genericRdf/src/main/java/eu/iksproject/rick/indexing/rdf/RdfIndexer.java (original)
+++ incubator/stanbol/trunk/rick/indexing/genericRdf/src/main/java/eu/iksproject/rick/indexing/rdf/RdfIndexer.java Sun Dec 5 18:00:18 2010
@@ -8,19 +8,63 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Dictionary;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
import java.util.zip.GZIPInputStream;
-import org.apache.clerezza.rdf.core.access.TcProvider;
-import org.apache.clerezza.rdf.jena.tdb.storage.TdbTcProvider;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.openjena.riot.Lang;
+import org.openjena.riot.RiotReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.hp.hpl.jena.datatypes.BaseDatatype;
+import com.hp.hpl.jena.datatypes.DatatypeFormatException;
+import com.hp.hpl.jena.datatypes.RDFDatatype;
+import com.hp.hpl.jena.datatypes.xsd.XSDDateTime;
+import com.hp.hpl.jena.datatypes.xsd.XSDDuration;
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.graph.Triple;
+import com.hp.hpl.jena.graph.impl.LiteralLabel;
+import com.hp.hpl.jena.query.Query;
+import com.hp.hpl.jena.query.QueryExecutionFactory;
+import com.hp.hpl.jena.query.QueryFactory;
+import com.hp.hpl.jena.query.QuerySolution;
+import com.hp.hpl.jena.query.ResultSet;
+import com.hp.hpl.jena.query.Syntax;
+import com.hp.hpl.jena.rdf.model.Literal;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
+import com.hp.hpl.jena.rdf.model.RDFNode;
+import com.hp.hpl.jena.tdb.TDBFactory;
+import com.hp.hpl.jena.tdb.TDBLoader;
+import com.hp.hpl.jena.tdb.base.file.Location;
+import com.hp.hpl.jena.tdb.store.DatasetGraphTDB;
+import com.hp.hpl.jena.tdb.store.bulkloader.BulkLoader;
+import com.hp.hpl.jena.tdb.store.bulkloader.Destination;
+import com.hp.hpl.jena.tdb.store.bulkloader.LoadMonitor;
+import com.hp.hpl.jena.tdb.store.bulkloader.LoaderNodeTupleTable;
+import com.hp.hpl.jena.util.iterator.ExtendedIterator;
+
+import eu.iksproject.rick.core.mapping.DefaultFieldMapperImpl;
+import eu.iksproject.rick.core.mapping.FieldMappingUtils;
+import eu.iksproject.rick.core.mapping.ValueConverterFactory;
+import eu.iksproject.rick.core.site.CacheUtils;
import eu.iksproject.rick.core.utils.ModelUtils;
import eu.iksproject.rick.servicesapi.mapping.FieldMapper;
+import eu.iksproject.rick.servicesapi.mapping.FieldMapping;
+import eu.iksproject.rick.servicesapi.model.Representation;
import eu.iksproject.rick.servicesapi.model.ValueFactory;
+import eu.iksproject.rick.servicesapi.model.rdf.RdfResourceEnum;
import eu.iksproject.rick.servicesapi.yard.Yard;
+import eu.iksproject.rick.servicesapi.yard.YardException;
/**
* This Class indexes Entities based on Information provided in a RDF Graph
@@ -36,16 +80,79 @@ import eu.iksproject.rick.servicesapi.ya
* this means using the {@link FieldMapper}
* <li> Support for filtering Entities based on rdf:type (will be in future
* version supported by the RICK Representation Mapping Infrastructure)
- * <li> any more?
+ * <li> Entity Rank support: It is not feasible to calculate the rankings in a
+ * generic fashion. Therefore this implementation supports two different
+ * ways. First it uses {@link RdfResourceEnum#signRank} relations in the
+ * parsed RDF Data and secondly it allows to provide entity rank information
+ * by parsing a map with the id as key and the rank as value by using the
+ * {@link #KEY_ENTITY_RANKINGS} in the configuration. Mappings parsed by the
+ * map will override mappings in the RDF data.
* <li>
* <li>
* <li>
* </ul>
- *
+ * Implementation Notes:<ul>
+ * <li> The Idea was to use the Clerezza TdbTcProvider and the JenaParserProvider
+ * outside of an OSGI Environment and than use the Clerezza Yard to
+ * access Resources to be indexed directly as Representations to perform
+ * the field mappings by using the parsed {@link FieldMapper}.
+ * <li> I am a bit worried with the ParsingProvide} because the interface
+ * allows no streaming. One needs to check if this causes problems for
+ * very big RDF files.<br>
+ * After looking into the source code of Clerezza, I am even more worried!
+ * The Parser creates an SimpleMGraph and wrapped it with an
+ * Jena Adapter. Than all Triples are parsed into memory. So I do not only
+ * have the triples in memory but also the instances of the jena adapter.
+ * <li> Maybe it is better to directly use Jena, because Clerezza does not support
+ * COUNT(..) for SPARQL queries. Without that feature it would be hard to
+ * support page ranks!<br>
+ * Maybe I can use both, because using a Clerezza wrapper over Jena would
+ * have the advantage to directly use the ClerezzaYard for creating
+ * RDF Representations for resource to be indexed!<br>
+ * Parsing and especially SPARQL queries would be better to do directly
+ * via the Jena API!<p>
+ * This should be no problem, because indexing is a read only operation!
+ * <li> However based on this findings I plan now to implement the loading of the
+ * RDF data to directly use the Jena TDB API because Clerezza seams not to
+ * be designed to handle the loading of RDF datasets that can not be
+ * kept in memory.<br>
+ * </ul>
+ *
* @author Rupert Westenthaler
+ * @author ogrisel (as parts of that code is taken from iks-autotagger)
*
*/
public class RdfIndexer {
+ /**
+ * The indexing mode defines if the RDF data are appended to existing
+ * {@link Representation}s in the target {@link Yard} or if {@link Representation}
+ * are replaced with RDF data used for indexing!
+ * @author Rupert Westenthaler
+ *
+ */
+ public static enum IndexingMode{ REPLACE, APPEND }
+
+ public static final String RDF_XML = "application/rdf+xml";
+ public static final String TURTLE = "text/turtle";
+ public static final String X_TURTLE = "application/x-turtle";
+ public static final String N_TRIPLE = "text/rdf+nt";
+ public static final String N3 = "text/rdf+n3";
+ public static final String RDF_JSON = "application/rdf+json";
+ //both html and xhtml can be rdf formats with RDFa
+ public static final String XHTML = "application/xhtml+xml";
+ public static final String HTML = "text/html";
+
+ protected static final String resourceQuery;
+ static {
+ StringBuilder cqb = new StringBuilder();
+ cqb.append("SELECT ?field ?value ");//, count(distinct ?incoming) AS ?count ");
+ cqb.append("{ ");
+ cqb.append(" <%s> ?field ?value .");
+// cqb.append(" OPTIONAL {?incoming ?relationship <%s> . } .");
+ cqb.append("} ");
+ resourceQuery = cqb.toString();
+ }
+
Logger log = LoggerFactory.getLogger(RdfIndexer.class);
/**
* Key used to parse the Yard used for indexing
@@ -61,7 +168,8 @@ public class RdfIndexer {
public static final String KEY_RDF_FILES = "eu.iksproject.rick.indexing.rdf.rdfFiles";
/**
* Key used to configure the fieldMappings used to determine what properties
- * are indexed for entities
+ * are indexed for entities. Values must implement {@link Iterable} and the
+ * {@link Object#toString()} is used to parse the different mappings.
*/
public static final String KEY_FIELD_MAPPINGS = "eu.iksproject.rick.indexing.rdf.fieldMappings";
@@ -70,42 +178,124 @@ public class RdfIndexer {
* indexing process. This data might be reused when resuming an indexing
* process.
*/
- public static final String KEY_INDEXING_DIR = "eu.iksproject.rick.indexing.rdf.indexingDir";
+ public static final String KEY_RDF_STORE_DIR = "eu.iksproject.rick.indexing.rdf.indexingDir";
/**
* Key used to configure the name of the model used to store the parsed
* RDF data before the indexing process. Parsing this name can be used to
* resume indexing based on previously parsed RDF data.
*/
public static final String KEY_MODEL_NAME = "eu.iksproject.rick.indexing.rdf.modelName";
+ /**
+ * Key used to parse the Iterable over all the rdf:types to be indexed. If
+ * not parsed or set to <code>null</code> or an empty list, than all Resources are
+ * accepted!
+ * The {@link Object#toString()} method is used on elements to get the actual type!
+ */
+ public static final String KEY_RDF_TYPES = "eu.iksproject.rick.indexing.rdf.indexedTypes";
+ /**
+ * Key used to parse the indexing mode. Values should be of instance {@link IndexingMode}
+ * or the {@link Object#toString()} value should be a member of this enum!
+ */
+ public static final String KEY_INDEXING_MODE = "eu.iksproject.rick.indexing.rdf.indexingMode";
+ /**
+ * If <code>true</code> than no RDF data are loaded. Instead it is assumed, that
+ * the Graph of the parsed {@link #KEY_MODEL_NAME} already contains all the needed
+ * data!<p>
+ * This can be useful if one first wants to index rdf:type A and than rdf:type B
+ * based on the same set of data
+ */
+ public static final String KEY_SKIP_READ = "eu.iksproject.rick.indexing.rdf.skipRead";
+ /**
+ * The number of {@link Representation}s stored at once in the SolrYard!
+ */
+ public static final String KEY_CHUNK_SIZE = "eu.iksproject.rick.indexing.rdf.chunkSize";
+ /**
+ * Can be used to parse a map with {@link String} entity id, {@link Float} rank
+ * for entities.<p>
+ * Such values are added to Representations for the {@link RdfResourceEnum#signRank}
+ * field.
+ */
+ public static final String KEY_ENTITY_RANKINGS = "eu.iksproject.rick.indexing.rdf.entityRankings";
+ /**
+ * Can be used to activate ignoring of Entities without a page rank
+ */
+ public static final String KEY_IGNORE_ENTITIES_WITHOUT_ENTITY_RANKING = "eu.iksproject.rick.indexing.rdf.ignoreEntitiesWithoutRankings";
+ /**
+ * If set to a value >= 0 this is used to exclude Entities with a lower or
+ * missing entity rank
+ */
+ public static final String KEY_REQUIRED_ENTITY_RANKING = "eu.iksproject.rick.indexing.rdf.requiredRanking";
+ /**
+ * The rank for entities with a missing rank. This takes only effect if
+ * {@link #KEY_IGNORE_ENTITIES_WITHOUT_ENTITY_RANKING} is set to <code>false</code>
+ * (the default)
+ */
+ public static final String KEY_DEFAULT_ENTITY_RANKING = "eu.iksproject.rick.indexing.rdf.defaultRanking";
+ /**
+ * Expert only: This allows to enable indexing based on the keys in the map parsed with the
+ * entity rankings. This will only index entities that are keys in that map.
+ * If no Map is parsed by {@link #KEY_ENTITY_RANKINGS}, than activating this mode
+ * will not be successful and a warning will be written.<p>
+ * This mode is about 50% slower than the usual indexing mode. Therefore this
+ * mode makes only sense id less than 50% of the entities are indexed.
+ */
+ public static final String KEY_ENTITY_RANKING_BASED_INDEXING_MODE = "eu.iksproject.rick.indexing.rdf.rankingBasedIndexingMode";
+ /**
+ * The resume Mode first checks if a Resource is already present in the parsed
+ * Yard. If this is the case, than the representation is not indexes again.<p>
+ * This mode is intended to resume indexing after stopping a previous call before
+ * finished. The default value = false.
+ */
+ public static final String KEY_RESUME_MODE = "eu.iksproject.rick.indexing.rdf.resumeMode";
+ private final IndexingMode indexingMode;
private final Yard yard;
private final ValueFactory vf;
private final List<File> rdfFiles;
private final File indexingDir;
private final String modelName;
+// private final ParsingProvider parser = new JenaParserProvider();
+ //private final WeightedTcProvider provider;
+ private final FieldMapper mapper;
+ private final Set<String> types;
+ //private MGraph indexingGraph;
+ private final DatasetGraphTDB indexingDataset;
+ private final boolean skipRead;
+ private Location modelLocation;
+ private int indexingChunkSize = 1000;
+
+ //vars for entity rankings
+ private Map<String,Float> entityRankings = null;
+ private boolean ignoreEntitiesWithoutRank = false;
+ private float defaultEntityRanking = -1;
+ private String entityRankingField = RdfResourceEnum.signRank.getUri();
+ private float minimumRequiredEntityRanking = -1;
+ private boolean rankingMode;
+ private boolean resumeMode;
+
public RdfIndexer(Dictionary<String, Object> config){
this.yard = (Yard)config.get(KEY_YARD);
if(yard == null){
throw new IllegalArgumentException("Parsed config MUST CONTAIN a Yard. Use the key "+KEY_YARD+" to parse the YardInstance used to store the geonames.org index!");
} else {
- log.info(String.format("Using Yard %s (id=%s) to index geonames.org",
+ log.info(String.format("Using Yard %s (id=%s) to index parsed RDF data",
yard.getName(),yard.getId()));
}
this.vf = yard.getValueFactory();
Object rdfFiles = config.get(KEY_RDF_FILES);
if(rdfFiles instanceof Iterable<?>){
this.rdfFiles = new ArrayList<File>();
- for(Object value : (Iterable[])rdfFiles){
+ for(Object value : (Iterable<?>)rdfFiles){
this.rdfFiles.add(checkFile(value.toString()));
}
} else {
this.rdfFiles = Collections.singletonList(checkFile(rdfFiles.toString()));
}
- Object indexingDir = config.get(KEY_INDEXING_DIR);
+ Object indexingDir = config.get(KEY_RDF_STORE_DIR);
if(indexingDir == null){
indexingDir = "indexingData";
- config.put(KEY_INDEXING_DIR, indexingDir);
+ config.put(KEY_RDF_STORE_DIR, indexingDir);
}
this.indexingDir = checkFile(indexingDir.toString(), false, true);
Object modelName = config.get(KEY_MODEL_NAME);
@@ -114,11 +304,347 @@ public class RdfIndexer {
config.put(KEY_MODEL_NAME, modelName);
}
this.modelName = modelName.toString();
+ //init the types!
+ Iterable<?> types = (Iterable<?>)config.get(KEY_RDF_TYPES);
+ if(types != null){
+ Set<String> typeSet = new HashSet<String>();
+ for(Object type : types){
+ if(type != null){
+ typeSet.add(type.toString());
+ log.info(" - adding Resoures with rdf:type "+type);
+ }
+ }
+ if(typeSet.isEmpty()){
+ log.info(" - adding all Types (no rdf:type based restriction for RDF Reseource present)");
+ this.types = null;
+ } else {
+ this.types = typeSet;
+ }
+ } else{
+ log.info(" - adding all Types (no rdf:type based restriction for RDF Reseource present)");
+ this.types = null; //null or an iterable with one or more elements!
+ }
+ //init the indexing mode
+ Object indexingMode = config.get(KEY_INDEXING_MODE);
+ if(indexingMode == null){
+ this.indexingMode = IndexingMode.REPLACE; //default to replace
+ } else if(indexingMode instanceof IndexingMode){
+ this.indexingMode = (IndexingMode)indexingMode;
+ } else {
+ try {
+ this.indexingMode = IndexingMode.valueOf(indexingMode.toString());
+ }catch (IllegalArgumentException e) {
+ //catch and re-throw with a better message!
+ throw new IllegalArgumentException(
+ String.format("Values of KEY \"%s\" MUST BE of Type %s or the toString() value MUST BE a member of this Enumeration. If the Key is missing %s is used!",
+ KEY_INDEXING_MODE,IndexingMode.class,IndexingMode.REPLACE),e);
+ }
+ }
+ //init the fieldMapper
+ Iterable<?> mappings = (Iterable<?>)config.get(KEY_FIELD_MAPPINGS);
+ List<FieldMapping> fieldMappings;
+ if(mappings != null){
+ fieldMappings = new ArrayList<FieldMapping>();
+ for(Object mappingString : mappings){
+ if(mappingString != null){
+ FieldMapping fieldMapping = FieldMappingUtils.parseFieldMapping(mappingString.toString());
+ if(fieldMapping != null){
+ fieldMappings.add(fieldMapping);
+ }
+ }
+ }
+ if(!fieldMappings.isEmpty()){
+ this.mapper = new DefaultFieldMapperImpl(ValueConverterFactory.getInstance(vf));
+ for(FieldMapping mapping : fieldMappings){
+ mapper.addMapping(mapping);
+ }
+ //we need to add a mapping for the field rankings (if a mapper is present)
+ mapper.addMapping(new FieldMapping(this.entityRankingField));
+ } else {
+ this.mapper = null;
+ }
+ } else {
+ this.mapper = null;
+ }
+ File modelDir = new File(this.indexingDir,this.modelName);
+ if(!modelDir.exists()){
+ modelDir.mkdir();
+ } else if(!modelDir.isDirectory()){
+ throw new IllegalStateException(String.format("A directory for %s already exists but is not a directory!",modelDir.getAbsoluteFile()));
+ } //else exists and is a dir -> nothing to do
+ Object skipRead = config.get(KEY_SKIP_READ);
+ if(skipRead != null){
+ if(skipRead instanceof Boolean){
+ this.skipRead = ((Boolean)skipRead).booleanValue();
+ } else {
+ this.skipRead = Boolean.parseBoolean(skipRead.toString());
+ }
+ } else {
+ this.skipRead = false;
+ }
+ Integer chunkSize = (Integer)config.get(KEY_CHUNK_SIZE);
+ if(chunkSize != null && chunkSize>0){
+ this.indexingChunkSize = chunkSize;
+ } //else use default value of 1000
+
+ this.modelLocation = new Location(modelDir.getAbsolutePath());
+ this.indexingDataset = TDBFactory.createDatasetGraph(modelLocation) ;
+ //this.provider = new IndexingModelProvider(this.indexingDir);
+
+ //init entity Ranking
+ try{
+ this.entityRankings = (Map<String,Float>)config.get(KEY_ENTITY_RANKINGS);
+ }catch (RuntimeException e) {
+ log.error("Parsed Entity Rankings MUST use the form Map<String,Float>");
+ System.exit(0);
+ }
+ Object ignore = config.get(KEY_IGNORE_ENTITIES_WITHOUT_ENTITY_RANKING);
+ if(ignore != null){
+ if(ignore instanceof Boolean){
+ this.ignoreEntitiesWithoutRank = (Boolean)ignore;
+ } else {
+ this.ignoreEntitiesWithoutRank = Boolean.parseBoolean(ignore.toString());
+ }
+ }
+ Object defaultRankingObject = config.get(KEY_DEFAULT_ENTITY_RANKING);
+ if(defaultRankingObject != null){
+ float defaultranking = -1;
+ if(defaultRankingObject instanceof Float){
+ defaultranking = (Float)defaultRankingObject;
+ } else {
+ try {
+ defaultranking = Float.parseFloat(defaultRankingObject.toString());
+ } catch (Exception e) {
+ log.error("Unable to parse Float value for the Default Entity Ranking from the value parsed for the KEY_DEFAULT_ENTITY_RANKING key (value: "+defaultRankingObject+")");
+ System.exit(0);
+ }
+ }
+ this.defaultEntityRanking = defaultranking;
+ }
+ Object minimumRequiredRankingObject = config.get(KEY_REQUIRED_ENTITY_RANKING);
+ if(minimumRequiredRankingObject != null){
+ float minRanking = -1;
+ if(minimumRequiredRankingObject instanceof Float){
+ minRanking = (Float)minimumRequiredRankingObject;
+ } else {
+ try {
+ minRanking = Float.parseFloat(minimumRequiredRankingObject.toString());
+ } catch (Exception e) {
+ log.error("Unable to parse Float value for the Minimum Required Entity Ranking from the value parsed for the KEY_DEFAULT_ENTITY_RANKING key (value: "+minimumRequiredRankingObject+")");
+ System.exit(0);
+ }
+ }
+ if(minRanking>=0){ //setting a valid required ranking automatically
+ //means that entities without a rank should be ignored!
+ this.ignoreEntitiesWithoutRank = true;
+ }
+ this.minimumRequiredEntityRanking = minRanking;
+ }
+ Object rankingMode = config.get(KEY_ENTITY_RANKING_BASED_INDEXING_MODE);
+ if(rankingMode != null){
+ if(rankingMode instanceof Boolean){
+ this.rankingMode = (Boolean)rankingMode;
+ } else {
+ this.rankingMode = Boolean.parseBoolean(rankingMode.toString());
+ }
+ }
+ if(this.rankingMode && this.entityRankings == null){
+ log.warn("The Entity Ranking based Indexing Mode can not be activated if no EntityRankings are parsed! -> deactivate Ranking Mode (intertes over all Resources in the RDF Data)");
+ this.rankingMode = false;
+ }
+ Object resumeMode = config.get(KEY_RESUME_MODE);
+ if(resumeMode != null) {
+ if(resumeMode instanceof Boolean){
+ this.resumeMode = (Boolean)resumeMode;
+ } else {
+ this.resumeMode = Boolean.parseBoolean(resumeMode.toString());
+ }
+ } else {
+ this.resumeMode = false;
+ }
+ }
+ public void index() throws YardException{
+ log.info("initialize ...");
+ if(!skipRead){
+ loadRdfFiles();
+ } else {
+ log.info(" ... skiping loading of RDF data");
+ }
+ if(rankingMode){
+ indexRanked();
+ } else {
+ indexResources();
+ }
+ writeCacheBaseConfiguration();
+ }
+ /**
+ * This Method is used to process the RDF Data if all Resource can be indexed,
+ * because it provides the best performance. Mainly because it reads everything
+ * from a single stream and therefore gives the OS the best opportunities to
+ * optimise file access.
+ * @throws YardException
+ */
+ private void indexResources() throws YardException{
+ StringBuilder qb = new StringBuilder();
+ /*
+ * NOTES:
+ * - selects all triples form the TDB store
+ * - GROUP BY is not needed because iteration order is anyway based
+ * on the resource (TDB uses internally a tree structure based on the
+ * subject
+ * - This is about tree times faster than first selecting all resource
+ * and than filtering for all triples with the resource. Mainly because
+ * hard disc access is much more efficient.
+ */
+ qb.append("SELECT ?resource ?field ?value");
+ qb.append("{ ");
+ qb.append(" ?resource ?field ?value . ");
+// qb.append(" OPTIONAL { ?incoming ?relationship ?resource . } . ");
+ //qb.append(" FILTER ( isURI(?resource) ) . ");
+ qb.append("} ");
+// qb.append("GROUP BY ?resource "); //needed because the count counts by GROUP BY
+// qb.append("ORDER BY DESC ( ?count ) ");
+// qb.append(String.format("OFFSET 0 LIMIT %d", maxTopResources));
+ Query q = QueryFactory.create(qb.toString(), Syntax.syntaxARQ);
+ final ResultSet resultSet = QueryExecutionFactory.create(q, indexingDataset.toDataset()).execSelect();
+ long count = 0;
+ long indexed = 0;
+ long lastIndexed = 0;
+ long stdCount = 0;
+ long indexedStdCount = 0;
+ long repStdCount = 0;;
+ long start = System.currentTimeMillis();
+ long startCurrent = start;
+ String current = null;
+ Representation source = null;
+ while(resultSet.hasNext()){
+ stdCount++;
+ repStdCount++;
+ QuerySolution solution =resultSet.next();
+ RDFNode subject = solution.get("resource");
+ if(subject.isURIResource()){
+ String resource = subject.asResource().toString();
+ if(!resource.equals(current)){ //start of next resource -> index current
+ count++;
+ if(count%10000==0){
+ long thisOne = System.currentTimeMillis()-startCurrent;
+ long all = System.currentTimeMillis()-start;
+ log.info(String.format("processed %d resources (%dall %dlast indexed) in %dms (%sms/last | avg: %sms/indexed) std/resource (%s indexed| %s non indexed)",
+ count,indexed,indexed-lastIndexed,thisOne,(float)thisOne/(indexed-lastIndexed),(float)all/indexed,(float)indexedStdCount/indexed,((float)stdCount-indexedStdCount)/(count-indexed)));
+ startCurrent = System.currentTimeMillis();
+ lastIndexed = indexed;
+ }
+ if(source != null){
+ if(processRanking(source)){
+ if(resumeMode && yard.isRepresentation(source.getId())){ //resume mode check
+ //log.info("S<source Resource:\n"+ModelUtils.getRepresentationInfo(source));
+ indexed++;
+ indexedStdCount=indexedStdCount+repStdCount;
+ storeRepresentation(source);
+ //here we need todo the indexing!
+ } //else already indexed -> nothing to do
+ } // else rankging to low -> do not index
+ } //else the first item to index -> ignore
+ //init next resource
+ source = vf.createRepresentation(resource);
+ current = resource;
+ repStdCount = 0;
+ }
+ RDFNode fieldNode = solution.get("field");
+ if(fieldNode.isURIResource()){
+ String field = fieldNode.asResource().getURI();
+ RDFNode value = solution.get("value");
+ if(value.isURIResource()){
+ source.addReference(field, value.asResource().getURI());
+ } else if(value.isLiteral()){
+ Literal literal = value.asLiteral();
+ if(literal.getDatatype() != null){
+ Object literalValue;
+ try {
+ literalValue = literal.getValue();
+ } catch (DatatypeFormatException e) {
+ log.warn(" Unable to convert "+literal.getLexicalForm()+" to "+literal.getDatatype()+"-> use lecicalForm");
+ literalValue = literal.getLexicalForm();
+ }
+ if(literalValue instanceof BaseDatatype.TypedValue){
+ source.add(field, ((BaseDatatype.TypedValue)literalValue).lexicalValue);
+ } else if(literalValue instanceof XSDDateTime) {
+ source.add(field, ((XSDDateTime)literalValue).asCalendar().getTime()); //Rick uses the time
+ } else if(literalValue instanceof XSDDuration) {
+ source.add(field, literalValue.toString());
+ } else {
+ source.add(field, literalValue);
+ }
+ } else {
+ String lang = literal.getLanguage();
+ if(lang != null && lang.isEmpty()){
+ lang = null;
+ }
+ source.addNaturalText(field, literal.getLexicalForm(),lang);
+ }
+ }
+ }
+ } else {
+ log.warn(String.format("Current Subject %s is not a URI Resource -> ignored",subject));
+ }
+ } //end while
+ long end = System.currentTimeMillis();
+ log.info(String.format("%d in %dms (%sms/item | %sstd/resource)",count,end-start,""+((float)end - start)/count,""+(float)stdCount/count));
+ }
+
+
+ private boolean processRanking(Representation source) {
+ Float ranking = entityRankings == null ? null :entityRankings.get(source.getId());
+ //ignore values lower than 0
+ if(ranking != null){
+ if(ranking < 0){
+ ranking = null;
+ }
+ }
+ if(ranking != null && ranking > 1){
+ log.warn("Parse Ranking Map contains Entity Ranking > 1 (ranking="+ranking+") for Entity "+source.getId()+" -> use 1.0 as Ranking!");
+ ranking = 1f;
+ }
+ if(ranking == null){
+ for(Iterator<Object> values =source.get(entityRankingField);values.hasNext() && ranking == null;){
+ Object value = values.next();
+ if(value instanceof Float){
+ ranking = (Float) value;
+ } else {
+ try {
+ ranking = Float.parseFloat(value.toString());
+ } catch (NumberFormatException e) {
+ log.warn(String.format("Unable to parse the Entity Ranking from field %s=%s[type=%s] -> The Document Boost MUST BE a Float value!",entityRankingField,value,value.getClass()));
+ }
+ }
+ }
+ } else {
+ source.set(entityRankingField, ranking);
+ }
+ if(ranking != null && ranking > 1){
+ log.warn("Parse RDF data include a entity ranking > 1 (ranking="+ranking+") for Entity "+source.getId()+" and Field "+entityRankingField+"-> use 1.0 as Ranking!");
+ ranking = 1f;
+ }
+ if(ranking == null && this.defaultEntityRanking >= 0){
+ //set to default
+ ranking = defaultEntityRanking;
+ source.set(entityRankingField, ranking);
+ }
+ if(ranking == null){
+ return !ignoreEntitiesWithoutRank; //return false to ignore
+ } else {
+ return ranking > minimumRequiredEntityRanking;
+ }
+
}
private File checkFile(String value) {
return checkFile(value,true,false);
}
private File checkFile(String value,boolean file, boolean create) {
+ if(value.startsWith(File.pathSeparator)){
+ //remove leading path separators!
+ value = value.substring(File.pathSeparator.length());
+ }
File testFile = new File(value.toString());
if(!testFile.exists()){
@@ -135,7 +661,7 @@ public class RdfIndexer {
}
}
} else { //not found
- throw new IllegalStateException("File "+value+" does not exist!");
+ throw new IllegalStateException("File "+testFile.getAbsolutePath()+" does not exist!");
}
}
if(file && !testFile.isFile()){
@@ -151,70 +677,314 @@ public class RdfIndexer {
}
private void loadRdfFiles(){
- TcProvider provider = new IndexingModelProvider(indexingDir);
-
+ //TcProvider provider = new IndexingModelProvider(indexingDir);
+ long start=System.currentTimeMillis();
+ log.info(String.format("Loding RDF %d File%s ...",rdfFiles.size(),rdfFiles.size()>1?"s":""));
for (File modelFile : rdfFiles) {
- System.out.printf("loading '%s' into model '%s'...", modelFile,
- modelName);
- InputStream is;
- try {
- is = new FileInputStream(modelFile);
- } catch (FileNotFoundException e) {
- //during init it is checked that files exists and are files and there is read access
- //so this can only happen if someone deletes the file inbetween
- throw new IllegalStateException(e);
- }
+ long startFile = System.currentTimeMillis();
+ log.info(String.format(" > loading '%s' into model '%s'...", modelFile, modelName));
String name = modelFile.getName();
- if (name.endsWith(".gz")) {
- try {
- is = new GZIPInputStream(is);
+ if(name.endsWith(".zip")){
+ log.info(" - processing Zip-Archive Entries:");
+ try {
+ ZipFile zipArchive = new ZipFile(modelFile);
+ Enumeration<ZipArchiveEntry> entries = zipArchive.getEntries();
+ while(entries.hasMoreElements()){
+ ZipArchiveEntry entry = entries.nextElement();
+ if(!entry.isDirectory()){
+ String entryName = entry.getName();
+ log.info(String.format(" o entry '%s' into model '%s'...", entryName, modelName));
+ importRdfData(zipArchive.getInputStream(entry), entryName);
+ }
+ }
} catch (IOException e) {
- //during init it is checked that files exists and are files and there is read access
- //so this can only happen if someone deletes the file inbetween
throw new IllegalStateException(e);
}
- name = name.replaceFirst("\\.gz$", "");
- } else if (name.endsWith(".bz2")) {
- try {
- is = new BZip2CompressorInputStream(is);
- } catch (IOException e) {
- //during init it is checked that files exists and are files and there is read access
- //so this can only happen if someone deletes the file inbetween
- throw new IllegalStateException(e);
- }
- name = name.replaceFirst("\\.bz2$", "");
+ } else {
+ InputStream is;
+ try {
+ is = new FileInputStream(modelFile);
+ } catch (FileNotFoundException e) {
+ //during init it is checked that files exists and are files and there is read access
+ //so this can only happen if someone deletes the file inbetween
+ throw new IllegalStateException(e);
+ }
+ importRdfData(is, name);
}
- String format = null;
- if (name.endsWith(".nt")) {
- format = "N-TRIPLE";
- } else if (name.endsWith(".n3")) {
- format = "N3";
- } // XML is the default format
-
-// model.read(is, null, format);
-// System.out.println(" done");
+ //add the parsed Triples to the indexing graph!
+ //QUESTION: Does that load the whole file into memory?
+// indexingGraph.addAll(parser.parse(is, format, null));
+ log.info(String.format(" - completed in %d seconds", (System.currentTimeMillis()-startFile)/1000));
}
-
+ log.info(String.format(" ... %d files imported in %d seconds", rdfFiles.size(),(System.currentTimeMillis()-start)/1000));
+ }
+ /**
+ * This method imports the data from an input stream. The name is used to
+ * guess the RDF format used. The stream may be come directly form a file,
+ * an archive, an URL or an entry in an ZIP file
+ * @param is
+ * @param name
+ */
+ private void importRdfData(InputStream is, String name) {
+ if (name.endsWith(".gz")) {
+ try {
+ is = new GZIPInputStream(is);
+ } catch (IOException e) {
+ //during init it is checked that files exists and are files and there is read access
+ //so this can only happen if someone deletes the file inbetween
+ throw new IllegalStateException(e);
+ }
+ name = name.replaceFirst("\\.gz$", "");
+ log.info(" - from GZIP Archive");
+ } else if (name.endsWith(".bz2")) {
+ try {
+ is = new BZip2CompressorInputStream(is);
+ } catch (IOException e) {
+ //during init it is checked that files exists and are files and there is read access
+ //so this can only happen if someone deletes the file inbetween
+ throw new IllegalStateException(e);
+ }
+ name = name.replaceFirst("\\.bz2$", "");
+ log.info(" - from BZip2 Archive");
+ }//TODO: No Zip Files inside Zip Files supported :o( ^^
+ Lang format = Lang.guess(name);
+// if (name.endsWith(".nt")) {
+// format = Lang.NTRIPLES;
+// } else if (name.endsWith(".n3")) {
+// format = Lang.N3;
+// } else {// XML is the default format
+// format = Lang.RDFXML;
+// }
+ //For N-Triple we can use the TDBLoader
+ if(format == Lang.NTRIPLES){
+ TDBLoader.load(indexingDataset, is,true);
+ } else if(format != Lang.RDFXML){
+ //use RIOT to parse the format but with a special configuration
+ //RiotReader!
+ TDBLoader loader = new TDBLoader() ;
+ loader.setShowProgress(true);
+ Destination<Triple> dest = createDestination();
+ dest.start() ;
+ RiotReader.parseTriples(is, format, null, dest) ;
+ dest.finish() ;
+ } else { //RDFXML
+ //in that case we need to use ARP
+ Model model = ModelFactory.createModelForGraph(indexingDataset.getDefaultGraph());
+ model.read(is, null);
+ }
+ }
+ /**
+ * Creates a triple destination for the default dataset of the
+ * {@link #indexingDataset}.
+ * This code is based on how Destinations are created in the {@link BulkLoader},
+ * implementation. Note that
+ * {@link BulkLoader#loadDefaultGraph(DatasetGraphTDB, InputStream, boolean)}
+ * can not be used for formats other than {@link Lang#NTRIPLES} because it
+ * hard codes this format for loading data form the parsed InputStream.
+ * @return the destination!
+ */
+ private Destination<Triple> createDestination() {
+ LoadMonitor monitor = new LoadMonitor(indexingDataset, log, "triples",50000,100000);
+ final LoaderNodeTupleTable loaderTriples = new LoaderNodeTupleTable(indexingDataset.getTripleTable().getNodeTupleTable(), "triples", monitor) ;
+
+ Destination<Triple> sink = new Destination<Triple>() {
+ long count = 0 ;
+ final public void start()
+ {
+ loaderTriples.loadStart() ;
+ loaderTriples.loadDataStart() ;
+ }
+ final public void send(Triple triple)
+ {
+ loaderTriples.load(triple.getSubject(), triple.getPredicate(), triple.getObject()) ;
+ count++ ;
+ }
+
+ final public void flush() { }
+ public void close() { }
+
+ final public void finish()
+ {
+ loaderTriples.loadDataFinish() ;
+ loaderTriples.loadIndexStart() ;
+ loaderTriples.loadIndexFinish() ;
+ loaderTriples.loadFinish() ;
+ }
+ } ;
+ return sink ;
}
- public static void main(String[] args) {
- System.out.println(Integer.MAX_VALUE);
+
+
+
+ /**
+ * The List used to cache up to {@link #indexingChunkSize} Representations
+ * before they are stored in the Yard.
+ */
+ private List<Representation> chunkCache = new ArrayList<Representation>(this.indexingChunkSize);
+ private void storeRepresentation(Representation source) throws YardException{
+ if(source != null){
+ chunkCache.add(
+ mapper ==null?source: //if no mappings -> store the source
+ //else process the field mappings
+ mapper.applyMappings(source,vf.createRepresentation(source.getId())));
+ }
+ if(chunkCache.size()>=indexingChunkSize){
+ yard.store(chunkCache);
+ chunkCache.clear();
+ }
}
/**
- * Simple wrapper to use {@link TdbTcProvider} without an OSGI Environment
- * @author Rupert Westenthaler
- *
+ * As the last step we need to create the baseMappings configuration
+ * needed to used the Index as RICK full cache!
+ * @throws YardException would be really bad if after successfully indexing
+ * about 8 millions of documents we get an error from the yard at the
+ * last possible opportunity :(
*/
- protected static final class IndexingModelProvider extends TdbTcProvider {
-
- protected IndexingModelProvider(File baseDir){
- super();
- activate(null);
+ private void writeCacheBaseConfiguration() throws YardException {
+ log.info("Write BaseMappings for geonames.org Cache");
+ if(mapper != null){
+ CacheUtils.storeBaseMappingsConfiguration(yard, mapper);
}
-
- public void close(){
- deactivate(null);
+ log.info(" < completed");
+ }
+//------------------------------------------------------------------------------
+// Other implemented variants with less performance than indexResource3!
+//------------------------------------------------------------------------------
+// private void indexResource2(Resource resource){
+// Query q = QueryFactory.create(String.format(resourceQuery,resource.getURI(),resource.getURI()), Syntax.syntaxARQ);
+// final ResultSet resultSet = QueryExecutionFactory.create(q, indexingDataset.toDataset()).execSelect();
+// Representation source = vf.createRepresentation(resource.getURI());
+// while(resultSet.hasNext()){
+// QuerySolution solution =resultSet.next();
+// RDFNode fieldNode = solution.get("field");
+// if(fieldNode.isURIResource()){
+// String field = fieldNode.asResource().getURI();
+// RDFNode value = solution.get("value");
+// if(value.isURIResource()){
+// source.addReference(field, value.asResource().getURI());
+// } else if(value.isLiteral()){
+// Literal literal = value.asLiteral();
+// if(literal.getDatatype() != null){
+// Object literalValue;
+// try {
+// literalValue = literal.getValue();
+// } catch (DatatypeFormatException e) {
+// log.warn(" Unable to convert "+literal.getLexicalForm()+" to "+literal.getDatatype()+"-> use lecicalForm");
+// literalValue = literal.getLexicalForm();
+// }
+// if(literalValue instanceof BaseDatatype.TypedValue){
+// source.add(field, literal.getLexicalForm());
+// } else {
+// source.add(field, literal.getValue());
+// }
+// } else {
+// String lang = literal.getLanguage();
+// if(lang != null && lang.isEmpty()){
+// lang = null;
+// }
+// source.addNaturalText(field, literal.getLexicalForm(),lang);
+// }
+// }
+// }
+// }
+// //log.info("S<source Resource:\n"+ModelUtils.getRepresentationInfo(source));
+// }
+ private void indexRanked() throws YardException {
+ if(entityRankings == null){
+ throw new IllegalStateException("Unable to index with Etity Ranking Mode if no Entity Rankings are present!");
+ }
+ long count = 0;
+ long alreadyIndexed = 0;
+ long stdCount = 0;
+ long notFound = 0;
+ long start = System.currentTimeMillis();
+ long startCurrent = System.currentTimeMillis();
+ for(Entry<String,Float> entry : entityRankings.entrySet()){
+ if(entry.getValue() < minimumRequiredEntityRanking){
+ continue; //ignore entities with rank < the min required one
+ }
+ count++;
+ if(count%1000 == 0){
+ long thisOne = System.currentTimeMillis()-startCurrent;
+ long all = System.currentTimeMillis()-start;
+ log.info(String.format("processed %s resources %s indexed in %sms (%sms/item | avg: %sms/item) %s std/resourc | %s not found",
+ count, count-alreadyIndexed, thisOne,(float)thisOne/1000,(float)all/count,(float)stdCount/(count-alreadyIndexed),notFound));
+ startCurrent = System.currentTimeMillis();
+ }
+ if(resumeMode && yard.isRepresentation(entry.getKey())){
+ alreadyIndexed++;
+ continue;
+ }
+ Representation source = vf.createRepresentation(entry.getKey());
+ Node resource = Node.createURI(entry.getKey());
+ ExtendedIterator<Triple> outgoing = indexingDataset.getDefaultGraph().find(resource, null, null);
+ boolean found = outgoing.hasNext();
+ while(outgoing.hasNext()){ //iterate over the statements for that resource
+ stdCount++;
+ Triple statement = outgoing.next();
+ Node predicate = statement.getPredicate();
+ if(predicate == null || !predicate.isURI()){
+ log.warn(String.format("Ignore field %s for resource %s because it is null or not an URI!",
+ predicate,resource));
+ } else {
+ String field = statement.getPredicate().getURI();
+ Node object = statement.getObject();
+ if(object == null){
+ log.warn(String.format("Encountered NULL value for field %s and resource %s",
+ predicate,resource));
+ }else if(object.isURI()){ //add a reference
+ source.addReference(field, object.getURI());
+ } else if(object.isLiteral()){ //add a value or a text depending on the dataType
+ LiteralLabel ll = object.getLiteral();
+ //if the dataType == null , than we can expect a plain literal
+ RDFDatatype dataType = ll.getDatatype();
+ if(dataType != null){ //add a value
+ Object literalValue;
+ try {
+ literalValue = ll.getValue();
+ if(literalValue instanceof BaseDatatype.TypedValue){
+ //used for unknown data types
+ // -> in such cases yust use the lecial type
+ source.add(field, ((BaseDatatype.TypedValue)literalValue).lexicalValue);
+ } else if(literalValue instanceof XSDDateTime) {
+ source.add(field, ((XSDDateTime)literalValue).asCalendar().getTime()); //Rick uses the time
+ } else if(literalValue instanceof XSDDuration) {
+ source.add(field, literalValue.toString());
+ } else {
+ source.add(field, literalValue);
+ }
+ } catch (DatatypeFormatException e) {
+ log.warn(" Unable to convert "+ll.getLexicalForm()+" to "+ll.getDatatype()+"-> use lecicalForm");
+ literalValue = ll.getLexicalForm();
+ }
+ } else { //add a text
+ String language = ll.language();
+ if(language!=null && language.length()<1){
+ language = null;
+ }
+ source.addNaturalText(field, ll.getLexicalForm(), language);
+ }
+ // "" is parsed if there is no language
+ } else {
+ if(object.isBlank()){
+ log.info(String.format("ignoreing blank node value %s for field %s and Resource %s!",
+ object,field,resource));
+ } else {
+ log.warn(String.format("ignoreing value %s for field %s and Resource %s because it is of an unsupported type!",
+ object,field,resource));
+ }
+ } //end different value node type
+ } //end else predicate != null
+ } //end iteration over resource triple
+ if(found) {
+ storeRepresentation(source);
+ //log.info("Resource: \n"+ModelUtils.getRepresentationInfo(source));
+ } else {
+ //log.info("No Statements found for "+entry.getKey()+" (ranking="+entry.getValue()+")!");
+ notFound++;
+ }
}
}
}
Propchange: incubator/stanbol/trunk/rick/indexing/geonames/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sun Dec 5 18:00:18 2010
@@ -0,0 +1,9 @@
+data
+
+target
+
+.settings
+
+.classpath
+
+.project