You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2011/05/04 14:51:31 UTC
svn commit: r1099435 [1/3] - in /incubator/stanbol/trunk:
entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/
entityhub/indexing/core/
entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/
entityhub/in...
Author: rwesten
Date: Wed May 4 12:51:29 2011
New Revision: 1099435
URL: http://svn.apache.org/viewvc?rev=1099435&view=rev
Log:
STANBOL-187: The DBLP indexer now uses the new Indexing Infrastructure:
Note that the DBLP Indexer does no longer contain any source code. It defines only the default configuration and the pom.xml to build (and assembly) the indexing Utility.
By calling the utility with the command "init" the configuration will be initialised with the defaults (see the README.txt for details).
other Changes:
SolrYard:
- has now support for the Solr commitWithin feature: This allows to tell solr the maximum duration until changes to the index need to be committed. This feature is very nice for indexing, because it reduces the number of commits.
- added a public close() method to the SolrYard to allow the deactivation of a SolrYard outside an OSGI environment.
The SolrYard IndexingDestination now uses the new features of the SolrYard
Indexing Core:
Added two very simple EntityScoreProvider. The first one simple returns null as score and is intended to be used if no scores are available. The second can be configured to return the same score for all entities (default is 1).
Parent POM:
- added all Entityhub Indexing bundles
- added SLF4J Bundles for log4j and the log4j bundle
Added:
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/NoEntityScoreProvider.java (with props)
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/StaticEntityScoreProvider.java (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/assembly/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/assembly/assembly.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/admin-extra.html (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/elevate.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/mapping-ISOLatin1Accent.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/protwords.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/schema.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/scripts.conf
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/solrconfig.xml (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/spellings.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/stopwords.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/synonyms.txt (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/xslt/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/xslt/example.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/xslt/example_atom.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/xslt/example_rss.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/xslt/luke.xsl (with props)
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/resources/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/resources/rdfData/
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/log4j.properties (with props)
Removed:
incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/java/org/
Modified:
incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java
incubator/stanbol/trunk/entityhub/indexing/core/pom.xml
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/Main.java
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConstants.java
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java
incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingSourceInitialiser.java
incubator/stanbol/trunk/entityhub/indexing/dblp/README.txt
incubator/stanbol/trunk/entityhub/indexing/dblp/pom.xml
incubator/stanbol/trunk/entityhub/indexing/destination/solryard/src/main/java/org/apache/stanbol/entityhub/indexing/destination/solryard/SolrYardIndexingDestination.java
incubator/stanbol/trunk/entityhub/indexing/source/jenatdb/pom.xml
incubator/stanbol/trunk/entityhub/yard/solr/pom.xml
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYard.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYardConfig.java
incubator/stanbol/trunk/parent/pom.xml
Modified: incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java (original)
+++ incubator/stanbol/trunk/entityhub/generic/core/src/main/java/org/apache/stanbol/entityhub/core/mapping/FieldMappingUtils.java Wed May 4 12:51:29 2011
@@ -260,7 +260,8 @@ public final class FieldMappingUtils {
if(dataType != null){
dataTypes.add(dataType.getUri());
} else {
- log.warn(String.format("DataType %s not supported! Datatype get not used by this Filter"));
+ log.warn(String.format("DataType %s not supported! Datatype get not used by this Filter",
+ dataTypeStrings[i]));
}
}
if(dataTypes.isEmpty()){
Modified: incubator/stanbol/trunk/entityhub/indexing/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/pom.xml?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/pom.xml (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/pom.xml Wed May 4 12:51:29 2011
@@ -67,17 +67,14 @@
<dependency>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.entityhub.servicesapi</artifactId>
- <version>${stanbol-version}</version>
</dependency>
<dependency>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.entityhub.core</artifactId>
- <version>${stanbol-version}</version>
</dependency>
<dependency> <!-- for the Main (command line utility) -->
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
- <version>1.2</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/Main.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/Main.java?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/Main.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/Main.java Wed May 4 12:51:29 2011
@@ -32,26 +32,53 @@ public class Main {
CommandLineParser parser = new PosixParser();
CommandLine line = parser.parse(options, args);
args = line.getArgs();
- if(line.hasOption('h')){
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(
- "java -Xmx{size} -jar org.apache.stanbol.indexing.core-*" +
- "-jar-with-dependencies.jar [options] [configDir]",
- "Indexing Commandline Utility: \n"+
- " size: Heap requirements depend on the dataset and the configuration.\n"+
- " 1024m should be a reasonable default.",
- options,
- null);
- System.exit(0);
+ if(line.hasOption('h') || args.length <= 0){
+ printHelp();
+ System.exit(0);
}
Indexer indexer;
IndexerFactory factory = IndexerFactory.getInstance();
- if(args.length > 0){
- indexer = factory.create(args[0]);
+ String path = null;
+ if(args.length > 1){
+ path = args[1];
+ }
+ if("init".equalsIgnoreCase(args[0]) ||
+ "index".equalsIgnoreCase(args[0])){
+ if(path != null){
+ indexer = factory.create(path);
+ } else {
+ indexer = factory.create();
+ }
+ if(line.hasOption('c')){
+ int cunckSize = Integer.parseInt(line.getOptionValue('c'));
+ indexer.setChunkSize(cunckSize);
+ }
+ if("index".equalsIgnoreCase(args[0])){
+ indexer.index();
+ }
} else {
- indexer = factory.create();
+ System.err.println("Unknown command "+args[0]+" (supported: init,index)\n\n");
+ printHelp();
}
- indexer.index();
+ System.exit(0);
+ }
+ /**
+ *
+ */
+ private static void printHelp() {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(
+ "java -Xmx{size} -jar org.apache.stanbol.indexing.core-*" +
+ "-jar-with-dependencies.jar [options] init|index [configDir]",
+ "Indexing Commandline Utility: \n"+
+ " size: Heap requirements depend on the dataset and the configuration.\n"+
+ " 1024m should be a reasonable default.\n" +
+ " init: Initialise the configuration with the defaults \n" +
+ " index: Needed to start the indexing process\n" +
+ " configDir: the path to the configuration directory (default:" +
+ " user.dir)",
+ options,
+ null);
}
}
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConfig.java Wed May 4 12:51:29 2011
@@ -1,13 +1,21 @@
package org.apache.stanbol.entityhub.indexing.core.config;
-import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.*;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_DESCRIPTION;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_ENTITY_DATA_ITERABLE;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_ENTITY_DATA_PROVIDER;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_ENTITY_ID_ITERATOR;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_ENTITY_PROCESSOR;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_ENTITY_SCORE_PROVIDER;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_INDEXING_DESTINATION;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_INDEX_FIELD_CONFIG;
+import static org.apache.stanbol.entityhub.indexing.core.config.IndexingConstants.KEY_NAME;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
@@ -17,8 +25,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;
import org.apache.stanbol.entityhub.core.mapping.FieldMappingUtils;
@@ -30,6 +41,7 @@ import org.apache.stanbol.entityhub.inde
import org.apache.stanbol.entityhub.indexing.core.IndexingDestination;
import org.apache.stanbol.entityhub.indexing.core.normaliser.DefaultNormaliser;
import org.apache.stanbol.entityhub.indexing.core.normaliser.ScoreNormaliser;
+import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapper;
import org.apache.stanbol.entityhub.servicesapi.mapping.FieldMapping;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,45 +63,133 @@ public class IndexingConfig {
private static final Logger log = LoggerFactory.getLogger(IndexingConfig.class);
private static final String DEFAULT_INDEX_FIELD_CONFIG_FILE_NAME = "indexFieldConfig.txt";
- private static final boolean ALLOW_CLASSPATH = true;
/**
- * This stores the context within the classpath to load the
+ * This stores the context within the classpath to initialise missing
+ * configurations and source based on the defaults in the classpath.
+ * This might be a directory or an jar file.
+ * @see {@link #loadViaClasspath(String)}
+ * @see #getConfigClasspathRootFolder()
*/
private final File classPathRootDir;
+ /**
+ * The root directory for the indexing (defaults to {@link #DEFAULT_ROOT_PATH})
+ */
private final File rootDir;
+ /**
+ * The root directory for the configuration
+ */
private final File configDir;
+ /**
+ * The root directory for the resources (indexing source files)
+ */
private final File sourceDir;
+ /**
+ * The root directory for the files created during the indexing process
+ */
private final File destinationDir;
+ /**
+ * The root directory for the distribution files created in the finalisation
+ * phase of the indexing (e.g. The archive with the index,
+ * OSGI configuration, ...)
+ */
private final File distributionDir;
+ /**
+ * Map between the relative paths stored in {@link #rootDir}, {@link #configDir},
+ * {@link #sourceDir}, {@link #destinationDir} and {@link #distributionDir}
+ * to the {@link File#getCanonicalFile()} counterparts as returned by the
+ * {@link #getRootFolder()} ... methods.
+ */
private final Map<File,File> canonicalDirs = new HashMap<File,File>();
+ /**
+ * The main indexing configuration as parsed form {@link #INDEXING_PROERTIES}
+ * file within the {@link #configDir}.
+ */
private final Map<String,Object> configuration;
+ /**
+ * The value of the {@link IndexingConstants#KEY_NAME} property
+ */
private String name;
-
+ /**
+ * The {@link EntityDataIterable} instance initialised based on the value
+ * of the {@link IndexingConstants#KEY_ENTITY_DATA_ITERABLE} key or
+ * <code>null</code> if not configured.
+ * This variable uses lazy initialisation
+ * @see #getDataInterable()
+ */
private EntityDataIterable entityDataIterable = null;
+ /**
+ * The {@link EntityDataProvider} instance initialised based on the value
+ * of the {@link IndexingConstants#KEY_ENTITY_DATA_PROVIDER} key or
+ * <code>null</code> if not configured.
+ * This variable uses lazy initialisation
+ * @see #getEntityDataProvider()
+ */
private EntityDataProvider entityDataProvider = null;
+ /**
+ * The {@link EntityIterator} instance initialised based on the value
+ * of the {@link IndexingConstants#KEY_ENTITY_ID_ITERATOR} key or
+ * <code>null</code> if not configured.
+ * This variable uses lazy initialisation
+ * @see #getEntityIdIterator()
+ */
private EntityIterator entityIdIterator = null;
+ /**
+ * The {@link EntityScoreProvider} instance initialised based on the value
+ * of the {@link IndexingConstants#KEY_ENTITY_SCORE_PROVIDER} key or
+ * <code>null</code> if not configured.
+ * This variable uses lazy initialisation
+ * @see #getEntityScoreProvider()
+ */
private EntityScoreProvider entityScoreProvider = null;
-
+ /**
+ * The {@link ScoreNormaliser} instance initialised based on the value
+ * of the {@link IndexingConstants#KEY_SCORE_NORMALIZER} key or
+ * <code>null</code> if not configured.
+ * This variable uses lazy initialisation
+ * @see #getNormaliser()
+ */
private ScoreNormaliser scoreNormaliser = null;
-
+ /**
+ * The {@link EntityProcessor} instance initialised based on the value
+ * of the {@link IndexingConstants#KEY_ENTITY_PROCESSOR} key or
+ * <code>null</code> if not configured.
+ * This variable uses lazy initialisation
+ * @see #getEntityProcessor()
+ */
private EntityProcessor entityProcessor = null;
-
+ /**
+ * The {@link IndexingDestination} instance initialised based on the value
+ * of the {@link IndexingConstants#KEY_INDEXING_DESTINATION} key or
+ * <code>null</code> if not configured.
+ * This variable uses lazy initialisation
+ * @see #getIndexingDestination()
+ */
private IndexingDestination indexingDestination = null;
/**
- * The configuration of the fields/languages included/excluded in the index.
+ * The configuration of the fields/languages included/excluded in the index
+ * as parsed based on the value of the
+ * {@link IndexingConstants#KEY_INDEX_FIELD_CONFIG} key.
*/
private Collection<FieldMapping> fieldMappings;
+ /**
+ * Creates an instance using {@link #DEFAULT_ROOT_PATH} (relative to the
+ * working directory) as {@link #getRootFolder()} for the indexing
+ */
public IndexingConfig(){
this(null);
}
-
+ /**
+ * Creates an isntace using the parsed offset plus {@link #DEFAULT_ROOT_PATH}
+ * as {@link #getRootFolder()} for the indexing
+ * @param rootPath
+ */
public IndexingConfig(String rootPath){
//first get the root
File root;// = new File(System.getProperty("user.dir"));
@@ -252,10 +352,7 @@ public class IndexingConfig {
/**
* Searches for a resource with the parsed name in the parsed directory.
- * If it can not be found, than it looks up the file within the classpath
- * and if found copies it to the parsed location. In case the parsed
- * file refers to a directory, then all the contents of that directory
- * are copied.
+ * If it can not be found it tries to initialise it via the classpath.
* @param root the (relative path) to the directory containing the file.
* typically on of {@link #configDir} or {@link #sourceDir}.
* @param fileName the name of the file (file or directory)
@@ -263,34 +360,112 @@ public class IndexingConfig {
*/
private File getResource(File root, String fileName) {
File resource = new File(root,fileName);
- File absoluteResource = resource.getAbsoluteFile();
log.info("reauest for Resource {} (folder: {})",fileName,root);
- if(absoluteResource.exists()){
+ if(resource.getAbsoluteFile().exists()){
log.info(" > rquested Resource present");
- } else if(classPathRootDir != null){
- log.info(" > rquested Resource missing ");
- log.info(" ... try to init via Classpath {}",classPathRootDir);
+ } else if(copyFromClasspath(resource)){
+ log.info(" > rquested Resource copied from Classpath ");
+ } else {
+ log.info(" > rquested Resource not found");
+ }
+ return resource.getAbsoluteFile();
+ }
+ /**
+ * This method copies Resources from the Classpath over to the target
+ * resource. It supports both files and directories. In case of directories
+ * all sub-directories and there files are copied.<p>
+ * One can not use {@link ClassLoader#getResource(String)} because it does
+ * only support files and no directories.
+ * @param resource the target resource (relative path also found in the jar)
+ * @return <code>true</code> if the resource was found and copied.
+ */
+ private boolean copyFromClasspath(File resource){
+ if(classPathRootDir == null){ //not available
+ return false;
+ } else if(classPathRootDir.isDirectory()){ // loaded from directory
File classpathResource = new File(classPathRootDir,resource.getPath());
try {
if(classpathResource.isFile()){
- FileUtils.copyFile(classpathResource, absoluteResource);
- log.info(" > resource created");
+ FileUtils.copyFile(classpathResource, resource.getAbsoluteFile());
+ return true;
} else if(classpathResource.isDirectory()){
- FileUtils.copyDirectory(classpathResource, absoluteResource);
- log.info(" > directory created");
+ FileUtils.copyDirectory(classpathResource, resource.getAbsoluteFile());
+ return true;
} else {
- log.info(" > not found in Classpath");
+ return false;
}
} catch(IOException e){
throw new IllegalStateException(
String.format("Unable to copy Configuration form classpath " +
"resource %s to target file %s!",
- classpathResource, absoluteResource),e);
+ classpathResource, resource.getAbsolutePath()),e);
}
- } else {
- log.info(" > initialisation of resources via classpath not possible ");
+ } else { //loaded form a jar file
+ boolean found = false;
+ JarFile jar = null;
+ try {
+ jar = new JarFile(classPathRootDir);
+ String resourceName = resource.getPath();
+ Enumeration<JarEntry> entries = jar.entries();
+ boolean completed = false;
+ //we need to iterate over the entries because the resource might
+ //refer to an file but missing the tailing '/'
+ while(entries.hasMoreElements() && !completed){
+ JarEntry entry = entries.nextElement();
+ String entryName = entry.getName();
+ if(entryName.startsWith(resourceName)){
+ log.info("found entry : {}[dir={}]",entryName,entry.isDirectory());
+ if(entryName.equals(resourceName) && !entry.isDirectory()){
+ //found the resource and it is an file -> copy and return
+ completed = true;
+ }
+ if(!entry.isDirectory()){ //copy a file
+ //still check if the target folder exist
+ //TODO: this depends on user.dir is root dir
+ File targetFolder = new File(
+ FilenameUtils.getPathNoEndSeparator(entryName));
+ if(targetFolder.exists() || targetFolder.mkdirs()){
+ File outFile = new File(targetFolder,
+ FilenameUtils.getName(entry.getName()));
+ InputStream is = jar.getInputStream(entry);
+ OutputStream os = new FileOutputStream(outFile);
+ IOUtils.copyLarge(is,os);
+ IOUtils.closeQuietly(is);
+ IOUtils.closeQuietly(os);
+ //found one resource
+ found = true;
+ log.info(" > created File {}",outFile);
+ } else {
+ throw new IllegalStateException("Unable to create" +
+ "folder "+targetFolder);
+ }
+ } else { //directory
+ //TODO: this depends on user.dir is root dir
+ File targetFolder = new File(entryName);
+ if(!targetFolder.exists() && !targetFolder.mkdirs()){
+ throw new IllegalStateException("Unable to create" +
+ "folder "+targetFolder);
+ } else { //created a directory
+ log.info(" > created Directory {}",targetFolder);
+ found = true;
+ }
+ }
+ } // else entry does not start with the parsed resource
+ } //end while entries
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to copy resources from" +
+ "jar file "+classPathRootDir+"!",e);
+ } finally {
+ if(jar != null){
+ try {
+ jar.close();
+ } catch (IOException e) {
+ //ignore
+ }
+ }
+ }
+ return found;
}
- return absoluteResource;
}
/**
@@ -324,11 +499,10 @@ public class IndexingConfig {
String contextResource = new File(configDir,INDEXING_PROERTIES).getPath();
URL contextUrl = loadViaClasspath(contextResource);
if(contextUrl == null){// if indexing.properties is not found via classpath
- log.warn("No '{}' found via classpath. Will try to init via '{}' but that usually does not work.",
- INDEXING_PROERTIES,IndexingConfig.class);
- //use this class ...
- contextResource = IndexingConfig.class.getName().replace('.', '/')+".class";
- contextUrl = loadViaClasspath(contextResource);
+ log.info("No '{}' found via classpath. Loading Resource via" +
+ "the classpath is deactivated.",
+ INDEXING_PROERTIES);
+ return null;
}
String resourcePath;
try {
@@ -357,16 +531,7 @@ public class IndexingConfig {
} else {
classpathRoot = new File(resourcePath.substring(0,resourcePath.length()-contextResource.length()));
}
- //and now the folder representing the rootDir (but within the Classpath)
- //to validate if this folder is present in the classpath
- File indexingClasspathRoot = new File(classpathRoot,rootDir.getPath());
- if(indexingClasspathRoot.isDirectory()){
- //but return still the root, because configDir, ... are all relative
- //to this root
- return classpathRoot;
- } else {
- return null;
- }
+ return classpathRoot;
}
/**
@@ -545,8 +710,8 @@ public class IndexingConfig {
public EntityIterator getEntityIdIterator() {
if(entityIdIterator != null){
return entityIdIterator;
- } else if(configuration.containsKey(KEY_ENTITY_ID_ITERATPR)){
- ConfigEntry config = parseConfigEntry(configuration.get(KEY_ENTITY_ID_ITERATPR).toString());
+ } else if(configuration.containsKey(KEY_ENTITY_ID_ITERATOR)){
+ ConfigEntry config = parseConfigEntry(configuration.get(KEY_ENTITY_ID_ITERATOR).toString());
try {
entityIdIterator = (EntityIterator)Class.forName(config.getClassName()).newInstance();
} catch (Exception e) {
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConstants.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConstants.java?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConstants.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/config/IndexingConstants.java Wed May 4 12:51:29 2011
@@ -13,7 +13,7 @@ public interface IndexingConstants {
String KEY_DESCRIPTION = "description";
String KEY_ENTITY_DATA_ITERABLE = "entityDataIterable";
String KEY_ENTITY_DATA_PROVIDER = "entityDataProvider";
- String KEY_ENTITY_ID_ITERATPR = "entityIdIterator";
+ String KEY_ENTITY_ID_ITERATOR = "entityIdIterator";
String KEY_ENTITY_SCORE_PROVIDER = "entityScoreProvider";
String KEY_INDEXING_DESTINATION = "indexingDestination";
String KEY_INDEX_FIELD_CONFIG = "fieldConfiguration";
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/FinishedEntityDaemon.java Wed May 4 12:51:29 2011
@@ -162,7 +162,7 @@ public class FinishedEntityDaemon extend
double itemSourceDurationAll = countedAll>0? sourceDurationAll/countedAll:-1;
double itemProcessingDurationAll = countedAll>0? processDurationAll/countedAll:-1;
- double itemStoreDurationAll = countedAll>0? sourceDurationAll/countedAll:-1;
+ double itemStoreDurationAll = countedAll>0? storeDurationAll/countedAll:-1;
double itemSourceDurationMajor = countedMajor>0? sourceDurationMajor/countedMajor:-1;
double itemProcessingDurationMajor = countedMajor>0? processDurationMajor/countedMajor:-1;
Modified: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingSourceInitialiser.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingSourceInitialiser.java?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingSourceInitialiser.java (original)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/impl/IndexingSourceInitialiser.java Wed May 4 12:51:29 2011
@@ -63,7 +63,7 @@ public class IndexingSourceInitialiser i
private static final long serialVersionUID = -1L;
private final IndexingComponent indexingSource;
protected IndexingSourceEventObject(IndexingSourceInitialiser initialiser,IndexingComponent source){
- super(source);
+ super(initialiser);
indexingSource = source;
}
@Override
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/NoEntityScoreProvider.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/NoEntityScoreProvider.java?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/NoEntityScoreProvider.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/NoEntityScoreProvider.java Wed May 4 12:51:29 2011
@@ -0,0 +1,48 @@
+package org.apache.stanbol.entityhub.indexing.core.source;
+
+import java.util.Map;
+
+import org.apache.stanbol.entityhub.indexing.core.EntityScoreProvider;
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+
+/**
+ * Returns <code>null</code> as score for each entity and therefore indicating
+ * that there is no score available for any entity.
+ * @author Rupert Westenthaler
+ *
+ */
+public class NoEntityScoreProvider implements EntityScoreProvider {
+
+ @Override
+ public boolean needsData() {
+ return false;
+ }
+
+ @Override
+ public Float process(String id) {
+ return null;
+ }
+
+ @Override
+ public Float process(Representation entity) {
+ return null;
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public void initialise() {
+ }
+
+ @Override
+ public boolean needsInitialisation() {
+ return false;
+ }
+
+ @Override
+ public void setConfiguration(Map<String,Object> config) {
+ }
+
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/NoEntityScoreProvider.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/StaticEntityScoreProvider.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/StaticEntityScoreProvider.java?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/StaticEntityScoreProvider.java (added)
+++ incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/StaticEntityScoreProvider.java Wed May 4 12:51:29 2011
@@ -0,0 +1,83 @@
+package org.apache.stanbol.entityhub.indexing.core.source;
+
+import java.util.Map;
+
+import org.apache.stanbol.entityhub.indexing.core.EntityScoreProvider;
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+
+/**
+ * Implementation of an {@link EntityScoreProvider} that parsed the same value
+ * for all entities. The default value is '1' but it can be configured by using
+ * {@link #PARAM_SCORE}("{@value #PARAM_SCORE}").
+ *
+ * @author Rupert Westenthaler
+ *
+ */
+public class StaticEntityScoreProvider implements EntityScoreProvider {
+
+ public static final String PARAM_SCORE = "score";
+ public static final Float DEFAULT_SCORE = Float.valueOf(1f);
+
+ private Float score = DEFAULT_SCORE;
+
+ public StaticEntityScoreProvider() {
+ this(null);
+ }
+ public StaticEntityScoreProvider(Float score){
+ setScore(score);
+ }
+ /**
+ * @param score
+ */
+ private void setScore(Float score) {
+ if(score == null){
+ this.score = DEFAULT_SCORE;
+ } else if (score <= 0){
+ throw new IllegalArgumentException("The parsed Score MUST NOT be <= 0!");
+ } else {
+ this.score = score;
+ }
+ }
+
+ @Override
+ public boolean needsData() {
+ return false;
+ }
+
+ @Override
+ public Float process(String id) {
+ return score;
+ }
+
+ @Override
+ public Float process(Representation entity) {
+ return score;
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public void initialise() {
+ }
+
+ @Override
+ public boolean needsInitialisation() {
+ return false;
+ }
+
+ @Override
+ public void setConfiguration(Map<String,Object> config) {
+ Object value = config.get(PARAM_SCORE);
+ if(value != null && !value.toString().isEmpty()){
+ try {
+ setScore(Float.valueOf(value.toString()));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(String.format(
+ "Unable to parse score from value {}",value),e);
+ }
+ }
+ }
+
+}
Propchange: incubator/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/source/StaticEntityScoreProvider.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/stanbol/trunk/entityhub/indexing/dblp/README.txt
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/README.txt?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/README.txt (original)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/README.txt Wed May 4 12:51:29 2011
@@ -9,30 +9,38 @@ If not yet build by the built process of
mvn install
in this directory.
-To create the runable jar that contains all the dependencies call
- mvn assembly:assembly
-
-If everything completes successfully, than there should be two jar files within
-the target directory.
-The one called
- org.apache.stanbol.entityhub.indexing.dblp-0.1*-jar-with-dependencies.jar
-is the one to be used for indexing.
+If the build succeeds go to the /target directory and copy the
+ org.apache.stanbol.entityhub.indexing.dblp-*-jar-with-dependencies.jar
+to the directory you would like to start the indexing.
-Creating the index:
+Index:
==================
-(1) download the dump from http://dblp.l3s.de/dblp.rdf.gz
-(2) rename the dump file to "dblp.nt.gz" to allow the RdfIndexer to correctly
- set the RDF format to NTRIPLES!
-(3) The Indexer will need a SolrServer. So you need to prepare the Solr Index
- to store the data.
- A default configuration is provided within the "/solrConf" directory. This
- can be used to configure a SorlServer or a new Core to an existing SolrServer.
- You can also copy the "dblp" folder within the "/solrConf" directory to an
- other location and than parse the absolute path as SolrServer location to the
- Tool. In that case an EmbeddedSolrServer will be used for indexing.
-(4) call the tool with the -h option to print the help screen
- java -jar ./target/org.apache.stanbol.entityhub.indexing.dblp-*-jar-with-dependencies.jar -h
+(1) Initialise the configuration by calling
+java -jar org.apache.stanbol.entityhub.indexing.dblp-*-jar-with-dependencies.jar init
-Indexing took about 3h on my Computer. Indexing time heavily depends on the
-used hard disc.
\ No newline at end of file
+This will create a sub-folder with the name indexing in the current directory.
+Within this folder all the
+ - configurations (indexing/config)
+ - source files (indexing/resources)
+ - created files (indexing/destination)
+ - distribution files (indexing/distribution)
+will be located.
+
+(2) Download the Source File:
+
+Download the DBLP RDF dump from http://dblp.l3s.de/dblp.rdf.gz to
+"indexing/resources/rdfData" and rename it to "dblp.nt.gz" (because this file
+does not use rdf/xml but N-Triples).
+You can use the following two commands to accomplish this step
+
+curl -C - -O http://dblp.l3s.de/dblp.rdf.gz
+mv dblp.rdf.gz indexing/resources/rdfData/dblp.rdf.gz
+
+(3) Start the indexing by calling
+java -Xmx1024m -jar org.apache.stanbol.entityhub.indexing.dblp-*-jar-with-dependencies.jar index
+
+Note that calling the utility with the option -h will print the help.
+
+Indexing took about 3h on a normal hard disk and about 40min on a SSD (on a
+2010 MacBook Pro).
\ No newline at end of file
Modified: incubator/stanbol/trunk/entityhub/indexing/dblp/pom.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/pom.xml?rev=1099435&r1=1099434&r2=1099435&view=diff
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/pom.xml (original)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/pom.xml Wed May 4 12:51:29 2011
@@ -19,112 +19,228 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
-<!-- Do not use a parent because of problems with missing dependencies with
- mvn assembly:assembly
<parent>
<groupId>org.apache.stanbol</groupId>
- <artifactId>org.apache.stanbol.entityhub.parent</artifactId>
+ <artifactId>org.apache.stanbol.entityhub.indexing.parent</artifactId>
<version>0.9-SNAPSHOT</version>
- <relativePath>../../parent</relativePath>
- </parent> -->
+ <relativePath>../parent</relativePath>
+ </parent>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.entityhub.indexing.dblp</artifactId>
<version>0.9-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Stanbol Entityhub Indexing for dblp</name>
<description>This uses the RDF dump provided by
- http://dblp.l3s.de/dblp.rdf.gz to create a Full Cache
- for DBLP Computer Science Bibliography
- (http://dblp.uni-trier.de) within an SolrYard
+ http://dblp.l3s.de/dblp.rdf.gz to create a local index of DBLP to be used
+ with the Apache Stanbol Entityhub
</description>
<scm>
<connection>
- scm:svn:http://svn.apache.org/repos/asf/incubator/stanbol/trunk/entityhub/indexing/dbpedia
+ scm:svn:http://svn.apache.org/repos/asf/incubator/stanbol/trunk/entityhub/indexing/dblp
</connection>
<developerConnection>
- scm:svn:https://svn.apache.org/repos/asf/incubator/stanbol/trunk/entityhub/indexing/dbpedia
+ scm:svn:https://svn.apache.org/repos/asf/incubator/stanbol/trunk/entityhub/indexing/dblp
</developerConnection>
<url>http://incubator.apache.org/stanbol</url>
</scm>
<dependencies>
- <!-- for logging -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.5.8</version>
- </dependency>
+ <!-- indexing dependencies -->
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>1.5.8</version>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.indexing.core</artifactId>
<scope>runtime</scope>
</dependency>
- <!-- RICK dependencies -->
<dependency>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.entityhub.servicesapi</artifactId>
- <version>0.9-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.stanbol</groupId>
- <artifactId>org.apache.stanbol.entityhub.core</artifactId>
- <version>0.9-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.stanbol</groupId>
- <artifactId>org.apache.stanbol.entityhub.yard.solr</artifactId>
- <version>0.9-SNAPSHOT</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.stanbol</groupId>
<artifactId>org.apache.stanbol.entityhub.core</artifactId>
- <version>0.9-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.stanbol</groupId>
- <artifactId>org.apache.stanbol.entityhub.indexing.genericrdf</artifactId>
- <version>0.9-SNAPSHOT</version>
+ <scope>runtime</scope>
</dependency>
- <!-- for the main -->
- <dependency>
+ <dependency> <!-- for the Main (command line utility) -->
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
- <version>1.2</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>1.4</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
- <version>1.0</version>
+ <scope>runtime</scope>
+ </dependency>
+
+ <!-- dependencies of the jena tdb source -->
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.indexing.source.jenatdb</artifactId>
+ <scope>runtime</scope>
</dependency>
- <!-- osgi stuff -->
<dependency>
- <groupId>org.apache.felix</groupId>
- <artifactId>org.apache.felix.configadmin</artifactId>
- <version>1.2.4</version>
+ <groupId>com.hp.hpl.jena</groupId>
+ <artifactId>jena</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.felix</groupId>
- <artifactId>org.osgi.foundation</artifactId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.hp.hpl.jena</groupId>
+ <artifactId>tdb</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.hp.hpl.jena</groupId>
+ <artifactId>arq</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.ibm.icu</groupId>
+ <artifactId>icu4j</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <!-- dependencies of the solr yard destination -->
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.indexing.destination.solryard</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.stanbol</groupId>
+ <artifactId>org.apache.stanbol.entityhub.yard.solr</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <!-- needed because the SolrYard uses ConfigurationExceptions -->
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.compendium</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-highlighter</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-memory</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-misc</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queries</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-snowball</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spellchecker</artifactId>
+ <scope>runtime</scope>
</dependency>
<dependency>
- <groupId>org.apache.felix</groupId>
- <artifactId>org.osgi.core</artifactId>
- <version>1.4.0</version>
- </dependency>
- <!-- for testing -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.7</version>
- <scope>test</scope>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-commons-csv</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>stax</groupId>
+ <artifactId>stax-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.woodstox</groupId>
+ <artifactId>wstx-asl</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <!-- The Solr-Core depends on the Servlet API and in case of running an
+ EmbeddedSolrServer the servlet-api is not available -->
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+
+ <!-- for logging -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency> <!-- we use log4j 1.2 -->
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency> <!-- see also the log4j.properties in src/main/resources -->
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>runtime</scope>
</dependency>
</dependencies>
@@ -142,15 +258,27 @@
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2</version>
<configuration>
- <descriptorRefs>
+ <descriptors>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </descriptors>
+ <!-- <descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
+ </descriptorRefs> -->
<archive>
<manifest>
- <mainClass>org.apache.stanbol.entityhub.indexing.dblp.cli.CommandLineRunner</mainClass>
+ <mainClass>org.apache.stanbol.entityhub.indexing.Main</mainClass>
</manifest>
</archive>
</configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
Added: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/assembly/assembly.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/assembly/assembly.xml?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/assembly/assembly.xml (added)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/assembly/assembly.xml Wed May 4 12:51:29 2011
@@ -0,0 +1,29 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>jar-with-dependencies</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <unpack>true</unpack>
+ <scope>runtime</scope>
+ <unpackOptions>
+ <excludes>
+ <exclude>META-INF/maven/**</exclude>
+ <exclude>log4j.properties</exclude>
+ </excludes>
+ </unpackOptions>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.outputDirectory}</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>**</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
Propchange: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/assembly/assembly.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/admin-extra.html
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/admin-extra.html?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/admin-extra.html (added)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/admin-extra.html Wed May 4 12:51:29 2011
@@ -0,0 +1,31 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- The content of this page will be statically included into the top
+of the admin page. Uncomment this as an example to see there the content
+will show up.
+
+<hr>
+<i>This line will appear before the first table</i>
+<tr>
+<td colspan="2">
+This row will be appended to the end of the first table
+</td>
+</tr>
+<hr>
+
+-->
Propchange: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/admin-extra.html
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/elevate.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/elevate.xml?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/elevate.xml (added)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/elevate.xml Wed May 4 12:51:29 2011
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- If this file is found in the config directory, it will only be
+ loaded once at startup. If it is found in Solr's data
+ directory, it will be re-loaded every commit.
+-->
+
+<elevate>
+<!--
+ <query text="foo bar">
+ <doc id="1" />
+ <doc id="2" />
+ <doc id="3" />
+ </query>
+
+ <query text="ipod">
+ <doc id="MA147LL/A" /> <!- put the actual ipod at the top ->
+ <doc id="IW-02" exclude="true" /> <!- exclude this cable ->
+ </query>
+-->
+</elevate>
Propchange: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/elevate.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/mapping-ISOLatin1Accent.txt
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/mapping-ISOLatin1Accent.txt?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/mapping-ISOLatin1Accent.txt (added)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/mapping-ISOLatin1Accent.txt Wed May 4 12:51:29 2011
@@ -0,0 +1,246 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Syntax:
+# "source" => "target"
+# "source".length() > 0 (source cannot be empty.)
+# "target".length() >= 0 (target can be empty.)
+
+# example:
+# "Ã" => "A"
+# "\u00C0" => "A"
+# "\u00C0" => "\u0041"
+# "Ã" => "ss"
+# "\t" => " "
+# "\n" => ""
+
+# Ã => A
+"\u00C0" => "A"
+
+# Ã => A
+"\u00C1" => "A"
+
+# Ã => A
+"\u00C2" => "A"
+
+# Ã => A
+"\u00C3" => "A"
+
+# Ã => A
+"\u00C4" => "A"
+
+# Ã
=> A
+"\u00C5" => "A"
+
+# Ã => AE
+"\u00C6" => "AE"
+
+# Ã => C
+"\u00C7" => "C"
+
+# Ã => E
+"\u00C8" => "E"
+
+# Ã => E
+"\u00C9" => "E"
+
+# Ã => E
+"\u00CA" => "E"
+
+# Ã => E
+"\u00CB" => "E"
+
+# Ã => I
+"\u00CC" => "I"
+
+# Ã => I
+"\u00CD" => "I"
+
+# Ã => I
+"\u00CE" => "I"
+
+# Ã => I
+"\u00CF" => "I"
+
+# IJ => IJ
+"\u0132" => "IJ"
+
+# Ã => D
+"\u00D0" => "D"
+
+# Ã => N
+"\u00D1" => "N"
+
+# Ã => O
+"\u00D2" => "O"
+
+# Ã => O
+"\u00D3" => "O"
+
+# Ã => O
+"\u00D4" => "O"
+
+# Ã => O
+"\u00D5" => "O"
+
+# Ã => O
+"\u00D6" => "O"
+
+# Ã => O
+"\u00D8" => "O"
+
+# Å => OE
+"\u0152" => "OE"
+
+# Ã
+"\u00DE" => "TH"
+
+# Ã => U
+"\u00D9" => "U"
+
+# Ã => U
+"\u00DA" => "U"
+
+# Ã => U
+"\u00DB" => "U"
+
+# Ã => U
+"\u00DC" => "U"
+
+# Ã => Y
+"\u00DD" => "Y"
+
+# Ÿ => Y
+"\u0178" => "Y"
+
+# Ã => a
+"\u00E0" => "a"
+
+# á => a
+"\u00E1" => "a"
+
+# â => a
+"\u00E2" => "a"
+
+# ã => a
+"\u00E3" => "a"
+
+# ä => a
+"\u00E4" => "a"
+
+# å => a
+"\u00E5" => "a"
+
+# æ => ae
+"\u00E6" => "ae"
+
+# ç => c
+"\u00E7" => "c"
+
+# è => e
+"\u00E8" => "e"
+
+# é => e
+"\u00E9" => "e"
+
+# ê => e
+"\u00EA" => "e"
+
+# ë => e
+"\u00EB" => "e"
+
+# ì => i
+"\u00EC" => "i"
+
+# Ã => i
+"\u00ED" => "i"
+
+# î => i
+"\u00EE" => "i"
+
+# ï => i
+"\u00EF" => "i"
+
+# ij => ij
+"\u0133" => "ij"
+
+# ð => d
+"\u00F0" => "d"
+
+# ñ => n
+"\u00F1" => "n"
+
+# ò => o
+"\u00F2" => "o"
+
+# ó => o
+"\u00F3" => "o"
+
+# ô => o
+"\u00F4" => "o"
+
+# õ => o
+"\u00F5" => "o"
+
+# ö => o
+"\u00F6" => "o"
+
+# ø => o
+"\u00F8" => "o"
+
+# Å => oe
+"\u0153" => "oe"
+
+# Ã => ss
+"\u00DF" => "ss"
+
+# þ => th
+"\u00FE" => "th"
+
+# ù => u
+"\u00F9" => "u"
+
+# ú => u
+"\u00FA" => "u"
+
+# û => u
+"\u00FB" => "u"
+
+# ü => u
+"\u00FC" => "u"
+
+# ý => y
+"\u00FD" => "y"
+
+# ÿ => y
+"\u00FF" => "y"
+
+# ï¬ => ff
+"\uFB00" => "ff"
+
+# ï¬ => fi
+"\uFB01" => "fi"
+
+# ï¬ => fl
+"\uFB02" => "fl"
+
+# ï¬ => ffi
+"\uFB03" => "ffi"
+
+# ï¬ => ffl
+"\uFB04" => "ffl"
+
+# ï¬
=> ft
+"\uFB05" => "ft"
+
+# ï¬ => st
+"\uFB06" => "st"
Propchange: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/mapping-ISOLatin1Accent.txt
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/protwords.txt
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/protwords.txt?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/protwords.txt (added)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/protwords.txt Wed May 4 12:51:29 2011
@@ -0,0 +1,21 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+# Use a protected word file to protect against the stemmer reducing two
+# unrelated words to the same base word.
+
+# Some non-words that normally won't be encountered,
+# just to test that they won't be stemmed.
+dontstems
+zwhacky
+
Propchange: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/protwords.txt
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/schema.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/schema.xml?rev=1099435&view=auto
==============================================================================
--- incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/schema.xml (added)
+++ incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/schema.xml Wed May 4 12:51:29 2011
@@ -0,0 +1,458 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This is the Solr schema file. This file should be named "schema.xml" and
+ should be in the conf directory under the solr home
+ (i.e. ./solr/conf/schema.xml by default)
+ or located where the classloader for the Solr webapp can find it.
+
+ This schema will be used by the Apache Stanbol SolrYard implementation to
+ index entities.
+ The implementation of the SolrYard does made several assumptions on
+ configurations defined in this Schema. So changes to this schema that do
+ affect such assumptions will most likelly cause unpredictable errors!
+
+ However there are also a lot of places where users can optimize this schema
+ to specific requirements. See the comments within this schema for more
+ details!
+
+ For more information, on how to customize the Solr schema.xml in general,
+ please see http://wiki.apache.org/solr/SchemaXml.
+
+-->
+
+<schema name="Apache Stanbol SolrYard Schema" version="1.2">
+ <!--
+ The SolrYard supports a list of types that is reflected by
+ "fieldType" specifications within this schema.
+ See the specific fieldType definition for more information
+ -->
+ <types>
+ <!--
+ This fieldType is used to store values with the dataType "xsd:string".
+ It is NOT used for natural language texts. Assume that this data type is
+ used for ISBN numbers, article numbers, string representations of
+ unsupported data types ...
+ -->
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+ <!--
+ This can be used as alternative to "string" to enable case insensitive
+ searches on string values.
+ The KeywordTokenizerFactory ensures that the whole string is preserved as
+ a single token.
+ -->
+ <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory" />
+ </analyzer>
+ </fieldType>
+
+
+ <!-- boolean type: "true" or "false" used to store values with the datatype "xsd:boolean" -->
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+ <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings.
+ Currently not used by the SolrYard implementation, but reserved for future use. -->
+ <fieldtype name="binary" class="solr.BinaryField"/>
+ <!--
+ Default numeric and date field types. By default used to index numeric values.
+ Note that the "solr.TrieIntField" does support indexing values at various
+ levels of precision to accelerate range queries. However the
+ precisionStep of 0 used by this fieldTypes disables this feature.
+ Change presisionStep to values > 0 to activate hierarchival indexing
+ for all numeric fields of that types. See Solr documentation for
+ suitable values and examples.
+ -->
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
+
+ <!--
+ Numeric and date field types that do activate indexing values at various
+ levels of precision to accelerate range queries.
+ This can be used to activate hierarchival indexing for specific
+ fields. See Notes within the field section.
+ -->
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/>
+
+ <!--
+ Natural Language Texts
+
+ Indexing of natural language texts are supported by the solr.TextField class that
+ allows the specification of custom text analyzers specified as a tokenizer and a
+ list of token filters.
+
+ For more info on customizing your analyzer chain, please see
+ http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
+
+ The SolrYard has support for different languages. Within the schema.xml one need
+ to define first a "fieldType" for language and second a dynamicField for the
+ prefix used by the SolrYard for this language.
+ For more information about the prefixes used by the SolrYard see the notes in the
+ field section of this configuration.
+
+ In addition the SolrYard also indexes natural language values (of any language)
+ together with string values within a special field to support searches for
+ texts without an specified language.
+ -->
+ <!--
+ A general unstemmed text field - good if one does not know the language of the field.
+ This is used as the default fieldType for fields that store values of different
+ languages.
+ It is also the default fieldType for languages that do not define special fieldTypes.
+ -->
+ <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!--
+ A text field that only splits on whitespace for exact matching of words.
+ Currently not used. May be used as an alternative to the textgen fieldType.
+ -->
+ <!--
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+ -->
+
+ <!--
+ This is the default fieldType used for english language texts.
+
+ Less flexible matching than the text_en field type, but less false matches.
+ Probably not ideal for product names, but may be good for SKUs.
+ Can insert dashes in the wrong place and still match.
+ -->
+ <fieldType name="text_en_Tight" class="solr.TextField" positionIncrementGap="100" >
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <!-- this filter can remove any duplicate tokens that appear at the same position - sometimes
+ possible with WordDelimiterFilter in conjuncton with stemming. -->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+
+ <!--
+ This can be used as an alternative to the "text_en_Tight" fieldTpye for
+ english langauge texts.
+
+ A text field that uses WordDelimiterFilter to enable splitting and matching of
+ words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
+ so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
+ Synonyms and stopwords are customized by external files, and stemming is enabled.
+ -->
+ <!--
+ <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ </analyzer>
+ </fieldType>
+ -->
+
+ <!--
+ The SolrYard allows leading Wildcards (e.g. "*aris"). To provide
+ good query performance for such queries one need to configure
+ fieldTypes that use the ReversedWildcardFilterFactory as shown by
+ this example.
+ See Solr documentation for details
+
+ A general unstemmed text field that indexes tokens normally and also
+ reversed (via ReversedWildcardFilterFactory), to enable more efficient
+ leading wildcard queries.
+ -->
+ <!--
+ <fieldType name="text_rev" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ReversedWildcardFilterFactory" withOriginal="true" maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ -->
+ <!-- charFilter + WhitespaceTokenizer -->
+ <!--
+ <fieldType name="textCharNorm" class="solr.TextField" positionIncrementGap="100" >
+ <analyzer>
+ <charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+ -->
+
+ <!--
+ This can be used to deactivate some functionality of the SolrYard or
+ to configure that some fields of a data set are not stored nor indexed
+ regardless of the Apache Stanbol Entityhub configuration!
+ -->
+ <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
+
+ </types>
+
+
+ <fields>
+ <!--
+ For Information about the different attributes for fields
+ see http://wiki.apache.org/solr/SchemaXml.
+ -->
+ <!--
+ The uri field is used as ID for documents indexed by the SolrYard.
+ Do not change this definition!
+ -->
+ <field name="uri" type="string" indexed="true" stored="true" required="true" />
+
+ <!--
+ used to index all natural language texts of all fields of a document
+ (via copyField). This is used as default search field.
+ The type may be changed.
+ -->
+ <field name="_text" type="textgen" indexed="true" stored="false" multiValued="true"/>
+ <!--
+ used to store all references of the document (via copyField).
+ This field may be used to search for related entities.
+ Do not change this definition!
+ -->
+ <field name="_ref" type="string" indexed="true" stored="false" multiValued="true"/>
+ <!--
+ Field used to store the domain in case multiple datasets are stored witin the same
+ Solr index (search for "Multi Yard Layout" for details).
+ Do not change this definition!
+ -->
+ <field name="_domain" type="string" indexed="true" stored="false" multiValued="true"/>
+
+ <!--
+ Dynamic field definitions (used if a field name is not found)
+ see http://wiki.apache.org/solr/SchemaXml for details.
+
+ The SolrYard heavily uses dynamic fields to index fields based on there
+ data type.
+ Notes:
+ - By default all dynamicField specifications use multiValued="true"
+ because this schema.xml does not make any assumptions on the data stored.
+ Even a boolean field may define both "true" and "false" as values.
+ - The SolrYard currently uses only prefixes to represent data types
+ and languages. However postfixes are reserved for future extensions.
+ That means that postfixes MUST NOT be used for dynamicField definitions
+ -->
+ <!--
+ Used for all fields with boolean data type.
+ One can change multivalued to "false" in case the data indexed do confirm to
+ this restriction
+ -->
+ <dynamicField name="bool/*" type="boolean" indexed="true" stored="true" multiValued="true"/>
+ <!--
+ Numeric dataTypes.
+ Note that thy type can be changed to fieldTypes supporting hierarchical
+ indexing of values to increase performance of ranged queries
+ -->
+ <dynamicField name="int/*" type="int" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="lon/*" type="long" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="flo/*" type="float" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="dou/*" type="double" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="cal/*" type="date" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="dur/*" type="string" indexed="true" stored="true" multiValued="true"/>
+ <!--
+ String fields that are not natural language
+ To support case insensitive searches in such fields change
+ the type to "lowercase"
+ -->
+ <dynamicField name="str/*" type="string" indexed="true" stored="true" multiValued="true"/>
+ <!--
+ references are values that represent IDs of other resources.
+ Typically this will store URIs but in principle also other IDs
+ could be used.
+ -->
+ <dynamicField name="ref/*" type="string" indexed="true" stored="true" multiValued="true"/>
+
+ <!--
+ DynamicFields representing natural language texts.
+
+ The type of such fields may be changed to one of the alternatives
+ descibed in the types section of this configuration.
+
+ The SolrYard prefixes natural language texts with "@"
+ followed by the defined language.
+ Currently no processing of the language is done. So
+ defineing the language "1xx5zr7" for Text will create
+ a field with the prefix "@1xx5zr7".
+
+ To define a special field type for a language one has
+ to use "@" plus the key for the language as prefix.
+ See the specification for English language texts as example.
+
+ Also prefixes for country specific languages can be
+ realized by defining dynamic fields like follows:
+ use @en* to match en-GB and en-US
+ use @en/* to match only en but not en-GB or en-US
+ use @en-GB/* and @en* to have a special field type for
+ en-GB and one for other english text
+ -->
+ <!--
+ Dynamic field for english languages.
+ Note that the prefix "@en*" matches also "@en-GB" and "@en-US"
+ -->
+ <dynamicField name="@en*" type="text_en_Tight" indexed="true" stored="true" multiValued="true"/>
+ <!--
+ The "@*" catches all the other languages including "@/"
+ (default language) used for texts without a defined language
+ -->
+ <dynamicField name="@*" type="textgen" indexed="true" stored="true" multiValued="true"/>
+
+ <!--
+ To add special configurations for specific fields one
+ has to include the fieldName within the prefix of the
+ dynamicField specification.
+ The SolrYard uses namespace prefixes to generate
+ field names. When defined prefixes defined in the
+ NamespaceEnum of the Entityhub are used.
+ Currently there is no way to define used prefixes for
+ other namespaces.
+
+ This example shows how to activate lower case search
+ for the dcmi-terms format property
+ <dynamicField name="str/dc:format*" type="lowercase" indexed="true" stored="true" multiValued="true"/>
+
+ This example shows how to activate fast ranged queries
+ for spatial searches
+ <dynamicField name="dou/geo:lat*" type="tdouble" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="dou/geo:lon*" type="tdouble" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="int/geo:alt*" type="tint" indexed="true" stored="true" multiValued="true"/>
+
+ This example defines to use the type "text_en" for
+ english language rdfs:comment values.
+
+ <dynamicField name="@en/rdfs:comment*" type="text_en" indexed="true" stored="true" multiValued="true"/>
+
+ Note that this would not match "en-GB" nor "en-US".
+ -->
+
+
+ <!--
+ The following dynamic field specifications define fiedls used
+ by the SolrYard implementation for special purposes.
+ -->
+
+ <!--
+ the "_!@*" fields contain a copy of all languages AND string values
+ for that field. This field is used for text queries with no specified
+ language.
+ This field need not to be stored. The type can be changed to alternatives
+ as described in the types section of this configuration.
+ -->
+ <dynamicField name="_!@*" type="textgen" indexed="true" stored="false" multiValued="true"/>
+ <!--
+ fields starting with "_config/" are used to store configurations about how the
+ index was created within the index (e.g. used namespace prefixes).
+ Do not change this definition!
+ -->
+ <dynamicField name="_config/*" type="string" indexed="false" multiValued="true"/>
+
+ </fields>
+
+ <!--
+ Field to use to determine and enforce document uniqueness.
+ -->
+ <uniqueKey>uri</uniqueKey>
+
+ <!--
+ field for the QueryParser to use when an explicit fieldname is absent.
+ The SolrYard does currently not take advantage of this. However it can
+ be used when directly accessing the SolrYard.
+ -->
+ <defaultSearchField>_text</defaultSearchField>
+
+ <!--
+ The SolrYard explizitly adds AND and OR for all boolean terms in
+ generated queries. So changing that should have no influence on
+ the SolrYard (not tested)
+
+ SolrQueryParser configuration: defaultOperator="AND|OR"
+ -->
+ <solrQueryParser defaultOperator="OR"/>
+
+ <!--
+ The SolrYard Implementation assumes the following copyField commands.
+ This commands MUST NOT be removed!
+ -->
+
+ <!--
+ Values of all fields that represent natural language texts
+ or string values are copied to the default search field
+ "_text".
+ Currently the SolrYard does not use this field, but it is
+ reserved for future useage and MUST therefoer already be
+ included when indexing documents
+ -->
+ <copyField source="@*" dest="_text"/>
+ <copyField source="str/*" dest="_text"/>
+ <!--
+ All references to other entities (documents) need to be
+ copied to the "_ref" field.
+ This field is required to query for dependencies of other
+ documents (e.g. when one needs to remove a docuemnt and
+ all references to it)
+ -->
+ <copyField source="ref/*" dest="_ref"/>
+
+</schema>
Propchange: incubator/stanbol/trunk/entityhub/indexing/dblp/src/main/resources/indexing/config/dblp/conf/schema.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain