You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by sf...@apache.org on 2010/12/12 16:13:37 UTC
svn commit: r1044832 [1/14] - in /incubator/stanbol/trunk/rick:
indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/
indexing/genericRdf/src/main/java/eu/iksproject/rick/indexing/rdf/
indexing/geonames/src/main/java/eu/iksproject/ric...
Author: sfermigier
Date: Sun Dec 12 15:13:35 2010
New Revision: 1044832
URL: http://svn.apache.org/viewvc?rev=1044832&view=rev
Log:
Expand tabs. Clean up.
Modified:
incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java
incubator/stanbol/trunk/rick/indexing/genericRdf/src/main/java/eu/iksproject/rick/indexing/rdf/RdfIndexer.java
incubator/stanbol/trunk/rick/indexing/geonames/src/main/java/eu/iksproject/rick/indexing/geonames/GeoNamesIndexer.java
incubator/stanbol/trunk/rick/indexing/geonames/src/main/java/eu/iksproject/rick/indexing/geonames/cli/CommandLineRunner.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JerseyEndpoint.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JerseyEndpointApplication.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JettyServer.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/resource/NavigationMixin.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/resource/ReferencedSiteRootResource.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/resource/RickEntityMappingResource.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/resource/RickSymbolResource.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/resource/SiteManagerRootResource.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/utils/JerseyUtils.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/FieldQueryToJSON.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/JettisonWriter.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/QueryResultListWriter.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/QueryResultsToJSON.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/QueryResultsToRDF.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/SignToJSON.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/SignToRDF.java
incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/writers/SignWriter.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/RdfReference.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/RdfRepresentation.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/RdfResourceUtils.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/RdfText.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/RdfValueFactory.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/impl/Literal2TextAdapter.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/impl/LiteralAdapter.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/impl/NaturalLanguageLiteralIterator.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/impl/ReferenceIterator.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/impl/Resource2ValueAdapter.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/impl/UriRefAdapter.java
incubator/stanbol/trunk/rick/model/clerezza/src/main/java/eu/iksproject/rick/model/clerezza/utils/Resource2StringAdapter.java
incubator/stanbol/trunk/rick/query/clerezza/src/main/java/eu/iksproject/rick/query/clerezza/RdfQueryResultList.java
incubator/stanbol/trunk/rick/query/clerezza/src/main/java/eu/iksproject/rick/query/clerezza/SparqlFieldQuery.java
incubator/stanbol/trunk/rick/query/clerezza/src/main/java/eu/iksproject/rick/query/clerezza/SparqlFieldQueryFactory.java
incubator/stanbol/trunk/rick/query/clerezza/src/main/java/eu/iksproject/rick/query/clerezza/SparqlQueryUtils.java
incubator/stanbol/trunk/rick/site/linkedData/src/main/java/eu/iksproject/rick/site/linkedData/impl/CoolUriDereferencer.java
incubator/stanbol/trunk/rick/site/linkedData/src/main/java/eu/iksproject/rick/site/linkedData/impl/LarqSearcher.java
incubator/stanbol/trunk/rick/site/linkedData/src/main/java/eu/iksproject/rick/site/linkedData/impl/SparqlDereferencer.java
incubator/stanbol/trunk/rick/site/linkedData/src/main/java/eu/iksproject/rick/site/linkedData/impl/SparqlEndpointUtils.java
incubator/stanbol/trunk/rick/site/linkedData/src/main/java/eu/iksproject/rick/site/linkedData/impl/SparqlSearcher.java
incubator/stanbol/trunk/rick/site/linkedData/src/main/java/eu/iksproject/rick/site/linkedData/impl/VirtuosoSearcher.java
incubator/stanbol/trunk/rick/yard/clerezza/src/main/java/eu/iksproject/rick/yard/clerezza/impl/ClerezzaYard.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/defaults/IndexDataTypeEnum.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/defaults/SolrConst.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrFieldMapper.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrQueryFactory.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYard.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYardConfig.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/AssignmentEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/DataTypeEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/FieldEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/GeEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/GtEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/LangEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/LeEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/LtEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/RegexEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/constraintEncoders/WildcardEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/FieldMapper.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/IndexDataType.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/IndexField.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/IndexValue.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/IndexValueFactory.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/NoConverterException.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/TypeConverter.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/UnsupportedIndexTypeException.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/model/UnsupportedValueException.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/query/ConstraintTypePosition.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/query/EncodedConstraintParts.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/query/IndexConstraintTypeEncoder.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/query/IndexConstraintTypeEnum.java
incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/utils/SolrUtil.java
incubator/stanbol/trunk/rick/yard/solr/src/test/java/eu/iksproject/rick/yard/solr/TestInitOutsideOfOsgi.java
Modified: incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java?rev=1044832&r1=1044831&r2=1044832&view=diff
==============================================================================
--- incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java (original)
+++ incubator/stanbol/trunk/rick/indexing/dbPedia/src/main/java/eu/iksproject/rick/indexing/dbPedia/cli/CommandLineRunner.java Sun Dec 12 15:13:35 2010
@@ -34,51 +34,51 @@ import eu.iksproject.rick.yard.solr.impl
import eu.iksproject.rick.yard.solr.impl.SolrYardConfig;
public class CommandLineRunner {
- public static final String[] defaultFieldMappings = new String [] {
- // --- Define the Languages for all fields ---
- //NOTE: the leading space is required for the global filter!
- // --- RDF, RDFS and OWL Mappings ---
- "rdfs:label", //rdf:label
- "rdfs:comment",//rdf:comment
- "rdf:type | d=rick:ref",//The types
- "owl:sameAs | d=rick:ref",//used by LOD to link to URIs used to identify the same Entity
- // --- Dublin Core ---
- "dc:*", //all DC Terms properties
- "dc-elements:*", //all DC Elements (one could also define the mappings to the DC Terms counterparts here
- // --- Spatial Things ---
- "geo:lat | d=xsd:double",
- "geo:long | d=xsd:double",
- "geo:alt | d=xsd:int;xsd:float", //also allow floating point if one needs to use fractions of meters
- // --- Thesaurus (via SKOS) ---
- //SKOS can be used to define hierarchical terminologies
- "skos:*",
- "skos:broader | d=rick:ref",
- "skos:narrower | d=rick:ref",
- "skos:related | d=rick:ref",
- "skos:member | d=rick:ref",
- "skos:subject | d=rick:ref",
- "skos:inScheme | d=rick:ref",
- "skos:hasTopConcept | d=rick:ref",
- "skos:topConceptOf | d=rick:ref",
- // --- Social Networks (via foaf) ---
- "foaf:*", //The Friend of a Friend schema often used to describe social relations between people
- "foaf:knows | d=rick:ref",
- "foaf:made | d=rick:ref",
- "foaf:maker | d=rick:ref",
- "foaf:member | d=rick:ref",
- "foaf:homepage | d=xsd:anyURI",
- "foaf:depiction | d=xsd:anyURI",
- "foaf:img | d=xsd:anyURI",
- "foaf:logo | d=xsd:anyURI",
- "foaf:page | d=xsd:anyURI", //page about the entity
- // --- dbPedia specific
- "dbp-ont:*",
- "dbp-ont:thumbnail | d=xsd:anyURI > foaf:depiction",
-// "dbp-prop:latitude | d=xsd:decimal > geo:lat",
-// "dbp-prop:longitude | d=xsd:decimal > geo:long",
- "dbp-prop:population | d=xsd:integer",
- "dbp-prop:website | d=xsd:anyURI > foaf:homepage"
- };
+ public static final String[] defaultFieldMappings = new String [] {
+ // --- Define the Languages for all fields ---
+ //NOTE: the leading space is required for the global filter!
+ // --- RDF, RDFS and OWL Mappings ---
+ "rdfs:label", //rdf:label
+ "rdfs:comment",//rdf:comment
+ "rdf:type | d=rick:ref",//The types
+ "owl:sameAs | d=rick:ref",//used by LOD to link to URIs used to identify the same Entity
+ // --- Dublin Core ---
+ "dc:*", //all DC Terms properties
+ "dc-elements:*", //all DC Elements (one could also define the mappings to the DC Terms counterparts here
+ // --- Spatial Things ---
+ "geo:lat | d=xsd:double",
+ "geo:long | d=xsd:double",
+ "geo:alt | d=xsd:int;xsd:float", //also allow floating point if one needs to use fractions of meters
+ // --- Thesaurus (via SKOS) ---
+ //SKOS can be used to define hierarchical terminologies
+ "skos:*",
+ "skos:broader | d=rick:ref",
+ "skos:narrower | d=rick:ref",
+ "skos:related | d=rick:ref",
+ "skos:member | d=rick:ref",
+ "skos:subject | d=rick:ref",
+ "skos:inScheme | d=rick:ref",
+ "skos:hasTopConcept | d=rick:ref",
+ "skos:topConceptOf | d=rick:ref",
+ // --- Social Networks (via foaf) ---
+ "foaf:*", //The Friend of a Friend schema often used to describe social relations between people
+ "foaf:knows | d=rick:ref",
+ "foaf:made | d=rick:ref",
+ "foaf:maker | d=rick:ref",
+ "foaf:member | d=rick:ref",
+ "foaf:homepage | d=xsd:anyURI",
+ "foaf:depiction | d=xsd:anyURI",
+ "foaf:img | d=xsd:anyURI",
+ "foaf:logo | d=xsd:anyURI",
+ "foaf:page | d=xsd:anyURI", //page about the entity
+ // --- dbPedia specific
+ "dbp-ont:*",
+ "dbp-ont:thumbnail | d=xsd:anyURI > foaf:depiction",
+// "dbp-prop:latitude | d=xsd:decimal > geo:lat",
+// "dbp-prop:longitude | d=xsd:decimal > geo:long",
+ "dbp-prop:population | d=xsd:integer",
+ "dbp-prop:website | d=xsd:anyURI > foaf:homepage"
+ };
public static final Map<String,Float> fieldBoosts;
static {
Map<String,Float> boosts = new HashMap<String, Float>();
@@ -91,22 +91,22 @@ public class CommandLineRunner {
fieldBoosts = Collections.unmodifiableMap(boosts);
}
- protected static final Logger log = LoggerFactory.getLogger(CommandLineRunner.class);
-
- private static final String header;
- static {
- StringBuilder builder = new StringBuilder();
- builder.append("Description:\nThis Utility creates a full Yard for dbPedia.org by using the SolrYard implementation.\n");
- builder.append("\nParameter:\n");
- builder.append(" - \"-Xmx\": This implementation does not need much memory. RDF data are loaded into the file based Jena TDB store. Indexing is done in chunks of 1000 (default). In case of OutOfMemory errors you need to increase this value!");
- builder.append(" - solrServerUri : The URL of the Solr Server used to index the data. Make sure to use the schema.xml as needed by the SolrYard!\n");
- builder.append(" - dbPediaDumpDir: The relative or absolute path to the Dir with the source RDF files to be used for indexing. You can direcly use the compressed archives. All files in that directory are used to create the index\n");
- builder.append("\nOptions:\n");
- header = builder.toString();
- builder = null;
- }
- private static final Options options;
- static {
+ protected static final Logger log = LoggerFactory.getLogger(CommandLineRunner.class);
+
+ private static final String header;
+ static {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Description:\nThis Utility creates a full Yard for dbPedia.org by using the SolrYard implementation.\n");
+ builder.append("\nParameter:\n");
+ builder.append(" - \"-Xmx\": This implementation does not need much memory. RDF data are loaded into the file based Jena TDB store. Indexing is done in chunks of 1000 (default). In case of OutOfMemory errors you need to increase this value!");
+ builder.append(" - solrServerUri : The URL of the Solr Server used to index the data. Make sure to use the schema.xml as needed by the SolrYard!\n");
+ builder.append(" - dbPediaDumpDir: The relative or absolute path to the Dir with the source RDF files to be used for indexing. You can direcly use the compressed archives. All files in that directory are used to create the index\n");
+ builder.append("\nOptions:\n");
+ header = builder.toString();
+ builder = null;
+ }
+ private static final Options options;
+ static {
options = new Options();
options.addOption("h", "help", false, "display this help and exit");
options.addOption("d", "debug", false, "show debug stacktrace upon error");
@@ -119,20 +119,20 @@ public class CommandLineRunner {
options.addOption("i","incomming",true,"the file with the incomming links for Entities (id tab num, highest num needs to be the first line!)");
options.addOption("ri","requiredIncomming",true,"the minimum number of incomming lins for Entities to be indexed");
options.addOption("r","resume",true,"resume a previous canceled indexing session (usually used with -s)");
- }
- private static final String footer;
- static {
- StringBuilder builder = new StringBuilder();
- builder.append("Default Field Mappings:\n");
- for(String mapping: defaultFieldMappings){
- builder.append(String.format("\t%s",mapping));
- }
- footer = builder.toString();
- builder = null;
- }
-
- private static float minRequiredRanking;
-
+ }
+ private static final String footer;
+ static {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Default Field Mappings:\n");
+ for(String mapping: defaultFieldMappings){
+ builder.append(String.format("\t%s",mapping));
+ }
+ footer = builder.toString();
+ builder = null;
+ }
+
+ private static float minRequiredRanking;
+
public static void main(String[] args) throws IOException, ParseException, YardException {
CommandLineParser parser = new PosixParser();
CommandLine line = parser.parse(options, args);
@@ -149,7 +149,7 @@ public class CommandLineRunner {
}
String yardName = line.getOptionValue("n");
if(yardName == null){
- yardName = "dbPedia";
+ yardName = "dbPedia";
}
Dictionary<String, Object> indexingConfig = new Hashtable<String, Object>();
//first the SolrServer used to store the index
@@ -164,78 +164,78 @@ public class CommandLineRunner {
//now the other properties
File dataDir = new File(line.getArgs()[1]);
if(!dataDir.exists()){
- log.error("Parsed Data Directory "+dataDir+" does not Exist on the File System");
- System.exit(0);
+ log.error("Parsed Data Directory "+dataDir+" does not Exist on the File System");
+ System.exit(0);
}
if(!dataDir.isDirectory()){
- log.error("Parsed Data Directory "+dataDir+" exists, but is not a Directory!");
- System.exit(0);
+ log.error("Parsed Data Directory "+dataDir+" exists, but is not a Directory!");
+ System.exit(0);
}
if(!dataDir.canRead()){
- log.error("Unable to read Data Directory "+dataDir+"!");
- System.exit(0);
+ log.error("Unable to read Data Directory "+dataDir+"!");
+ System.exit(0);
}
File[] files = dataDir.listFiles();
indexingConfig.put(KEY_RDF_FILES, Arrays.asList(files));
indexingConfig.put(KEY_RDF_STORE_DIR, "dbPedia-rdf-data");
Integer chunkSize;
try {
- chunkSize = Integer.valueOf(line.getOptionValue("c", "1000"));
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Value for option \"chunkSize\" need to be a valid Integer");
- }
- if(chunkSize<0){
- log.warn("Negative number parsed for option \"chunkSize\". Use '1000' as default.");
- chunkSize = 1000;
- }
- if(!line.hasOption("m")){
- indexingConfig.put(KEY_FIELD_MAPPINGS, Arrays.asList(defaultFieldMappings));
- } else {
- File fieldMappingFile = new File(line.getOptionValue("m"));
- if(fieldMappingFile.exists() && fieldMappingFile.isFile() && fieldMappingFile.canRead()){
- String[] fieldMappings = IOUtils.toString(new FileInputStream(fieldMappingFile)).split("\n");
- indexingConfig.put(KEY_FIELD_MAPPINGS, Arrays.asList(fieldMappings));
- } else {
- log.error(String.format("Invalid fieldMapping File (exist: %s | isFile: %s | canRead: %s)",
- fieldMappingFile.exists(),fieldMappingFile.isFile(),fieldMappingFile.canRead()));
- System.exit(0);
- }
-
- }
- if(line.hasOption("s")){
- indexingConfig.put(KEY_SKIP_READ, Boolean.TRUE);
- }
+ chunkSize = Integer.valueOf(line.getOptionValue("c", "1000"));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Value for option \"chunkSize\" need to be a valid Integer");
+ }
+ if(chunkSize<0){
+ log.warn("Negative number parsed for option \"chunkSize\". Use '1000' as default.");
+ chunkSize = 1000;
+ }
+ if(!line.hasOption("m")){
+ indexingConfig.put(KEY_FIELD_MAPPINGS, Arrays.asList(defaultFieldMappings));
+ } else {
+ File fieldMappingFile = new File(line.getOptionValue("m"));
+ if(fieldMappingFile.exists() && fieldMappingFile.isFile() && fieldMappingFile.canRead()){
+ String[] fieldMappings = IOUtils.toString(new FileInputStream(fieldMappingFile)).split("\n");
+ indexingConfig.put(KEY_FIELD_MAPPINGS, Arrays.asList(fieldMappings));
+ } else {
+ log.error(String.format("Invalid fieldMapping File (exist: %s | isFile: %s | canRead: %s)",
+ fieldMappingFile.exists(),fieldMappingFile.isFile(),fieldMappingFile.canRead()));
+ System.exit(0);
+ }
+
+ }
+ if(line.hasOption("s")){
+ indexingConfig.put(KEY_SKIP_READ, Boolean.TRUE);
+ }
indexingConfig.put(KEY_CHUNK_SIZE, chunkSize);
indexingConfig.put(KEY_MODEL_NAME, "indexingModel-49e176b9-0138-dd4c-2b87-89af85b89a57");
//entityRank related properties
if(line.hasOption("i")){
- File tsvScoreFile = new File(line.getOptionValue("i"));
- if(tsvScoreFile.exists() && tsvScoreFile.isFile() && tsvScoreFile.canRead()){
- int minIncommings = -1;
- try {
- minIncommings = Integer.parseInt(line.getOptionValue("ri", "-1"));
- } catch (Exception e) {
- log.error("Value of option --minIncomming/-mi MUST BE a valid integer");
- System.exit(0);
- }
+ File tsvScoreFile = new File(line.getOptionValue("i"));
+ if(tsvScoreFile.exists() && tsvScoreFile.isFile() && tsvScoreFile.canRead()){
+ int minIncommings = -1;
+ try {
+ minIncommings = Integer.parseInt(line.getOptionValue("ri", "-1"));
+ } catch (Exception e) {
+ log.error("Value of option --minIncomming/-mi MUST BE a valid integer");
+ System.exit(0);
+ }
Map<String,Float> entityRankings = clacEntityRanks(tsvScoreFile,minIncommings);
indexingConfig.put(KEY_ENTITY_RANKINGS, entityRankings);
log.info(String.format(" ... set min required score to %s (represents %s incomming links",minRequiredRanking,minIncommings));
indexingConfig.put(KEY_REQUIRED_ENTITY_RANKING, minRequiredRanking);
- } else {
- log.error(String.format("Parsed File with the incommung links is invalid (esists:%s,isFile:%s,canRead:%s)",
- tsvScoreFile.exists(), tsvScoreFile.isFile(), tsvScoreFile.canRead()));
- }
+ } else {
+ log.error(String.format("Parsed File with the incommung links is invalid (esists:%s,isFile:%s,canRead:%s)",
+ tsvScoreFile.exists(), tsvScoreFile.isFile(), tsvScoreFile.canRead()));
+ }
} else {
- if(line.hasOption("ri")){
- log.warn("Option --requiredIncomming/-ri is only supported of Option --incomming/-i is active!");
- }
+ if(line.hasOption("ri")){
+ log.warn("Option --requiredIncomming/-ri is only supported of Option --incomming/-i is active!");
+ }
}
//THis mode uses the id of the entity rnking map as main lookup for
//entities to index. This is faster than the usual mode if less than
//50% of the entities are indexed!
if(line.hasOption("r")){
- //resume makes only really sense with the RANKING BASED MODE
+ //resume makes only really sense with the RANKING BASED MODE
indexingConfig.put(KEY_ENTITY_RANKING_BASED_INDEXING_MODE, Boolean.TRUE);
//set the RESUME MODE
indexingConfig.put(KEY_RESUME_MODE, Boolean.TRUE);
@@ -244,9 +244,9 @@ public class CommandLineRunner {
indexer.index();
}
@SuppressWarnings("unchecked")
- private static Map<String,Float> clacEntityRanks(File tsvScoreFile,int minIncommings) throws IOException {
- TreeMap<String,Float> entityRankings = new TreeMap<String, Float>();
- final Iterator<String> lines = IOUtils.lineIterator(
+ private static Map<String,Float> clacEntityRanks(File tsvScoreFile,int minIncommings) throws IOException {
+ TreeMap<String,Float> entityRankings = new TreeMap<String, Float>();
+ final Iterator<String> lines = IOUtils.lineIterator(
new FileInputStream(tsvScoreFile), "utf-8");
long lineNumber = 0;
int maxIncommung = 0;
@@ -265,34 +265,34 @@ public class CommandLineRunner {
// culturally generated distribution)
float score = (float)Math.log1p(incomming);
if (lineNumber == 0 && score > 0) {
- maxIncommung = incomming;
+ maxIncommung = incomming;
maxScore = score;
if(minIncommings <= 0){
- minRequiredRanking = -1f; //deactivate
+ minRequiredRanking = -1f; //deactivate
} else {
- float min = (float)Math.log1p(minIncommings);
- minRequiredRanking = min/maxScore;
- if(minRequiredRanking > 1){
- log.error("Parsed minimum required incomming links is bigger than the highest number of incomming links for any entity!");
- System.exit(0);
- }
+ float min = (float)Math.log1p(minIncommings);
+ minRequiredRanking = min/maxScore;
+ if(minRequiredRanking > 1){
+ log.error("Parsed minimum required incomming links is bigger than the highest number of incomming links for any entity!");
+ System.exit(0);
+ }
}
}
score = score/maxScore;
if(score > 1){
- log.error("Found Entity wiht more incomming links than the entity in the first line");
- log.error("current:"+line);
- System.exit(0);
+ log.error("Found Entity wiht more incomming links than the entity in the first line");
+ log.error("current:"+line);
+ System.exit(0);
}
if(score >= minRequiredRanking){
- entityRankings.put(parts[0], score);
+ entityRankings.put(parts[0], score);
} else {
- filtered ++;
+ filtered ++;
}
lineNumber++;
}
log.info(String.format(" ... processed %s entities (%s with ranking > required | %s filtered",
- lineNumber,lineNumber-filtered,filtered));
+ lineNumber,lineNumber-filtered,filtered));
return entityRankings;
}