You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rya.apache.org by mi...@apache.org on 2016/08/26 00:07:07 UTC
[46/52] [abbrv] incubator-rya git commit: RYA-101 RYA-102 and RYA-133,
fixing issues with secondary indexers
RYA-101 RYA-102 and RYA-133, fixing issues with secondary indexers
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/8597b78b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/8597b78b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/8597b78b
Branch: refs/heads/master
Commit: 8597b78b35cbfc0385bb5e74765926df21149237
Parents: 573aedf
Author: David W. Lotts <da...@parsons.com>
Authored: Wed Aug 17 15:26:07 2016 -0400
Committer: Aaron Mihalik <mi...@alum.mit.edu>
Committed: Tue Aug 23 10:50:49 2016 -0400
----------------------------------------------------------------------
.../mvm/rya/mongodb/MongoConnectorFactory.java | 109 ++++++++++++++++---
.../rya/mongodb/MongoDBRdfConfiguration.java | 6 +-
.../java/mvm/rya/mongodb/MongoDBRyaDAO.java | 26 +++--
.../mvm/rya/mongodb/MongoSecondaryIndex.java | 12 ++
.../rya/indexing/FilterFunctionOptimizer.java | 20 +---
.../indexing/mongodb/AbstractMongoIndexer.java | 44 ++++----
.../mongodb/freetext/MongoFreeTextIndexer.java | 9 +-
.../indexing/mongodb/geo/MongoGeoIndexer.java | 11 +-
.../mongodb/temporal/MongoTemporalIndexer.java | 15 +--
.../mongo/MongoFreeTextIndexerTest.java | 16 +--
.../indexing/mongo/MongoGeoIndexerSfTest.java | 4 +-
.../rya/indexing/mongo/MongoGeoIndexerTest.java | 36 +++---
.../mongo/MongoTemporalIndexerTest.java | 4 +-
.../src/main/java/MongoRyaDirectExample.java | 19 +++-
14 files changed, 204 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java
index 747ad1e..b91c9df 100644
--- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java
+++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoConnectorFactory.java
@@ -21,13 +21,19 @@ package mvm.rya.mongodb;
import java.net.UnknownHostException;
import java.util.Arrays;
+import java.io.IOException;
+import org.apache.commons.configuration.ConfigurationRuntimeException;
import org.apache.hadoop.conf.Configuration;
import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
+import com.mongodb.MongoException;
import com.mongodb.ServerAddress;
+import de.flapdoodle.embed.mongo.distribution.Version;
+import de.flapdoodle.embed.mongo.tests.MongodForTestsFactory;
+
/**
* Mongo convention generally allows for a single instance of a {@link MongoClient}
* throughout the life cycle of an application. This MongoConnectorFactory lazy
@@ -37,30 +43,101 @@ import com.mongodb.ServerAddress;
public class MongoConnectorFactory {
private static MongoClient mongoClient;
+ private final static String MSG_INTRO = "Failed to connect to MongoDB: ";
+
/**
* @param conf The {@link Configuration} defining how to construct the MongoClient.
* @return A {@link MongoClient}. This client is lazy loaded and the same one
* is used throughout the lifecycle of the application.
- * @throws NumberFormatException - Thrown if the configured port is not a valid number
- * @throws UnknownHostException - The configured host cannot be found.
+ * @throws IOException - if MongodForTestsFactory constructor has an io exception.
+ * @throws ConfigurationRuntimeException - Thrown if the configured server, port, user, or others are missing.
+ * @throws MongoException if can't connect despite conf parameters are given
*/
- public static synchronized MongoClient getMongoClient(final Configuration conf) throws NumberFormatException, UnknownHostException {
- if(mongoClient == null) {
- final String host = conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE);
- final int port = Integer.parseInt(conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE_PORT));
- final ServerAddress server = new ServerAddress(host, port);
-
- //check for authentication credentials
- if (conf.get(MongoDBRdfConfiguration.MONGO_USER) != null) {
- final String username = conf.get(MongoDBRdfConfiguration.MONGO_USER);
- final String dbName = conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME);
- final char[] pswd = conf.get(MongoDBRdfConfiguration.MONGO_USER_PASSWORD).toCharArray();
- final MongoCredential cred = MongoCredential.createCredential(username, dbName, pswd);
- mongoClient = new MongoClient(server, Arrays.asList(cred));
+ public static synchronized MongoClient getMongoClient(final Configuration conf)
+ throws ConfigurationRuntimeException, MongoException {
+ if (mongoClient == null) {
+ // The static client has not yet created, is it a test/mock instance, or a service?
+ if (conf.getBoolean(MongoDBRdfConfiguration.USE_TEST_MONGO, false)) {
+ createMongoClientForTests();
} else {
- mongoClient = new MongoClient(server);
+ createMongoClientForServer(conf);
}
}
return mongoClient;
}
+
+ /**
+ * Create a local temporary MongoDB instance and client object and assign it to this class's static mongoClient
+ * @throws MongoException if can't connect
+ */
+ private static void createMongoClientForTests() throws MongoException {
+ try {
+ MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION);
+ mongoClient = testsFactory.newMongo();
+ } catch (IOException e) {
+ // Rethrow as an unchecked error. Since we are in a test mode here, just fail fast.
+ throw new MongoException(MSG_INTRO+"creating a factory for a test/mock MongoDB instance.",e);
+ }
+ }
+
+ /**
+ * Create a MongoDB client object and assign it to this class's static mongoClient
+ * @param conf configuration containing connection parameters
+ * @throws ConfigurationRuntimeException - Thrown if the configured server, port, user, or others are missing.
+ * @throws MongoException if can't connect despite conf parameters are given
+ */
+ private static void createMongoClientForServer(final Configuration conf)
+ throws ConfigurationRuntimeException, MongoException {
+ // Connect to a running Mongo server
+ final String host = requireNonNull(conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE), MSG_INTRO+"host name is required");
+ final int port = requireNonNullInt(conf.get(MongoDBRdfConfiguration.MONGO_INSTANCE_PORT), MSG_INTRO+"Port number is required.");
+ ServerAddress server;
+ try {
+ server = new ServerAddress(host, port);
+ } catch (UnknownHostException e) {
+ throw new MongoException(MSG_INTRO + "cannot find host="+host,e);
+ }
+ // check for authentication credentials
+ if (conf.get(MongoDBRdfConfiguration.MONGO_USER) != null) {
+ final String username = conf.get(MongoDBRdfConfiguration.MONGO_USER);
+ final String dbName = requireNonNull(conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME),
+ MSG_INTRO + MongoDBRdfConfiguration.MONGO_DB_NAME + " is null but required configuration if "
+ + MongoDBRdfConfiguration.MONGO_USER + " is configured.");
+ final char[] pswd = requireNonNull(conf.get(MongoDBRdfConfiguration.MONGO_USER_PASSWORD),
+ MSG_INTRO + MongoDBRdfConfiguration.MONGO_USER_PASSWORD + " is null but required configuration if "
+ + MongoDBRdfConfiguration.MONGO_USER + " is configured.").toCharArray();
+ final MongoCredential cred = MongoCredential.createCredential(username, dbName, pswd);
+ mongoClient = new MongoClient(server, Arrays.asList(cred));
+ } else {
+ // No user was configured:
+ mongoClient = new MongoClient(server);
+ }
+ }
+
+ /**
+ * Throw exception for un-configured required values.
+ *
+ * @param required String to check
+ * @param message throw configuration exception with this description
+ * @return unaltered required string
+ * @throws ConfigurationRuntimeException if required is null
+ */
+ private static String requireNonNull(String required, String message) throws ConfigurationRuntimeException {
+ if (required == null)
+ throw new ConfigurationRuntimeException(message);
+ return required;
+ }
+
+ /*
+ * Same as above, check that it is a integer and return the parsed integer.
+ */
+ private static int requireNonNullInt(String required, String message) throws ConfigurationRuntimeException {
+ if (required == null)
+ throw new ConfigurationRuntimeException(message);
+ try {
+ return Integer.parseInt(required);
+ } catch (NumberFormatException e) {
+ throw new ConfigurationRuntimeException(message);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java
index 0c38337..e8e301d 100644
--- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java
+++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRdfConfiguration.java
@@ -104,7 +104,7 @@ public class MongoDBRdfConfiguration extends RdfCloudTripleStoreConfiguration {
return this.get(MONGO_COLLECTION_PREFIX, "rya") + "_ns";
}
- public void setAdditionalIndexers(Class<? extends RyaSecondaryIndexer>... indexers) {
+ public void setAdditionalIndexers(Class<? extends MongoSecondaryIndex>... indexers) {
List<String> strs = Lists.newArrayList();
for (Class<?> ai : indexers){
strs.add(ai.getName());
@@ -113,8 +113,8 @@ public class MongoDBRdfConfiguration extends RdfCloudTripleStoreConfiguration {
setStrings(CONF_ADDITIONAL_INDEXERS, strs.toArray(new String[]{}));
}
- public List<RyaSecondaryIndexer> getAdditionalIndexers() {
- return getInstances(CONF_ADDITIONAL_INDEXERS, RyaSecondaryIndexer.class);
+ public List<MongoSecondaryIndex> getAdditionalIndexers() {
+ return getInstances(CONF_ADDITIONAL_INDEXERS, MongoSecondaryIndex.class);
}
public void setMongoClient(MongoClient client){
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java
index 15537e5..b0fd1db 100644
--- a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java
+++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoDBRyaDAO.java
@@ -32,8 +32,10 @@ import org.apache.log4j.Logger;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;
+import com.mongodb.DuplicateKeyException;
import com.mongodb.InsertOptions;
import com.mongodb.MongoClient;
+import com.mongodb.MongoException.DuplicateKey;
import de.flapdoodle.embed.mongo.tests.MongodForTestsFactory;
import mvm.rya.api.RdfCloudTripleStoreConfiguration;
@@ -64,7 +66,7 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{
private MongoDBNamespaceManager nameSpaceManager;
private MongodForTestsFactory testsFactory;
- private List<RyaSecondaryIndexer> secondaryIndexers;
+ private List<MongoSecondaryIndex> secondaryIndexers;
/**
* Creates a new {@link MongoDBRyaDAO}
@@ -73,14 +75,9 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{
*/
public MongoDBRyaDAO(final MongoDBRdfConfiguration conf) throws RyaDAOException, NumberFormatException, UnknownHostException {
this.conf = conf;
- try {
- mongoClient = MongoConnectorFactory.getMongoClient(conf);
- conf.setMongoClient(mongoClient);
- init();
- } catch (NumberFormatException | UnknownHostException e) {
- log.error("Unable to create a connection to mongo.", e);
- throw e;
- }
+ mongoClient = MongoConnectorFactory.getMongoClient(conf);
+ conf.setMongoClient(mongoClient);
+ init();
}
@@ -117,8 +114,9 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{
@Override
public void init() throws RyaDAOException {
secondaryIndexers = conf.getAdditionalIndexers();
- for(final RyaSecondaryIndexer index: secondaryIndexers) {
+ for(final MongoSecondaryIndex index: secondaryIndexers) {
index.setConf(conf);
+ index.setClient(mongoClient);
}
db = mongoClient.getDB(conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME));
@@ -127,6 +125,9 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{
queryEngine = new MongoDBQueryEngine(conf, mongoClient);
storageStrategy = new SimpleMongoDBStorageStrategy();
storageStrategy.createIndices(coll);
+ for(final MongoSecondaryIndex index: secondaryIndexers) {
+ index.init();
+ }
}
@Override
@@ -154,10 +155,13 @@ public final class MongoDBRyaDAO implements RyaDAO<MongoDBRdfConfiguration>{
for(final RyaSecondaryIndexer index: secondaryIndexers) {
index.storeStatement(statement);
}
- } catch (final IOException e) {
+ } catch (IOException e) {
log.error("Unable to add: " + statement.toString());
throw new RyaDAOException(e);
}
+ catch (DuplicateKeyException e){
+ log.error("Attempting to load duplicate triple: " + statement.toString());
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java
new file mode 100644
index 0000000..553a78f
--- /dev/null
+++ b/dao/mongodb.rya/src/main/java/mvm/rya/mongodb/MongoSecondaryIndex.java
@@ -0,0 +1,12 @@
+package mvm.rya.mongodb;
+
+import com.mongodb.MongoClient;
+
+import mvm.rya.api.persist.index.RyaSecondaryIndexer;
+
+public interface MongoSecondaryIndex extends RyaSecondaryIndexer{
+ public void init();
+
+ public void setClient(MongoClient client);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java
index ad9fe22..6c0cd9a 100644
--- a/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java
+++ b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java
@@ -68,6 +68,7 @@ import mvm.rya.indexing.accumulo.freetext.FreeTextTupleSet;
import mvm.rya.indexing.accumulo.geo.GeoMesaGeoIndexer;
import mvm.rya.indexing.accumulo.geo.GeoTupleSet;
import mvm.rya.indexing.accumulo.temporal.AccumuloTemporalIndexer;
+import mvm.rya.indexing.mongodb.AbstractMongoIndexer;
import mvm.rya.indexing.mongodb.freetext.MongoFreeTextIndexer;
import mvm.rya.indexing.mongodb.geo.MongoGeoIndexer;
import mvm.rya.indexing.mongodb.temporal.MongoTemporalIndexer;
@@ -99,29 +100,18 @@ public class FilterFunctionOptimizer implements QueryOptimizer, Configurable {
this.conf = conf;
//reset the init.
init = false;
- try {
init();
- } catch (final NumberFormatException | UnknownHostException e) {
- LOG.error("Unable to update to use new config, falling back to the old config.", e);
- init = true;
- }
}
- private synchronized void init() throws NumberFormatException, UnknownHostException {
+ private synchronized void init() {
if (!init) {
if (ConfigUtils.getUseMongo(conf)) {
- try {
- final MongoClient mongoClient = MongoConnectorFactory.getMongoClient(conf);
- geoIndexer = new MongoGeoIndexer(mongoClient);
+ geoIndexer = new MongoGeoIndexer();
geoIndexer.setConf(conf);
- freeTextIndexer = new MongoFreeTextIndexer(mongoClient);
+ freeTextIndexer = new MongoFreeTextIndexer();
freeTextIndexer.setConf(conf);
- temporalIndexer = new MongoTemporalIndexer(mongoClient);
+ temporalIndexer = new MongoTemporalIndexer();
temporalIndexer.setConf(conf);
- } catch (NumberFormatException | UnknownHostException e) {
- LOG.error("Unable to connect to mongo.", e);
- throw e;
- }
} else {
geoIndexer = new GeoMesaGeoIndexer();
geoIndexer.setConf(conf);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java
index 078cbd2..daa78e4 100644
--- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java
+++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java
@@ -1,7 +1,5 @@
package mvm.rya.indexing.mongodb;
-import static com.google.common.base.Preconditions.checkNotNull;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -42,20 +40,23 @@ import com.mongodb.QueryBuilder;
import info.aduna.iteration.CloseableIteration;
import mvm.rya.api.domain.RyaStatement;
import mvm.rya.api.domain.RyaURI;
-import mvm.rya.api.persist.index.RyaSecondaryIndexer;
import mvm.rya.api.resolver.RyaToRdfConversions;
import mvm.rya.indexing.StatementConstraints;
+import mvm.rya.mongodb.MongoConnectorFactory;
import mvm.rya.mongodb.MongoDBRdfConfiguration;
+import mvm.rya.mongodb.MongoDBRyaDAO;
+import mvm.rya.mongodb.MongoSecondaryIndex;
/**
* Secondary Indexer using MondoDB
* @param <T> - The {@link AbstractMongoIndexingStorageStrategy} this indexer uses.
*/
-public abstract class AbstractMongoIndexer<T extends IndexingMongoDBStorageStrategy> implements RyaSecondaryIndexer {
+public abstract class AbstractMongoIndexer<T extends IndexingMongoDBStorageStrategy> implements MongoSecondaryIndex {
private static final Logger LOG = Logger.getLogger(AbstractMongoIndexer.class);
private boolean isInit = false;
protected Configuration conf;
+ protected MongoDBRyaDAO dao;
protected MongoClient mongoClient;
protected String dbName;
protected DB db;
@@ -64,31 +65,32 @@ public abstract class AbstractMongoIndexer<T extends IndexingMongoDBStorageStrat
protected T storageStrategy;
- /**
- * Creates a new {@link AbstractMongoIndexer} with the provided mongo client.
- * @param mongoClient The {@link MongoClient} to use with this indexer.
- */
- public AbstractMongoIndexer(final MongoClient mongoClient) {
- this.mongoClient = checkNotNull(mongoClient);
- }
-
- protected void init() throws NumberFormatException, IOException{
+ protected void initCore() {
dbName = conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME);
db = this.mongoClient.getDB(dbName);
collection = db.getCollection(conf.get(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya") + getCollectionName());
}
+
+ public void setClient(MongoClient client){
+ this.mongoClient = client;
+ }
+
+ // TODO this method is only intended to be used in testing
+ public void initIndexer(final Configuration conf, final MongoClient client) {
+ setConf(conf);
+ setClient(client);
+ if (!isInit) {
+ init();
+ isInit = true;
+ }
+ }
@Override
public void setConf(final Configuration conf) {
this.conf = conf;
- if (!isInit) {
- try {
- init();
- isInit = true;
- } catch (final NumberFormatException | IOException e) {
- LOG.warn("Unable to initialize index. Throwing Runtime Exception. ", e);
- throw new RuntimeException(e);
- }
+ if (!isInit){
+ setClient(MongoConnectorFactory.getMongoClient(conf));
+ init();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java
index 7a65d26..0689feb 100644
--- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java
+++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/freetext/MongoFreeTextIndexer.java
@@ -24,7 +24,6 @@ import org.apache.log4j.Logger;
import org.openrdf.model.Statement;
import org.openrdf.query.QueryEvaluationException;
-import com.mongodb.MongoClient;
import com.mongodb.QueryBuilder;
import info.aduna.iteration.CloseableIteration;
@@ -37,13 +36,9 @@ public class MongoFreeTextIndexer extends AbstractMongoIndexer<TextMongoDBStorag
private static final String COLLECTION_SUFFIX = "freetext";
private static final Logger logger = Logger.getLogger(MongoFreeTextIndexer.class);
- public MongoFreeTextIndexer(final MongoClient mongoClient) {
- super(mongoClient);
- }
-
@Override
- protected void init() throws IOException{
- super.init();
+ public void init() {
+ initCore();
predicates = ConfigUtils.getFreeTextPredicates(conf);
storageStrategy = new TextMongoDBStorageStrategy();
storageStrategy.createIndices(collection);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java
index 303848a..7589f03 100644
--- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java
+++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/geo/MongoGeoIndexer.java
@@ -22,14 +22,11 @@ import static mvm.rya.indexing.mongodb.geo.GeoMongoDBStorageStrategy.GeoQueryTyp
import static mvm.rya.indexing.mongodb.geo.GeoMongoDBStorageStrategy.GeoQueryType.INTERSECTS;
import static mvm.rya.indexing.mongodb.geo.GeoMongoDBStorageStrategy.GeoQueryType.WITHIN;
-import java.io.IOException;
-
import org.apache.log4j.Logger;
import org.openrdf.model.Statement;
import org.openrdf.query.QueryEvaluationException;
import com.mongodb.DBObject;
-import com.mongodb.MongoClient;
import com.vividsolutions.jts.geom.Geometry;
import info.aduna.iteration.CloseableIteration;
@@ -44,13 +41,9 @@ public class MongoGeoIndexer extends AbstractMongoIndexer<GeoMongoDBStorageStrat
private static final String COLLECTION_SUFFIX = "geo";
private static final Logger logger = Logger.getLogger(MongoGeoIndexer.class);
- public MongoGeoIndexer(final MongoClient mongoClient) {
- super(mongoClient);
- }
-
@Override
- protected void init() throws NumberFormatException, IOException{
- super.init();
+ public void init() {
+ initCore();
predicates = ConfigUtils.getGeoPredicates(conf);
storageStrategy = new GeoMongoDBStorageStrategy(Double.valueOf(conf.get(MongoDBRdfConfiguration.MONGO_GEO_MAXDISTANCE, "1e-10")));
storageStrategy.createIndices(collection);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java
index 003d9c9..be991c8 100644
--- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java
+++ b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/temporal/MongoTemporalIndexer.java
@@ -22,15 +22,12 @@ import static mvm.rya.indexing.mongodb.temporal.TemporalMongoDBStorageStrategy.I
import static mvm.rya.indexing.mongodb.temporal.TemporalMongoDBStorageStrategy.INTERVAL_END;
import static mvm.rya.indexing.mongodb.temporal.TemporalMongoDBStorageStrategy.INTERVAL_START;
-import java.io.IOException;
-
import org.apache.log4j.Logger;
import org.openrdf.model.Statement;
import org.openrdf.query.QueryEvaluationException;
import com.google.common.annotations.VisibleForTesting;
import com.mongodb.DBCollection;
-import com.mongodb.MongoClient;
import com.mongodb.QueryBuilder;
import info.aduna.iteration.CloseableIteration;
@@ -48,17 +45,9 @@ public class MongoTemporalIndexer extends AbstractMongoIndexer<TemporalMongoDBSt
private static final String COLLECTION_SUFFIX = "temporal";
private static final Logger LOG = Logger.getLogger(MongoTemporalIndexer.class);
- /**
- * Creates a new {@link MongoTemporalIndexer}
- * @param mongoClient - The {@link MongoClient} used to interact with MongoDB.
- */
- public MongoTemporalIndexer(final MongoClient mongoClient) {
- super(mongoClient);
- }
-
@Override
- protected void init() throws IOException{
- super.init();
+ public void init() {
+ initCore();
predicates = ConfigUtils.getTemporalPredicates(conf);
storageStrategy = new TemporalMongoDBStorageStrategy();
storageStrategy.createIndices(collection);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java
index 91ba955..66ddc33 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoFreeTextIndexerTest.java
@@ -69,8 +69,8 @@ public class MongoFreeTextIndexerTest {
@Test
public void testSearch() throws Exception {
- try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
@@ -93,8 +93,8 @@ public class MongoFreeTextIndexerTest {
@Test
public void testDelete() throws Exception {
- try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
@@ -143,8 +143,8 @@ public class MongoFreeTextIndexerTest {
public void testRestrictPredicatesSearch() throws Exception {
conf.setStrings(ConfigUtils.FREETEXT_PREDICATES_LIST, "pred:1,pred:2");
- try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) {
+ f.initIndexer(conf, mongoClient);
// These should not be stored because they are not in the predicate list
f.storeStatement(new RyaStatement(new RyaURI("foo:subj1"), new RyaURI(RDFS.LABEL.toString()), new RyaType("invalid")));
@@ -176,8 +176,8 @@ public class MongoFreeTextIndexerTest {
@Test
public void testContextSearch() throws Exception {
- try (MongoFreeTextIndexer f = new MongoFreeTextIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoFreeTextIndexer f = new MongoFreeTextIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final URI subject = new URIImpl("foo:subj");
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java
index 3368d6e..fe6b51a 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerSfTest.java
@@ -121,8 +121,8 @@ public class MongoGeoIndexerSfTest {
final MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION);
final MongoClient mongoClient = testsFactory.newMongo();
- g = new MongoGeoIndexer(mongoClient);
- g.setConf(conf);
+ g = new MongoGeoIndexer();
+ g.initIndexer(conf, mongoClient);
g.storeStatement(statement(A));
g.storeStatement(statement(B));
g.storeStatement(statement(C));
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java
index 7a2dc8e..7a20deb 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoGeoIndexerTest.java
@@ -85,8 +85,8 @@ public class MongoGeoIndexerTest {
@Test
public void testRestrictPredicatesSearch() throws Exception {
conf.setStrings(ConfigUtils.GEO_PREDICATES_LIST, "pred:1,pred:2");
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
@@ -132,8 +132,8 @@ public class MongoGeoIndexerTest {
@Test
public void testPrimeMeridianSearch() throws Exception {
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource subject = vf.createURI("foo:subj");
@@ -176,8 +176,8 @@ public class MongoGeoIndexerTest {
@Test
public void testDcSearch() throws Exception {
// test a ring around dc
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource subject = vf.createURI("foo:subj");
@@ -205,8 +205,8 @@ public class MongoGeoIndexerTest {
@Test
public void testDeleteSearch() throws Exception {
// test a ring around dc
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource subject = vf.createURI("foo:subj");
@@ -244,8 +244,8 @@ public class MongoGeoIndexerTest {
@Test
public void testDcSearchWithContext() throws Exception {
// test a ring around dc
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource subject = vf.createURI("foo:subj");
@@ -273,8 +273,8 @@ public class MongoGeoIndexerTest {
@Test
public void testDcSearchWithSubject() throws Exception {
// test a ring around dc
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource subject = vf.createURI("foo:subj");
@@ -301,8 +301,8 @@ public class MongoGeoIndexerTest {
@Test
public void testDcSearchWithSubjectAndContext() throws Exception {
// test a ring around dc
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource subject = vf.createURI("foo:subj");
@@ -334,8 +334,8 @@ public class MongoGeoIndexerTest {
@Test
public void testDcSearchWithPredicate() throws Exception {
// test a ring around dc
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource subject = vf.createURI("foo:subj");
@@ -364,8 +364,8 @@ public class MongoGeoIndexerTest {
// @Test
public void testAntiMeridianSearch() throws Exception {
// verify that a search works if the bounding box crosses the anti meridian
- try (MongoGeoIndexer f = new MongoGeoIndexer(mongoClient)) {
- f.setConf(conf);
+ try (MongoGeoIndexer f = new MongoGeoIndexer()) {
+ f.initIndexer(conf, mongoClient);
final ValueFactory vf = new ValueFactoryImpl();
final Resource context = vf.createURI("foo:context");
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java
index a8481fe..c612338 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/mongo/MongoTemporalIndexerTest.java
@@ -192,8 +192,8 @@ public final class MongoTemporalIndexerTest {
final MongodForTestsFactory testsFactory = MongodForTestsFactory.with(Version.Main.PRODUCTION);
final MongoClient mongoClient = testsFactory.newMongo();
- tIndexer = new MongoTemporalIndexer(mongoClient);
- tIndexer.setConf(conf);
+ tIndexer = new MongoTemporalIndexer();
+ tIndexer.initIndexer(conf, mongoClient);
final String dbName = conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8597b78b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java
----------------------------------------------------------------------
diff --git a/extras/indexingExample/src/main/java/MongoRyaDirectExample.java b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java
index 91a31f9..5dbd260 100644
--- a/extras/indexingExample/src/main/java/MongoRyaDirectExample.java
+++ b/extras/indexingExample/src/main/java/MongoRyaDirectExample.java
@@ -60,6 +60,10 @@ public class MongoRyaDirectExample {
private static final boolean PRINT_QUERIES = true;
private static final String MONGO_DB = "rya";
private static final String MONGO_COLL_PREFIX = "rya_";
+ private static final boolean USE_MOCK = true;
+ private static final boolean USE_INFER = true;
+ private static final String MONGO_INSTANCE_URL = "localhost";
+ private static final String MONGO_INSTANCE_PORT = "27017";
public static void main(String[] args) throws Exception {
Configuration conf = getConf();
@@ -80,7 +84,10 @@ public class MongoRyaDirectExample {
testAddNamespaces(conn);
testAddPointAndWithinSearch(conn);
testAddAndFreeTextSearchWithPCJ(conn);
- testInfer(conn, sail);
+ // to test out inference, set inference to true in the conf
+ if (USE_INFER){
+ testInfer(conn, sail);
+ }
log.info("TIME: " + (System.currentTimeMillis() - start) / 1000.);
} finally {
@@ -249,10 +256,16 @@ public class MongoRyaDirectExample {
MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration();
conf.set(ConfigUtils.USE_MONGO, "true");
+
// User name and password must be filled in:
conf.set(MongoDBRdfConfiguration.MONGO_USER, "fill this in");
conf.set(MongoDBRdfConfiguration.MONGO_USER_PASSWORD, "fill this in");
- conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, "true");
+ conf.set(MongoDBRdfConfiguration.USE_TEST_MONGO, Boolean.toString(USE_MOCK));
+ if (!USE_MOCK){
+ conf.set(MongoDBRdfConfiguration.MONGO_INSTANCE, MONGO_INSTANCE_URL);
+ conf.set(MongoDBRdfConfiguration.MONGO_INSTANCE_PORT, MONGO_INSTANCE_PORT);
+
+ }
conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, MONGO_DB);
conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, MONGO_COLL_PREFIX);
conf.set(ConfigUtils.GEO_PREDICATES_LIST, "http://www.opengis.net/ont/geosparql#asWKT");
@@ -261,6 +274,8 @@ public class MongoRyaDirectExample {
conf.setTablePrefix(MONGO_COLL_PREFIX);
conf.set(ConfigUtils.GEO_PREDICATES_LIST, GeoConstants.GEO_AS_WKT.stringValue());
conf.set(ConfigUtils.FREETEXT_PREDICATES_LIST, RDFS.LABEL.stringValue());
+ conf.set(ConfigUtils.FREETEXT_PREDICATES_LIST, RDFS.LABEL.stringValue());
+ conf.set(RdfCloudTripleStoreConfiguration.CONF_INFER, Boolean.toString(USE_INFER));
return conf;
}