You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rya.apache.org by ca...@apache.org on 2017/06/22 20:19:26 UTC
incubator-rya git commit: RYA-304 Fixed rya.merger integration tests.
Closes #173.
Repository: incubator-rya
Updated Branches:
refs/heads/master 60090ad52 -> 36f271b69
RYA-304 Fixed rya.merger integration tests. Closes #173.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/36f271b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/36f271b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/36f271b6
Branch: refs/heads/master
Commit: 36f271b695a0a9ca01b681bee354c9301002c758
Parents: 60090ad
Author: eric.white <Er...@parsons.com>
Authored: Thu Jun 22 11:35:03 2017 -0400
Committer: Caleb Meier <ca...@parsons.com>
Committed: Thu Jun 22 12:47:58 2017 -0700
----------------------------------------------------------------------
.../mr/merge/mappers/BaseCopyToolMapper.java | 11 ++--
.../mr/merge/util/AccumuloInstanceDriver.java | 13 +++--
.../rya/accumulo/mr/merge/util/TimeUtils.java | 3 +-
.../rya/accumulo/mr/merge/RulesetCopyIT.java | 55 +++++++++++---------
.../driver/AccumuloDualInstanceDriver.java | 3 +-
5 files changed, 43 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java
----------------------------------------------------------------------
diff --git a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java
index 96f0ce6..1e73d1f 100644
--- a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java
+++ b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/mappers/BaseCopyToolMapper.java
@@ -37,11 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.Mapper;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-
+import org.apache.log4j.Logger;
import org.apache.rya.accumulo.AccumuloRdfConfiguration;
import org.apache.rya.accumulo.AccumuloRyaDAO;
import org.apache.rya.accumulo.mr.MRUtils;
@@ -50,7 +46,10 @@ import org.apache.rya.accumulo.mr.merge.MergeTool;
import org.apache.rya.accumulo.mr.merge.util.AccumuloRyaUtils;
import org.apache.rya.api.persist.RyaDAOException;
import org.apache.rya.api.resolver.RyaTripleContext;
-import twitter4j.Logger;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
/**
* The base {@link Mapper} for the copy tool which initializes the mapper for use. The mapper will take all
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java
----------------------------------------------------------------------
diff --git a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java
index 9d67e38..d3dde34 100644
--- a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java
+++ b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/AccumuloInstanceDriver.java
@@ -42,18 +42,17 @@ import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.SystemUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.Files;
-
+import org.apache.log4j.Logger;
import org.apache.rya.accumulo.AccumuloRdfConfiguration;
import org.apache.rya.accumulo.AccumuloRyaDAO;
import org.apache.rya.accumulo.mr.MRUtils;
import org.apache.rya.accumulo.mr.merge.MergeTool;
import org.apache.rya.api.RdfCloudTripleStoreConstants;
import org.apache.rya.api.persist.RyaDAOException;
-import twitter4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Files;
/**
* Handles running a single {@link MiniAccumuloCluster} or a single {@link MockInstance} for an instance.
@@ -151,10 +150,10 @@ public class AccumuloInstanceDriver {
* @throws Exception
*/
public void setUp() throws Exception {
+ setUpConfig();
setUpInstance();
setUpTables();
setUpDao();
- setUpConfig();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java
----------------------------------------------------------------------
diff --git a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java
index db760e5..5c3c72e 100644
--- a/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java
+++ b/extras/rya.merger/src/main/java/org/apache/rya/accumulo/mr/merge/util/TimeUtils.java
@@ -33,13 +33,12 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.net.ntp.NTPUDPClient;
import org.apache.commons.net.ntp.TimeInfo;
+import org.apache.log4j.Logger;
import org.codehaus.plexus.util.StringUtils;
import org.mortbay.jetty.HttpMethods;
import com.google.common.net.HttpHeaders;
-import twitter4j.Logger;
-
/**
* Utility methods for time.
*/
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java
index ec3f673..b85ee4e 100644
--- a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java
+++ b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/RulesetCopyIT.java
@@ -31,6 +31,23 @@ import java.util.Set;
import org.apache.accumulo.core.client.Connector;
import org.apache.hadoop.conf.Configuration;
import org.apache.log4j.Logger;
+import org.apache.rya.accumulo.AccumuloRdfConfiguration;
+import org.apache.rya.accumulo.AccumuloRyaDAO;
+import org.apache.rya.accumulo.mr.MRUtils;
+import org.apache.rya.accumulo.mr.merge.common.InstanceType;
+import org.apache.rya.accumulo.mr.merge.demo.util.DemoUtilities;
+import org.apache.rya.accumulo.mr.merge.demo.util.DemoUtilities.LoggingDetail;
+import org.apache.rya.accumulo.mr.merge.driver.AccumuloDualInstanceDriver;
+import org.apache.rya.accumulo.mr.merge.util.AccumuloRyaUtils;
+import org.apache.rya.accumulo.mr.merge.util.TestUtils;
+import org.apache.rya.api.RdfCloudTripleStoreConfiguration;
+import org.apache.rya.api.domain.RyaStatement;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.domain.RyaURI;
+import org.apache.rya.api.persist.RyaDAOException;
+import org.apache.rya.api.resolver.RyaToRdfConversions;
+import org.apache.rya.indexing.accumulo.ConfigUtils;
+import org.apache.rya.sail.config.RyaSailFactory;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -56,21 +73,6 @@ import org.openrdf.sail.Sail;
import info.aduna.iteration.CloseableIteration;
import junit.framework.Assert;
-import org.apache.rya.accumulo.AccumuloRdfConfiguration;
-import org.apache.rya.accumulo.AccumuloRyaDAO;
-import org.apache.rya.accumulo.mr.MRUtils;
-import org.apache.rya.accumulo.mr.merge.common.InstanceType;
-import org.apache.rya.accumulo.mr.merge.driver.AccumuloDualInstanceDriver;
-import org.apache.rya.accumulo.mr.merge.util.AccumuloRyaUtils;
-import org.apache.rya.accumulo.mr.merge.util.TestUtils;
-import org.apache.rya.api.RdfCloudTripleStoreConfiguration;
-import org.apache.rya.api.domain.RyaStatement;
-import org.apache.rya.api.domain.RyaType;
-import org.apache.rya.api.domain.RyaURI;
-import org.apache.rya.api.persist.RyaDAOException;
-import org.apache.rya.api.resolver.RyaToRdfConversions;
-import org.apache.rya.indexing.accumulo.ConfigUtils;
-import org.apache.rya.sail.config.RyaSailFactory;
public class RulesetCopyIT {
private static final Logger log = Logger.getLogger(RulesetCopyIT.class);
@@ -144,32 +146,33 @@ public class RulesetCopyIT {
@BeforeClass
public static void setUpPerClass() throws Exception {
+ DemoUtilities.setupLogging(LoggingDetail.LIGHT);
accumuloDualInstanceDriver = new AccumuloDualInstanceDriver(IS_MOCK, true, true, false, false);
accumuloDualInstanceDriver.setUpInstances();
}
@Before
public void setUpPerTest() throws Exception {
- parentConfig = accumuloDualInstanceDriver.getParentConfig();
- childConfig = accumuloDualInstanceDriver.getChildConfig();
- accumuloDualInstanceDriver.setUpTables();
accumuloDualInstanceDriver.setUpConfigs();
+ accumuloDualInstanceDriver.setUpTables();
accumuloDualInstanceDriver.setUpDaos();
+ parentConfig = accumuloDualInstanceDriver.getParentConfig();
+ childConfig = accumuloDualInstanceDriver.getChildConfig();
parentDao = accumuloDualInstanceDriver.getParentDao();
}
@After
public void tearDownPerTest() throws Exception {
log.info("tearDownPerTest(): tearing down now.");
- accumuloDualInstanceDriver.tearDownDaos();
accumuloDualInstanceDriver.tearDownTables();
+ accumuloDualInstanceDriver.tearDownDaos();
+ if (rulesetTool != null) {
+ rulesetTool.shutdown();
+ }
}
@AfterClass
public static void tearDownPerClass() throws Exception {
- if (rulesetTool != null) {
- rulesetTool.shutdown();
- }
accumuloDualInstanceDriver.tearDown();
}
@@ -185,6 +188,9 @@ public class RulesetCopyIT {
AccumuloRyaUtils.printTablePretty(table, parentConfig, false);
}
+ parentConfig.set(RdfCloudTripleStoreConfiguration.CONF_INFER, Boolean.toString(infer));
+ childConfig.set(RdfCloudTripleStoreConfiguration.CONF_INFER, Boolean.toString(infer));
+
rulesetTool = new CopyTool();
rulesetTool.setupAndRun(new String[] {
makeArgument(MRUtils.AC_MOCK_PROP, Boolean.toString(IS_MOCK)),
@@ -210,8 +216,8 @@ public class RulesetCopyIT {
});
final Configuration toolConfig = rulesetTool.getConf();
- childConfig.set(MRUtils.AC_ZK_PROP, toolConfig.get(MRUtils.AC_ZK_PROP + CHILD_SUFFIX));
- MergeTool.setDuplicateKeys(childConfig);
+ final String zooKeepers = toolConfig.get(MRUtils.AC_ZK_PROP + CHILD_SUFFIX);
+ MergeTool.setDuplicateKeysForProperty(childConfig, MRUtils.AC_ZK_PROP, zooKeepers);
log.info("Finished running tool.");
@@ -272,7 +278,6 @@ public class RulesetCopyIT {
try {
final Sail extSail = RyaSailFactory.getInstance(conf);
repository = new SailRepository(extSail);
- repository.initialize();
conn = repository.getConnection();
final ResultHandler handler = new ResultHandler();
final TupleQuery tq = conn.prepareTupleQuery(QueryLanguage.SPARQL, query);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/36f271b6/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java
----------------------------------------------------------------------
diff --git a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java
index c5a5a29..0cd4942 100644
--- a/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java
+++ b/extras/rya.merger/src/test/java/org/apache/rya/accumulo/mr/merge/driver/AccumuloDualInstanceDriver.java
@@ -34,13 +34,12 @@ import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
-
+import org.apache.log4j.Logger;
import org.apache.rya.accumulo.AccumuloRdfConfiguration;
import org.apache.rya.accumulo.AccumuloRyaDAO;
import org.apache.rya.accumulo.mr.merge.util.AccumuloInstanceDriver;
import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.persist.RyaDAOException;
-import twitter4j.Logger;
/**
* Handles running a {@link MiniAccumuloCluster} or a {@link MockInstance} for a parent and child instance for testing.