You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2016/05/13 13:46:55 UTC
lucene-solr:branch_6x: SOLR-9072: Move morphline-core zk tests to
SolrCloudTestCase
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 32e16cb0e -> 18f54d6f7
SOLR-9072: Move morphline-core zk tests to SolrCloudTestCase
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/18f54d6f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/18f54d6f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/18f54d6f
Branch: refs/heads/branch_6x
Commit: 18f54d6f7721394d5dd3e1ee137abb8b690e2995
Parents: 32e16cb
Author: Alan Woodward <ro...@apache.org>
Authored: Thu May 5 12:09:55 2016 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Fri May 13 14:40:17 2016 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solr/AbstractSolrMorphlineZkTestBase.java | 114 ++++++-------------
.../solr/SolrMorphlineZkAliasTest.java | 51 +++------
.../solr/SolrMorphlineZkAvroTest.java | 48 +++-----
.../morphlines/solr/SolrMorphlineZkTest.java | 17 ++-
5 files changed, 79 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/18f54d6f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6cf1e47..80476a7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -252,6 +252,8 @@ Other Changes
* SOLR-9105: Fix a bunch of typos across 103 files (Bartosz Krasi\u0144ski via janhoy)
+* SOLR-9072: Migrate morphline-core tests to SolrCloudTestCase. (Alan Woodward)
+
================== 6.0.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/18f54d6f/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java
index 4d95a4f..535fe9d 100644
--- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java
+++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java
@@ -24,14 +24,13 @@ import java.util.Locale;
import com.codahale.metrics.MetricRegistry;
import com.google.common.collect.ListMultimap;
import com.typesafe.config.Config;
-import org.apache.commons.io.FileUtils;
import org.apache.lucene.util.Constants;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
-import org.apache.solr.cloud.AbstractZkTestCase;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.kitesdk.morphline.api.Collector;
import org.kitesdk.morphline.api.Command;
@@ -42,72 +41,58 @@ import org.kitesdk.morphline.base.FaultTolerance;
import org.kitesdk.morphline.base.Notifications;
import org.kitesdk.morphline.stdlib.PipeBuilder;
-public abstract class AbstractSolrMorphlineZkTestBase extends AbstractFullDistribZkTestBase {
- private static File solrHomeDirectory;
-
- protected static final String RESOURCES_DIR = getFile("morphlines-core.marker").getParent();
- private static final File SOLR_INSTANCE_DIR = new File(RESOURCES_DIR + "/solr");
- private static final File SOLR_CONF_DIR = new File(RESOURCES_DIR + "/solr/collection1");
+public abstract class AbstractSolrMorphlineZkTestBase extends SolrCloudTestCase {
- protected Collector collector;
- protected Command morphline;
+ protected static final String COLLECTION = "collection1";
- @Override
- public String getSolrHome() {
- return solrHomeDirectory.getPath();
- }
-
- public AbstractSolrMorphlineZkTestBase() {
- sliceCount = 3;
- fixShardCount(3);
+ protected static final int TIMEOUT = 30;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2)
+ .addConfig("conf", SOLR_CONF_DIR.toPath())
+ .configure();
+
+ CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1)
+ .processAndWait(cluster.getSolrClient(), TIMEOUT);
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
+ false, true, TIMEOUT);
}
+
+ protected static final String RESOURCES_DIR = getFile("morphlines-core.marker").getParent();
+ private static final File SOLR_CONF_DIR = new File(RESOURCES_DIR + "/solr/collection1/conf");
+
+ protected Collector collector;
+ protected Command morphline;
@BeforeClass
public static void setupClass() throws Exception {
- assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9);
+
+ assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)",
+ Constants.JRE_IS_MINIMUM_JAVA9);
assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)",
new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage()));
- solrHomeDirectory = createTempDir().toFile();
- AbstractZkTestCase.SOLRHOME = solrHomeDirectory;
- FileUtils.copyDirectory(SOLR_INSTANCE_DIR, solrHomeDirectory);
- }
-
- @AfterClass
- public static void tearDownClass() throws Exception {
- solrHomeDirectory = null;
+
}
-
- @Override
- public void distribSetUp() throws Exception {
- super.distribSetUp();
- System.setProperty("host", "127.0.0.1");
- System.setProperty("numShards", Integer.toString(sliceCount));
- uploadConfFiles();
+
+ @Before
+ public void setup() throws Exception {
collector = new Collector();
}
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- System.clearProperty("host");
- System.clearProperty("numShards");
- }
-
- @Override
+
protected void commit() throws Exception {
- Notifications.notifyCommitTransaction(morphline);
- super.commit();
+ Notifications.notifyCommitTransaction(morphline);
}
protected Command parse(String file) throws IOException {
- return parse(file, "collection1");
+ return parse(file, COLLECTION);
}
protected Command parse(String file, String collection) throws IOException {
SolrLocator locator = new SolrLocator(createMorphlineContext());
locator.setCollectionName(collection);
- locator.setZkHost(zkServer.getZkAddress());
+ locator.setZkHost(cluster.getZkServer().getZkAddress());
//locator.setServerUrl(cloudJettys.get(0).url); // TODO: download IndexSchema from solrUrl not yet implemented
//locator.setSolrHomeDir(SOLR_HOME_DIR.getPath());
Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), locator.toConfig("SOLR_LOCATOR"));
@@ -145,33 +130,4 @@ public abstract class AbstractSolrMorphlineZkTestBase extends AbstractFullDistri
return record;
}
- private void putConfig(SolrZkClient zkClient, String name) throws Exception {
- File file = new File(new File(SOLR_CONF_DIR, "conf"), name);
- String destPath = "/configs/conf1/" + name;
- System.out.println("put " + file.getAbsolutePath() + " to " + destPath);
- zkClient.makePath(destPath, file, false, true);
- }
-
- private void uploadConfFiles(SolrZkClient zkClient, File dir, String prefix) throws Exception {
- boolean found = false;
- for (File f : dir.listFiles()) {
- String name = f.getName();
- if (name.startsWith(".")) continue;
- if (f.isFile()) {
- putConfig(zkClient, prefix + name);
- found = true;
- } else if (f.isDirectory()) {
- uploadConfFiles(zkClient, new File(dir, name), prefix + name + "/");
- }
- }
- assertTrue("Config folder '" + dir + "' with files to upload to zookeeper was empty.", found);
- }
-
- private void uploadConfFiles() throws Exception {
- // upload our own config files
- SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), 10000);
- uploadConfFiles(zkClient, new File(SOLR_CONF_DIR, "conf"), "");
- zkClient.close();
- }
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/18f54d6f/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAliasTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAliasTest.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAliasTest.java
index a654e94..ddaf2f6 100644
--- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAliasTest.java
+++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAliasTest.java
@@ -17,39 +17,32 @@
package org.apache.solr.morphlines.solr;
import java.io.File;
-import java.io.IOException;
import java.util.Iterator;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.params.CollectionParams.CollectionAction;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.BadHdfsThreadsFilter;
import org.junit.Test;
import org.kitesdk.morphline.api.Record;
import org.kitesdk.morphline.base.Fields;
import org.kitesdk.morphline.base.Notifications;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
@Slow
public class SolrMorphlineZkAliasTest extends AbstractSolrMorphlineZkTestBase {
-
@Test
public void test() throws Exception {
-
- waitForRecoveriesToFinish(false);
-
- createAlias("aliascollection", "collection1");
+
+ CollectionAdminRequest.createAlias("aliascollection", "collection1")
+ .process(cluster.getSolrClient());
morphline = parse("test-morphlines" + File.separator + "loadSolrBasic", "aliascollection");
Record record = new Record();
@@ -84,9 +77,11 @@ public class SolrMorphlineZkAliasTest extends AbstractSolrMorphlineZkTestBase {
assertFalse(citer.hasNext());
- commit();
+ Notifications.notifyCommitTransaction(morphline);
+ new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION);
- QueryResponse rsp = cloudClient.query(new SolrQuery("*:*").setRows(100000).addSort(Fields.ID, SolrQuery.ORDER.asc));
+ QueryResponse rsp = cluster.getSolrClient()
+ .query(COLLECTION, new SolrQuery("*:*").setRows(100000).addSort(Fields.ID, SolrQuery.ORDER.asc));
//System.out.println(rsp);
Iterator<SolrDocument> iter = rsp.getResults().iterator();
assertEquals(expected.getFields(), next(iter));
@@ -95,26 +90,14 @@ public class SolrMorphlineZkAliasTest extends AbstractSolrMorphlineZkTestBase {
Notifications.notifyRollbackTransaction(morphline);
Notifications.notifyShutdown(morphline);
-
-
- createAlias("aliascollection", "collection1,collection2");
-
- try {
+
+ CollectionAdminRequest.createAlias("aliascollection", "collection1,collection2")
+ .processAndWait(cluster.getSolrClient(), TIMEOUT);
+
+ expectThrows(IllegalArgumentException.class, () -> {
parse("test-morphlines" + File.separator + "loadSolrBasic", "aliascollection");
- fail("Expected IAE because update alias maps to multiple collections");
- } catch (IllegalArgumentException e) {
-
- }
- }
-
- private NamedList<Object> createAlias(String alias, String collections) throws SolrServerException, IOException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("collections", collections);
- params.set("name", alias);
- params.set("action", CollectionAction.CREATEALIAS.toString());
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
- return cloudClient.request(request);
+ });
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/18f54d6f/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java
index 1c30a84..4f3b27f 100644
--- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java
+++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java
@@ -17,19 +17,23 @@
package org.apache.solr.morphlines.solr;
import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
-import org.apache.avro.Schema.Field;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.google.common.base.Preconditions;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.FileReader;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.util.BadHdfsThreadsFilter;
@@ -38,16 +42,6 @@ import org.kitesdk.morphline.api.Record;
import org.kitesdk.morphline.base.Fields;
import org.kitesdk.morphline.base.Notifications;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction.Action;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.io.Files;
-
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
@@ -57,33 +51,33 @@ public class SolrMorphlineZkAvroTest extends AbstractSolrMorphlineZkTestBase {
@Test
public void test() throws Exception {
- Joiner joiner = Joiner.on(File.separator);
- File file = new File(joiner.join(RESOURCES_DIR, "test-documents", "sample-statuses-20120906-141433-medium.avro"));
-
- waitForRecoveriesToFinish(false);
-
+
+ Path avro = Paths.get(RESOURCES_DIR).resolve("test-documents").resolve("sample-statuses-20120906-141433-medium.avro");
+
// load avro records via morphline and zk into solr
morphline = parse("test-morphlines" + File.separator + "tutorialReadAvroContainer");
Record record = new Record();
- byte[] body = Files.toByteArray(file);
+ byte[] body = Files.readAllBytes(avro);
record.put(Fields.ATTACHMENT_BODY, body);
startSession();
Notifications.notifyBeginTransaction(morphline);
assertTrue(morphline.process(record));
assertEquals(1, collector.getNumStartEvents());
- commit();
+ Notifications.notifyCommitTransaction(morphline);
+ new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION);
// fetch sorted result set from solr
- QueryResponse rsp = cloudClient.query(new SolrQuery("*:*").setRows(100000).addSort("id", SolrQuery.ORDER.asc));
+ QueryResponse rsp = cluster.getSolrClient()
+ .query(COLLECTION, new SolrQuery("*:*").setRows(100000).addSort("id", SolrQuery.ORDER.asc));
assertEquals(2104, collector.getRecords().size());
assertEquals(collector.getRecords().size(), rsp.getResults().size());
Collections.sort(collector.getRecords(), (r1, r2) -> r1.get("id").toString().compareTo(r2.get("id").toString()));
// fetch test input data and sort like solr result set
- List<GenericData.Record> records = new ArrayList();
- FileReader<GenericData.Record> reader = new DataFileReader(file, new GenericDatumReader());
+ List<GenericData.Record> records = new ArrayList<>();
+ FileReader<GenericData.Record> reader = new DataFileReader(avro.toFile(), new GenericDatumReader());
while (reader.hasNext()) {
GenericData.Record expected = reader.next();
records.add(expected);
@@ -121,7 +115,7 @@ public class SolrMorphlineZkAvroTest extends AbstractSolrMorphlineZkTestBase {
Notifications.notifyRollbackTransaction(morphline);
Notifications.notifyShutdown(morphline);
- cloudClient.close();
+
}
private void assertTweetEquals(GenericData.Record expected, Record actual, int i) {
@@ -144,12 +138,4 @@ public class SolrMorphlineZkAvroTest extends AbstractSolrMorphlineZkTestBase {
}
}
- private String toString(GenericData.Record avroRecord) {
- Record record = new Record();
- for (Field field : avroRecord.getSchema().getFields()) {
- record.put(field.name(), avroRecord.get(field.pos()));
- }
- return record.toString(); // prints sorted by key for human readability
- }
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/18f54d6f/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java
index 97c6dfb..24d8682 100644
--- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java
+++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java
@@ -19,8 +19,10 @@ package org.apache.solr.morphlines.solr;
import java.io.File;
import java.util.Iterator;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.util.BadHdfsThreadsFilter;
@@ -29,21 +31,16 @@ import org.kitesdk.morphline.api.Record;
import org.kitesdk.morphline.base.Fields;
import org.kitesdk.morphline.base.Notifications;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
@Slow
public class SolrMorphlineZkTest extends AbstractSolrMorphlineZkTestBase {
-
@Test
public void test() throws Exception {
- waitForRecoveriesToFinish(false);
-
- morphline = parse("test-morphlines" + File.separator + "loadSolrBasic");
+ morphline = parse("test-morphlines" + File.separator + "loadSolrBasic");
Record record = new Record();
record.put(Fields.ID, "id0-innsbruck");
record.put("text", "mytext");
@@ -76,9 +73,11 @@ public class SolrMorphlineZkTest extends AbstractSolrMorphlineZkTestBase {
assertFalse(citer.hasNext());
- commit();
+ Notifications.notifyCommitTransaction(morphline);
+ new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION);
- QueryResponse rsp = cloudClient.query(new SolrQuery("*:*").setRows(100000).addSort(Fields.ID, SolrQuery.ORDER.asc));
+ QueryResponse rsp = cluster.getSolrClient()
+ .query(COLLECTION, new SolrQuery("*:*").setRows(100000).addSort(Fields.ID, SolrQuery.ORDER.asc));
//System.out.println(rsp);
Iterator<SolrDocument> iter = rsp.getResults().iterator();
assertEquals(expected.getFields(), next(iter));
@@ -87,7 +86,7 @@ public class SolrMorphlineZkTest extends AbstractSolrMorphlineZkTestBase {
Notifications.notifyRollbackTransaction(morphline);
Notifications.notifyShutdown(morphline);
- cloudClient.close();
+
}
}