You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2017/07/13 06:45:20 UTC
[1/2] lucene-solr:branch_7x: SOLR-11045: The new replica created by
MoveReplica will have to have same name and coreName as the old one in case
of HDFS
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x e8c74e8ce -> 01d18b7a6
SOLR-11045: The new replica created by MoveReplica will have to have same name and coreName as the old one in case of HDFS
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0f93c470
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0f93c470
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0f93c470
Branch: refs/heads/branch_7x
Commit: 0f93c470ab0f609e872ec99f0be214b8d3c4241b
Parents: dd9b595
Author: Cao Manh Dat <da...@apache.org>
Authored: Thu Jul 13 13:43:21 2017 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Thu Jul 13 13:44:51 2017 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../java/org/apache/solr/cloud/CloudUtil.java | 4 +-
.../org/apache/solr/cloud/MoveReplicaCmd.java | 24 +++++++--
.../org/apache/solr/cloud/ZkController.java | 5 +-
.../solr/cloud/MoveReplicaHDFSUlogDirTest.java | 53 +++++++++++++++++---
5 files changed, 72 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f93c470/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 59771f8..2d08e16 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -315,6 +315,9 @@ Bug Fixes
* SOLR-11041: MoveReplicaCmd do not specify ulog dir in case of HDFS (Cao Manh Dat)
+* SOLR-11045: The new replica created by MoveReplica will have to have same name and coreName as the
+ old one in case of HDFS (Cao Manh Dat)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f93c470/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
index ac09621..c05072d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
@@ -31,6 +31,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrResourceLoader;
@@ -64,10 +65,11 @@ public class CloudUtil {
String cnn = replica.getName();
String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
+ boolean isSharedFs = replica.getStr(CoreAdminParams.DATA_DIR) != null;
log.debug("compare against coreNodeName={} baseUrl={}", cnn, baseUrl);
if (thisCnn != null && thisCnn.equals(cnn)
- && !thisBaseUrl.equals(baseUrl)) {
+ && !thisBaseUrl.equals(baseUrl) && isSharedFs) {
if (cc.getLoadedCoreNames().contains(desc.getName())) {
cc.unload(desc.getName());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f93c470/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
index 4598f14..117edf9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
@@ -35,6 +35,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.update.UpdateLog;
+import org.apache.solr.util.TimeOut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -115,8 +116,6 @@ public class MoveReplicaCmd implements Cmd{
private void moveHdfsReplica(ClusterState clusterState, NamedList results, String dataDir, String targetNode, String async,
DocCollection coll, Replica replica, Slice slice, int timeout) throws Exception {
- String newCoreName = Assign.buildCoreName(coll, slice.getName(), replica.getType());
-
ZkNodeProps removeReplicasProps = new ZkNodeProps(
COLLECTION_PROP, coll.getName(),
SHARD_ID_PROP, slice.getName(),
@@ -135,17 +134,32 @@ public class MoveReplicaCmd implements Cmd{
return;
}
+ TimeOut timeOut = new TimeOut(20L, TimeUnit.SECONDS);
+ while (!timeOut.hasTimedOut()) {
+ coll = ocmh.zkStateReader.getClusterState().getCollection(coll.getName());
+ if (coll.getReplica(replica.getName()) != null) {
+ Thread.sleep(100);
+ } else {
+ break;
+ }
+ }
+ if (timeOut.hasTimedOut()) {
+ results.add("failure", "Still see deleted replica in clusterstate!");
+ return;
+ }
+
String ulogDir = replica.getStr(CoreAdminParams.ULOG_DIR);
ZkNodeProps addReplicasProps = new ZkNodeProps(
COLLECTION_PROP, coll.getName(),
SHARD_ID_PROP, slice.getName(),
CoreAdminParams.NODE, targetNode,
- CoreAdminParams.NAME, newCoreName,
+ CoreAdminParams.CORE_NODE_NAME, replica.getName(),
+ CoreAdminParams.NAME, replica.getCoreName(),
CoreAdminParams.ULOG_DIR, ulogDir.substring(0, ulogDir.lastIndexOf(UpdateLog.TLOG_NAME)),
CoreAdminParams.DATA_DIR, dataDir);
if(async!=null) addReplicasProps.getProperties().put(ASYNC, async);
NamedList addResult = new NamedList();
- ocmh.addReplica(clusterState, addReplicasProps, addResult, null);
+ ocmh.addReplica(ocmh.zkStateReader.getClusterState(), addReplicasProps, addResult, null);
if (addResult.get("failure") != null) {
String errorString = String.format(Locale.ROOT, "Failed to create replica for collection=%s shard=%s" +
" on node=%s", coll.getName(), slice.getName(), targetNode);
@@ -154,7 +168,7 @@ public class MoveReplicaCmd implements Cmd{
return;
} else {
String successString = String.format(Locale.ROOT, "MOVEREPLICA action completed successfully, moved replica=%s at node=%s " +
- "to replica=%s at node=%s", replica.getCoreName(), replica.getNodeName(), newCoreName, targetNode);
+ "to replica=%s at node=%s", replica.getCoreName(), replica.getNodeName(), replica.getCoreName(), targetNode);
results.add("success", successString);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f93c470/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index c8d763f..8a7925d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -2184,10 +2184,7 @@ public class ZkController {
DocCollection collection = clusterState.getCollectionOrNull(desc
.getCloudDescriptor().getCollectionName());
if (collection != null) {
- boolean autoAddReplicas = ClusterStateUtil.isAutoAddReplicas(getZkStateReader(), collection.getName());
- if (autoAddReplicas) {
- CloudUtil.checkSharedFSFailoverReplaced(cc, desc);
- }
+ CloudUtil.checkSharedFSFailoverReplaced(cc, desc);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f93c470/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSUlogDirTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSUlogDirTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSUlogDirTest.java
index dfcf58e..a27a39d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSUlogDirTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSUlogDirTest.java
@@ -21,10 +21,13 @@ import java.io.IOException;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
+import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterStateUtil;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@@ -65,7 +68,7 @@ public class MoveReplicaHDFSUlogDirTest extends SolrCloudTestCase {
}
@Test
- public void testDataDirAndUlogAreMaintained() throws IOException, SolrServerException {
+ public void testDataDirAndUlogAreMaintained() throws Exception {
String coll = "movereplicatest_coll2";
CollectionAdminRequest.createCollection(coll, "conf1", 1, 1)
.setCreateNodeSet("")
@@ -82,7 +85,8 @@ public class MoveReplicaHDFSUlogDirTest extends SolrCloudTestCase {
ulogDir += "/tlog";
ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000);
+ assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
+
DocCollection docCollection = zkStateReader.getClusterState().getCollection(coll);
Replica replica = docCollection.getReplicas().iterator().next();
assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/'));
@@ -90,14 +94,49 @@ public class MoveReplicaHDFSUlogDirTest extends SolrCloudTestCase {
new CollectionAdminRequest.MoveReplica(coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
.process(cluster.getSolrClient());
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000);
+ assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
docCollection = zkStateReader.getClusterState().getCollection(coll);
assertEquals(1, docCollection.getSlice("shard1").getReplicas().size());
- replica = docCollection.getReplicas().iterator().next();
- assertEquals(replica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName());
- assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/'));
- assertTrue(replica.getStr("dataDir"),replica.getStr("dataDir").equals(dataDir) || replica.getStr("dataDir").equals(dataDir+'/'));
+ Replica newReplica = docCollection.getReplicas().iterator().next();
+ assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName());
+ assertTrue(newReplica.getStr("ulogDir"), newReplica.getStr("ulogDir").equals(ulogDir) || newReplica.getStr("ulogDir").equals(ulogDir+'/'));
+ assertTrue(newReplica.getStr("dataDir"),newReplica.getStr("dataDir").equals(dataDir) || newReplica.getStr("dataDir").equals(dataDir+'/'));
+
+ assertEquals(replica.getName(), newReplica.getName());
+ assertEquals(replica.getCoreName(), newReplica.getCoreName());
+ assertFalse(replica.getNodeName().equals(newReplica.getNodeName()));
+ final int numDocs = 100;
+ addDocs(coll, numDocs); // indexed but not committed
+
+ cluster.getJettySolrRunner(1).stop();
+ Thread.sleep(5000);
+ new CollectionAdminRequest.MoveReplica(coll, newReplica.getName(), cluster.getJettySolrRunner(0).getNodeName())
+ .process(cluster.getSolrClient());
+ assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
+
+ // assert that the old core will be removed on startup
+ cluster.getJettySolrRunner(1).start();
+ assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
+ docCollection = zkStateReader.getClusterState().getCollection(coll);
+ assertEquals(1, docCollection.getReplicas().size());
+ newReplica = docCollection.getReplicas().iterator().next();
+ assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(0).getNodeName());
+ assertTrue(newReplica.getStr("ulogDir"), newReplica.getStr("ulogDir").equals(ulogDir) || newReplica.getStr("ulogDir").equals(ulogDir+'/'));
+ assertTrue(newReplica.getStr("dataDir"),newReplica.getStr("dataDir").equals(dataDir) || newReplica.getStr("dataDir").equals(dataDir+'/'));
+
+ assertEquals(0, cluster.getJettySolrRunner(1).getCoreContainer().getCores().size());
+
+ cluster.getSolrClient().commit(coll);
+ assertEquals(numDocs, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
+ }
+ private void addDocs(String collection, int numDocs) throws SolrServerException, IOException {
+ SolrClient solrClient = cluster.getSolrClient();
+ for (int docId = 1; docId <= numDocs; docId++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", docId);
+ solrClient.add(collection, doc);
+ }
}
}
[2/2] lucene-solr:branch_7x: Merge branch 'branch_7x' of
https://git-wip-us.apache.org/repos/asf/lucene-solr into branch_7x
Posted by da...@apache.org.
Merge branch 'branch_7x' of https://git-wip-us.apache.org/repos/asf/lucene-solr into branch_7x
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/01d18b7a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/01d18b7a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/01d18b7a
Branch: refs/heads/branch_7x
Commit: 01d18b7a6c956ecc2405c501c3ea9317a66373b2
Parents: 0f93c47 e8c74e8
Author: Cao Manh Dat <da...@apache.org>
Authored: Thu Jul 13 13:45:10 2017 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Thu Jul 13 13:45:10 2017 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 6 ++
.../org/apache/solr/handler/StreamHandler.java | 2 +-
...chema-add-schema-fields-update-processor.xml | 12 +--
.../conf/schema-blockjoinfacetcomponent.xml | 6 +-
.../collection1/conf/schema-custom-field.xml | 4 +-
.../org/apache/solr/schema/TestPointFields.java | 17 ++--
.../solrj/io/eval/ResidualsEvaluator.java | 82 ++++++++++++++++++++
.../org/apache/solr/common/cloud/Replica.java | 7 +-
.../solrj/io/stream/StreamExpressionTest.java | 24 ++++++
9 files changed, 137 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/01d18b7a/solr/CHANGES.txt
----------------------------------------------------------------------