You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/07/13 15:38:15 UTC
[18/47] lucene-solr:jira/solr-11000: SOLR-11041: MoveReplicaCmd do
not specify ulog dir in case of HDFS
SOLR-11041: MoveReplicaCmd do not specify ulog dir in case of HDFS
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ad7f2a78
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ad7f2a78
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ad7f2a78
Branch: refs/heads/jira/solr-11000
Commit: ad7f2a787ddbd978af967a535795e28e1fa70409
Parents: 57f7493
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed Jul 12 14:07:27 2017 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed Jul 12 14:07:27 2017 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/cloud/AddReplicaCmd.java | 4 ++
.../org/apache/solr/cloud/MoveReplicaCmd.java | 3 ++
.../solr/handler/admin/CollectionsHandler.java | 2 +
.../apache/solr/cloud/MoveReplicaHDFSTest.java | 45 ++++++++++++++++++++
.../solrj/request/CollectionAdminRequest.java | 13 ++++++
6 files changed, 69 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad7f2a78/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 02d2364..1f357f7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -330,6 +330,8 @@ Bug Fixes
* SOLR-11039: Next button in Solr admin UI for collection list pagination does not work. (janhoy)
+* SOLR-11041: MoveReplicaCmd do not specify ulog dir in case of HDFS (Cao Manh Dat)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad7f2a78/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
index c42d073..e9bfebf 100644
--- a/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
@@ -139,6 +139,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
String configName = zkStateReader.readConfigName(collection);
String routeKey = message.getStr(ShardParams._ROUTE_);
String dataDir = message.getStr(CoreAdminParams.DATA_DIR);
+ String ulogDir = message.getStr(CoreAdminParams.ULOG_DIR);
String instanceDir = message.getStr(CoreAdminParams.INSTANCE_DIR);
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString());
@@ -161,6 +162,9 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
if (dataDir != null) {
params.set(CoreAdminParams.DATA_DIR, dataDir);
}
+ if (ulogDir != null) {
+ params.set(CoreAdminParams.ULOG_DIR, ulogDir);
+ }
if (instanceDir != null) {
params.set(CoreAdminParams.INSTANCE_DIR, instanceDir);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad7f2a78/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
index 4d6e26d..4598f14 100644
--- a/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
@@ -34,6 +34,7 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.update.UpdateLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -134,11 +135,13 @@ public class MoveReplicaCmd implements Cmd{
return;
}
+ String ulogDir = replica.getStr(CoreAdminParams.ULOG_DIR);
ZkNodeProps addReplicasProps = new ZkNodeProps(
COLLECTION_PROP, coll.getName(),
SHARD_ID_PROP, slice.getName(),
CoreAdminParams.NODE, targetNode,
CoreAdminParams.NAME, newCoreName,
+ CoreAdminParams.ULOG_DIR, ulogDir.substring(0, ulogDir.lastIndexOf(UpdateLog.TLOG_NAME)),
CoreAdminParams.DATA_DIR, dataDir);
if(async!=null) addReplicasProps.getProperties().put(ASYNC, async);
NamedList addResult = new NamedList();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad7f2a78/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 256400e..a4155ea 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -126,6 +126,7 @@ import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR;
import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX;
import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR;
import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR;
+import static org.apache.solr.common.params.CoreAdminParams.ULOG_DIR;
import static org.apache.solr.common.params.ShardParams._ROUTE_;
import static org.apache.solr.common.util.StrUtils.formatString;
@@ -633,6 +634,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
CoreAdminParams.NAME,
INSTANCE_DIR,
DATA_DIR,
+ ULOG_DIR,
REPLICA_TYPE);
return copyPropertiesWithPrefix(req.getParams(), props, COLL_PROP_PREFIX);
}),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad7f2a78/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
index 3677563..c350dd5b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
@@ -16,14 +16,24 @@
*/
package org.apache.solr.cloud;
+import java.io.IOException;
+
import com.carrotsearch.randomizedtesting.ThreadFilter;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
+import org.apache.solr.common.cloud.ClusterStateUtil;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkConfigManager;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Test;
/**
*
@@ -54,6 +64,41 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
dfsCluster = null;
}
+ @Test
+ public void testDataDirAndUlogAreMaintained() throws IOException, SolrServerException {
+ String coll = "movereplicatest_coll2";
+ CollectionAdminRequest.createCollection(coll, "conf1", 1, 1)
+ .setCreateNodeSet("")
+ .process(cluster.getSolrClient());
+ String hdfsUri = HdfsTestUtil.getURI(dfsCluster);
+ String dataDir = hdfsUri + "/dummyFolder/dataDir";
+ String ulogDir = hdfsUri + "/dummyFolder2/ulogDir";
+ CollectionAdminResponse res = CollectionAdminRequest
+ .addReplicaToShard(coll, "shard1")
+ .setDataDir(dataDir)
+ .setUlogDir(ulogDir)
+ .setNode(cluster.getJettySolrRunner(0).getNodeName())
+ .process(cluster.getSolrClient());
+
+ ulogDir += "/tlog";
+ ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000);
+ DocCollection docCollection = zkStateReader.getClusterState().getCollection(coll);
+ Replica replica = docCollection.getReplicas().iterator().next();
+ assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/'));
+ assertTrue(replica.getStr("dataDir"),replica.getStr("dataDir").equals(dataDir) || replica.getStr("dataDir").equals(dataDir+'/'));
+
+ new CollectionAdminRequest.MoveReplica(coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
+ .process(cluster.getSolrClient());
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000);
+ docCollection = zkStateReader.getClusterState().getCollection(coll);
+ assertEquals(1, docCollection.getSlice("shard1").getReplicas().size());
+ replica = docCollection.getReplicas().iterator().next();
+ assertEquals(replica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName());
+ assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/'));
+ assertTrue(replica.getStr("dataDir"),replica.getStr("dataDir").equals(dataDir) || replica.getStr("dataDir").equals(dataDir+'/'));
+
+ }
public static class ForkJoinThreadsFilter implements ThreadFilter {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad7f2a78/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index f039ba4..88100f3 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -1364,6 +1364,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
protected String routeKey;
protected String instanceDir;
protected String dataDir;
+ protected String ulogDir;
protected Properties properties;
protected Replica.Type type;
@@ -1408,6 +1409,10 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return instanceDir;
}
+ public String getUlogDir() {
+ return ulogDir;
+ }
+
public AddReplica setInstanceDir(String instanceDir) {
this.instanceDir = instanceDir;
return this;
@@ -1432,6 +1437,11 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
+ public AddReplica setUlogDir(String ulogDir) {
+ this.ulogDir = ulogDir;
+ return this;
+ }
+
@Override
public SolrParams getParams() {
ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
@@ -1452,6 +1462,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
if (dataDir != null) {
params.add("dataDir", dataDir);
}
+ if (ulogDir != null) {
+ params.add("ulogDir", ulogDir);
+ }
if (coreName != null) {
params.add("name", coreName);
}