You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by er...@apache.org on 2016/07/03 02:16:21 UTC
lucene-solr:branch_6x: SOLR-9194: Enhance the bin/solr script to
perform file operations to/from Zookeeper - Fix Windows failures
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x ee0dbf55f -> dd761584c
SOLR-9194: Enhance the bin/solr script to perform file operations to/from Zookeeper - Fix Windows failures
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/dd761584
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/dd761584
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/dd761584
Branch: refs/heads/branch_6x
Commit: dd761584c7715b205a30ea2781dc19406b7d06e7
Parents: ee0dbf5
Author: Erick Erickson <er...@apache.org>
Authored: Sat Jul 2 16:07:38 2016 -0700
Committer: Erick Erickson <er...@apache.org>
Committed: Sat Jul 2 19:15:21 2016 -0700
----------------------------------------------------------------------
.../src/java/org/apache/solr/util/SolrCLI.java | 6 +-
.../apache/solr/cloud/SolrCLIZkUtilsTest.java | 82 ++++++++++----------
.../solr/common/cloud/ZkConfigManager.java | 37 +++++----
3 files changed, 68 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dd761584/solr/core/src/java/org/apache/solr/util/SolrCLI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
index d6f4203..ff5d483 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
@@ -1441,7 +1441,7 @@ public class SolrCLI {
} else if (configExistsInZk) {
echo("Re-using existing configuration directory "+confname);
} else {
- Path confPath = ZkConfigManager.getConfigsetPath(confname, cli.getOptionValue("confdir", DEFAULT_CONFIG_SET),
+ Path confPath = ZkConfigManager.getConfigsetPath(cli.getOptionValue("confdir", DEFAULT_CONFIG_SET),
cli.getOptionValue("configsetsDir"));
echo("Uploading " + confPath.toAbsolutePath().toString() +
@@ -1687,7 +1687,7 @@ public class SolrCLI {
OptionBuilder
.withArgName("configsetsDir")
.hasArg()
- .isRequired()
+ .isRequired(false)
.withDescription("Parent directory of example configsets")
.create("configsetsDir"),
OptionBuilder
@@ -1714,7 +1714,7 @@ public class SolrCLI {
String confName = cli.getOptionValue("confname");
try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
echo("\nConnecting to ZooKeeper at " + zkHost + " ...");
- Path confPath = ZkConfigManager.getConfigsetPath(confName, cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir"));
+ Path confPath = ZkConfigManager.getConfigsetPath(cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir"));
echo("Uploading " + confPath.toAbsolutePath().toString() +
" for config " + cli.getOptionValue("confname") + " to ZooKeeper at " + zkHost);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dd761584/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
index d6c0bc6..91c0126 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java
@@ -60,11 +60,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test
public void testUpconfig() throws Exception {
// Use a full, explicit path for configset.
- Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
- Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
- copyConfigUp(src, configSet, "upconfig1");
+
+ Path configSet = TEST_PATH().resolve("configsets");
+ Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
+ copyConfigUp(configSet, "cloud-subdirs", "upconfig1");
// Now do we have that config up on ZK?
- verifyZkLocalPathsMatch(src, "/configs/upconfig1");
+ verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig1");
// Now just use a name in the configsets directory, do we find it?
configSet = TEST_PATH().resolve("configsets");
@@ -81,7 +82,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
assertEquals("tool should have returned 0 for success ", 0, res);
// Now do we have that config up on ZK?
- verifyZkLocalPathsMatch(src, "/configs/upconfig2");
+ verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig2");
// do we barf on a bogus path?
args = new String[]{
@@ -104,11 +105,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
Path tmp = createTempDir("downConfigNewPlace");
// First we need a configset on ZK to bring down.
- Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
- Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
- copyConfigUp(src, configSet, "downconfig1");
+
+ Path configSet = TEST_PATH().resolve("configsets");
+ Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
+ copyConfigUp(configSet, "cloud-subdirs", "downconfig1");
// Now do we have that config up on ZK?
- verifyZkLocalPathsMatch(src, "/configs/downconfig1");
+ verifyZkLocalPathsMatch(srcPathCheck, "/configs/downconfig1");
String[] args = new String[]{
"-confname", "downconfig1",
@@ -126,10 +128,11 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test
public void testCp() throws Exception {
// First get something up on ZK
- Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
- Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
- copyConfigUp(src, configSet, "cp1");
+ Path configSet = TEST_PATH().resolve("configsets");
+ Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
+
+ copyConfigUp(configSet, "cloud-subdirs", "cp1");
// Now copy it somewhere else on ZK.
String[] args = new String[]{
@@ -176,7 +179,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// try with local->zk
args = new String[]{
- "-src", src.toAbsolutePath().toString(),
+ "-src", srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp3",
"-recurse", "true",
"-zkHost", zkAddr,
@@ -184,11 +187,11 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
assertEquals("Copy should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(src, "/cp3");
+ verifyZkLocalPathsMatch(srcPathCheck, "/cp3");
// try with local->zk, file: specified
args = new String[]{
- "-src", "file:" + src.toAbsolutePath().toString(),
+ "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp4",
"-recurse", "true",
"-zkHost", zkAddr,
@@ -196,12 +199,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
assertEquals("Copy should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(src, "/cp4");
+ verifyZkLocalPathsMatch(srcPathCheck, "/cp4");
// try with recurse not specified
args = new String[]{
- "-src", "file:" + src.toAbsolutePath().toString(),
+ "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp5Fail",
"-zkHost", zkAddr,
};
@@ -211,7 +214,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// try with recurse = false
args = new String[]{
- "-src", "file:" + src.toAbsolutePath().toString(),
+ "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
"-dst", "zk:/cp6Fail",
"-recurse", "false",
"-zkHost", zkAddr,
@@ -241,7 +244,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// copy to ZK ending in '/'.
//src and cp3 are valid
args = new String[]{
- "-src", "file:" + src.normalize().toAbsolutePath().toString() + "/solrconfig.xml",
+ "-src", "file:" + srcPathCheck.normalize().toAbsolutePath().toString() + "/solrconfig.xml",
"-dst", "zk:/powerup/",
"-recurse", "false",
"-zkHost", zkAddr,
@@ -254,7 +257,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// copy individual file up
//src and cp3 are valid
args = new String[]{
- "-src", "file:" + src.normalize().toAbsolutePath().toString() + "/solrconfig.xml",
+ "-src", "file:" + srcPathCheck.normalize().toAbsolutePath().toString() + "/solrconfig.xml",
"-dst", "zk:/copyUpFile.xml",
"-recurse", "false",
"-zkHost", zkAddr,
@@ -309,10 +312,11 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
public void testMv() throws Exception {
// First get something up on ZK
- Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
- Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
- copyConfigUp(src, configSet, "mv1");
+ Path configSet = TEST_PATH().resolve("configsets");
+ Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
+
+ copyConfigUp(configSet, "cloud-subdirs", "mv1");
// Now move it somewhere else.
String[] args = new String[]{
@@ -327,7 +331,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertEquals("Move should have succeeded.", 0, res);
// Now does the moved directory match the original on disk?
- verifyZkLocalPathsMatch(src, "/mv2");
+ verifyZkLocalPathsMatch(srcPathCheck, "/mv2");
// And are we sure the old path is gone?
assertFalse("/configs/mv1 Znode should not be there: ", zkClient.exists("/configs/mv1", true));
@@ -356,7 +360,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertFalse("Znode /mv3 really should be gone", zkClient.exists("/mv3", true));
// Now does the moved directory match the original on disk?
- verifyZkLocalPathsMatch(src, "/mv4");
+ verifyZkLocalPathsMatch(srcPathCheck, "/mv4");
args = new String[]{
"-src", "/mv4/solrconfig.xml",
@@ -387,10 +391,9 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test
public void testLs() throws Exception {
- Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
- Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
+ Path configSet = TEST_PATH().resolve("configsets");
- copyConfigUp(src, configSet, "lister");
+ copyConfigUp(configSet, "cloud-subdirs", "lister");
// Should only find a single level.
String[] args = new String[]{
@@ -485,11 +488,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
@Test
public void testRm() throws Exception {
- Path src = TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf");
- Path configSet = TEST_PATH().resolve("configsets").resolve("cloud-subdirs");
-
- copyConfigUp(src, configSet, "rm1");
- copyConfigUp(src, configSet, "rm2");
+
+ Path configSet = TEST_PATH().resolve("configsets");
+ Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
+
+ copyConfigUp(configSet, "cloud-subdirs", "rm1");
+ copyConfigUp(configSet, "cloud-subdirs", "rm2");
// Should fail if recurse not set.
String[] args = new String[]{
@@ -504,7 +508,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
assertTrue("Should have failed to remove node with children unless -recurse is set to true", res != 0);
// Are we sure all the znodes are still there?
- verifyZkLocalPathsMatch(src, "/configs/rm1");
+ verifyZkLocalPathsMatch(srcPathCheck, "/configs/rm1");
args = new String[]{
"-path", "zk:/configs/rm1",
@@ -545,7 +549,7 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
"-zkHost", zkAddr,
};
- copyConfigUp(src, configSet, "rm3");
+ copyConfigUp(configSet, "cloud-subdirs", "rm3");
res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
assertFalse("Should fail when trying to remove /.", res == 0);
}
@@ -559,12 +563,12 @@ public class SolrCLIZkUtilsTest extends SolrCloudTestCase {
// We can use this for testing since the goal is to move "some stuff" up to ZK.
// The fact that they're in configsets is irrelevant.
- private void copyConfigUp(Path src, Path configSet, String confName) throws Exception {
+ private void copyConfigUp(Path configSetDir, String srcConfigSet, String dstConfigName) throws Exception {
String[] args = new String[]{
- "-confname", confName,
- "-confdir", src.toAbsolutePath().toString(),
+ "-confname", dstConfigName,
+ "-confdir", srcConfigSet,
"-zkHost", zkAddr,
- "-configsetsDir", configSet.toAbsolutePath().toString(),
+ "-configsetsDir", configSetDir.toAbsolutePath().toString(),
};
SolrCLI.ConfigSetUploadTool tool = new SolrCLI.ConfigSetUploadTool();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/dd761584/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
index c626a37..0b3608b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
@@ -180,34 +180,41 @@ public class ZkConfigManager {
// This method is used by configSetUploadTool and CreateTool to resolve the configset directory.
// Check several possibilities:
- // 1> configSetDirParam + confname is defined in configsets, thus has a conf/solrconfig.xml
- // 2> configSet is a path that contains conf/solrconfig.xml
- // 3> configSet is t
- public static Path getConfigsetPath(String confname, String configSet, String configSetDirParam) throws IOException {
+ // 1> confDir/solrconfig.xml exists
+ // 2> confDir/conf/solrconfig.xml exists
+ // 3> configSetDir/confDir/conf/solrconfig.xml exists (canned configs)
+
+ // Order is important here since "confDir" may be
+ // 1> a full path to the parent of a solrconfig.xml or parent of /conf/solrconfig.xml
+ // 2> one of the canned config sets only, e.g. basic_configs
+ // and trying to assemble a path for configsetDir/confDir is A Bad Idea. if confDir is a full path.
+
+ public static Path getConfigsetPath(String confDir, String configSetDir) throws IOException {
- // one of the canned configsets.
- Path ret = Paths.get(configSetDirParam, configSet, "conf", "solrconfig.xml").normalize();
+ // A local path to the source, probably already includes "conf".
+ Path ret = Paths.get(confDir, "solrconfig.xml").normalize();
if (Files.exists(ret)) {
- return Paths.get(configSetDirParam, configSet, "conf").normalize();
+ return Paths.get(confDir).normalize();
}
// a local path to the parent of a "conf" directory
- ret = Paths.get(configSet, "conf", "solrconfig.xml").normalize();
+ ret = Paths.get(confDir, "conf", "solrconfig.xml").normalize();
if (Files.exists(ret)) {
- return Paths.get(configSet, "conf").normalize();
+ return Paths.get(confDir, "conf").normalize();
}
- // A local path to the source, probably already includes "conf".
- ret = Paths.get(configSet, "solrconfig.xml").normalize();
+ // one of the canned configsets.
+ ret = Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize();
if (Files.exists(ret)) {
- return Paths.get(configSet).normalize();
+ return Paths.get(configSetDir, confDir, "conf").normalize();
}
+
throw new IllegalArgumentException(String.format(Locale.ROOT,
"Could not find solrconfig.xml at %s, %s or %s",
- Paths.get(configSetDirParam, configSet, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(),
- Paths.get(configSet, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(),
- Paths.get(configSet, "solrconfig.xml").normalize().toAbsolutePath().toString()
+ Paths.get(configSetDir, "solrconfig.xml").normalize().toAbsolutePath().toString(),
+ Paths.get(configSetDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString(),
+ Paths.get(configSetDir, confDir, "conf", "solrconfig.xml").normalize().toAbsolutePath().toString()
));
}
}