You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2016/04/07 18:35:55 UTC
lucene-solr:branch_5x: SOLR-8938: Add optional -excluderegex argument
to ZkCLI.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_5x 515066e2b -> 521b9b101
SOLR-8938: Add optional -excluderegex argument to ZkCLI.
(Resolved conflicts for solr/CHANGES.txt file and added back ZkCLI's OnReconnect import which was removed in branch_6x and master but is still needed in branch_5x.)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/521b9b10
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/521b9b10
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/521b9b10
Branch: refs/heads/branch_5x
Commit: 521b9b1015959f0d2444873afecc615d18340ace
Parents: 515066e
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Apr 7 10:44:08 2016 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Apr 7 16:17:32 2016 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../src/java/org/apache/solr/cloud/ZkCLI.java | 12 +++++-
.../test/org/apache/solr/cloud/ZkCLITest.java | 41 ++++++++++++++------
.../solr/common/cloud/ZkConfigManager.java | 27 +++++++++++--
.../solr/common/cloud/TestZkConfigManager.java | 7 ++++
5 files changed, 73 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/521b9b10/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 192a21c..93bc722 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -17,6 +17,8 @@ New Features
* SOLR-5730: Make Lucene's SortingMergePolicy and EarlyTerminatingSortingCollector configurable in Solr.
(Christine Poerschke, hossmann, Tomás Fernández Löbbe, Shai Erera)
+* SOLR-8938: Add optional -excluderegex argument to ZkCLI. (Christine Poerschke)
+
Other Changes
----------------------
* SOLR-8677: Prevent shards containing invalid characters from being created. Checks added server-side
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/521b9b10/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
index 8d52ea2..5e3ddbb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
@@ -48,6 +48,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.TimeoutException;
+import java.util.regex.Pattern;
public class ZkCLI {
@@ -66,7 +67,9 @@ public class ZkCLI {
private static final String RUNZK = "runzk";
private static final String SOLRHOME = "solrhome";
private static final String BOOTSTRAP = "bootstrap";
- private static final String UPCONFIG = "upconfig";
+ static final String UPCONFIG = "upconfig";
+ static final String EXCLUDE_REGEX = "excluderegex";
+ static final String EXCLUDE_REGEX_DEFAULT = ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX;
private static final String COLLECTION = "collection";
private static final String CLEAR = "clear";
private static final String LIST = "list";
@@ -120,6 +123,9 @@ public class ZkCLI {
options.addOption("c", COLLECTION, true,
"for " + LINKCONFIG + ": name of the collection");
+ options.addOption(EXCLUDE_REGEX, true,
+ "for " + UPCONFIG + ": files matching this regular expression won't be uploaded");
+
options
.addOption(
"r",
@@ -213,13 +219,15 @@ public class ZkCLI {
}
String confDir = line.getOptionValue(CONFDIR);
String confName = line.getOptionValue(CONFNAME);
+ final String excludeExpr = line.getOptionValue(EXCLUDE_REGEX, EXCLUDE_REGEX_DEFAULT);
if(!ZkController.checkChrootPath(zkServerAddress, true)) {
System.out.println("A chroot was specified in zkHost but the znode doesn't exist. ");
System.exit(1);
}
ZkConfigManager configManager = new ZkConfigManager(zkClient);
- configManager.uploadConfigDir(Paths.get(confDir), confName);
+ final Pattern excludePattern = Pattern.compile(excludeExpr);
+ configManager.uploadConfigDir(Paths.get(confDir), confName, excludePattern);
} else if (line.getOptionValue(CMD).equalsIgnoreCase(DOWNCONFIG)) {
if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
System.out.println("-" + CONFDIR + " and -" + CONFNAME
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/521b9b10/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
index 19cea70..81b810b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
@@ -101,6 +101,13 @@ public class ZkCLITest extends SolrTestCaseJ4 {
}
@Test
+ public void testCmdConstants() throws Exception {
+ assertEquals("upconfig", ZkCLI.UPCONFIG);
+ assertEquals("excluderegex", ZkCLI.EXCLUDE_REGEX);
+ assertEquals(ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT);
+ }
+
+ @Test
public void testBootstrapWithChroot() throws Exception {
String chroot = "/foo/bar";
assertFalse(zkClient.exists(chroot, true));
@@ -193,14 +200,22 @@ public class ZkCLITest extends SolrTestCaseJ4 {
// test upconfig
String confsetname = "confsetone";
- String[] args = new String[] {
- "-zkhost",
- zkServer.getZkAddress(),
- "-cmd",
- "upconfig",
- "-confdir",
- ExternalPaths.TECHPRODUCTS_CONFIGSET, "-confname", confsetname};
- ZkCLI.main(args);
+ final String[] upconfigArgs;
+ if (random().nextBoolean()) {
+ upconfigArgs = new String[] {
+ "-zkhost", zkServer.getZkAddress(),
+ "-cmd", ZkCLI.UPCONFIG,
+ "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET,
+ "-confname", confsetname};
+ } else {
+ upconfigArgs = new String[] {
+ "-zkhost", zkServer.getZkAddress(),
+ "-cmd", ZkCLI.UPCONFIG,
+ "-"+ZkCLI.EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT,
+ "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET,
+ "-confname", confsetname};
+ }
+ ZkCLI.main(upconfigArgs);
assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true));
@@ -208,7 +223,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
// ZkCLI.main(new String[0]);
// test linkconfig
- args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"linkconfig", "-collection", "collection1", "-confname", confsetname};
ZkCLI.main(args);
@@ -236,8 +251,12 @@ public class ZkCLITest extends SolrTestCaseJ4 {
int indexOfRelativePath = sourceFile.getAbsolutePath().lastIndexOf("sample_techproducts_configs" + File.separator + "conf");
String relativePathofFile = sourceFile.getAbsolutePath().substring(indexOfRelativePath + 33, sourceFile.getAbsolutePath().length());
File downloadedFile = new File(confDir,relativePathofFile);
- assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists());
- assertTrue("Content didn't change",FileUtils.contentEquals(sourceFile,downloadedFile));
+ if (ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) {
+ assertFalse(sourceFile.getAbsolutePath() + " exists in ZK, downloaded:" + downloadedFile.getAbsolutePath(), downloadedFile.exists());
+ } else {
+ assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists());
+ assertTrue(relativePathofFile+" content changed",FileUtils.contentEquals(sourceFile,downloadedFile));
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/521b9b10/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
index 9123c53..eb87c34 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
@@ -30,6 +30,8 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* Class that manages named configs in Zookeeper
@@ -41,6 +43,9 @@ public class ZkConfigManager {
/** ZkNode where named configs are stored */
public static final String CONFIGS_ZKNODE = "/configs";
+ public static final String UPLOAD_FILENAME_EXCLUDE_REGEX = "^\\..*$";
+ public static final Pattern UPLOAD_FILENAME_EXCLUDE_PATTERN = Pattern.compile(UPLOAD_FILENAME_EXCLUDE_REGEX);
+
private final SolrZkClient zkClient;
/**
@@ -51,7 +56,8 @@ public class ZkConfigManager {
this.zkClient = zkClient;
}
- private void uploadToZK(final Path rootPath, final String zkPath) throws IOException {
+ private void uploadToZK(final Path rootPath, final String zkPath,
+ final Pattern filenameExclusions) throws IOException {
if (!Files.exists(rootPath))
throw new IOException("Path " + rootPath + " does not exist");
@@ -60,8 +66,10 @@ public class ZkConfigManager {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String filename = file.getFileName().toString();
- if (filename.startsWith("."))
+ if (filenameExclusions != null && filenameExclusions.matcher(filename).matches()) {
+ logger.info("uploadToZK skipping '{}' due to filenameExclusions '{}'", filename, filenameExclusions);
return FileVisitResult.CONTINUE;
+ }
String zkNode = createZkNodeName(zkPath, rootPath, file);
try {
zkClient.makePath(zkNode, file.toFile(), false, true);
@@ -118,7 +126,20 @@ public class ZkConfigManager {
* if an I/O error occurs or the path does not exist
*/
public void uploadConfigDir(Path dir, String configName) throws IOException {
- uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName);
+ uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, UPLOAD_FILENAME_EXCLUDE_PATTERN);
+ }
+
+ /**
+ * Upload matching files from a given path to a config in Zookeeper
+ * @param dir {@link java.nio.file.Path} to the files
+ * @param configName the name to give the config
+ * @param filenameExclusions files matching this pattern will not be uploaded
+ * @throws IOException
+ * if an I/O error occurs or the path does not exist
+ */
+ public void uploadConfigDir(Path dir, String configName,
+ Pattern filenameExclusions) throws IOException {
+ uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, filenameExclusions);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/521b9b10/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java
index fd7a0ac..4727bcc 100644
--- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java
+++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java
@@ -19,6 +19,7 @@ package org.apache.solr.common.cloud;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cloud.ZkCLI;
import org.apache.solr.cloud.ZkTestServer;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
@@ -54,6 +55,12 @@ public class TestZkConfigManager extends SolrTestCaseJ4 {
}
@Test
+ public void testConstants() throws Exception {
+ assertEquals("/configs", ZkConfigManager.CONFIGS_ZKNODE);
+ assertEquals("^\\..*$", ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX);
+ }
+
+ @Test
public void testUploadConfig() throws IOException {
zkServer.ensurePathExists("/solr");