You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by er...@apache.org on 2013/08/16 19:48:09 UTC
svn commit: r1514821 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/classification/ lucene/classification/src/
lucene/code...
Author: erick
Date: Fri Aug 16 17:48:08 2013
New Revision: 1514821
URL: http://svn.apache.org/r1514821
Log:
SOLR-5156 Provide a way to move the contents of a file to ZooKeeper with ZkCLI
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ (props changed)
lucene/dev/branches/branch_4x/lucene/classification/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ivy.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/src/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/replicator/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1514821&r1=1514820&r2=1514821&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Fri Aug 16 17:48:08 2013
@@ -63,7 +63,8 @@ New Features
* SOLR-4808: Persist and use router,replicationFactor and maxShardsPerNode at Collection and Shard level (Noble Paul, Shalin Mangar)
* SOLR-5006: CREATESHARD command for 'implicit' shards (Noble Paul)
* SOLR-5017: Allow sharding based on the value of a field (Noble Paul)
-* SOLR-4222:create custom sharded collection via collections API (Noble Paul)
+* SOLR-4222: create custom sharded collection via collections API (Noble Paul)
+* SOLR-5156: Enhance ZkCLI to allow uploading of arbitrary files to ZK.
Bug Fixes
----------------------
@@ -105,8 +106,6 @@ Bug Fixes
of divide by zero, and makes estimated hit counts meaningful in non-optimized
indexes. (hossman)
-* SOLR-5164: Can not create a collection via collections API (cloud mode) (Erick Erickson)
-
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java?rev=1514821&r1=1514820&r2=1514821&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java Fri Aug 16 17:48:08 2013
@@ -3,6 +3,7 @@ package org.apache.solr.cloud;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -19,7 +20,9 @@ import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.List;
import java.util.concurrent.TimeoutException;
@@ -44,6 +47,7 @@ public class ZkCLI {
private static final String MAKEPATH = "makepath";
private static final String PUT = "put";
+ private static final String PUT_FILE = "putfile";
private static final String DOWNCONFIG = "downconfig";
private static final String ZK_CLI_NAME = "ZkCLI";
private static final String HELP = "help";
@@ -87,7 +91,8 @@ public class ZkCLI {
.hasArg(true)
.withDescription(
"cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG
- + ", " + LINKCONFIG + ", " + MAKEPATH + ", "+ PUT + ", "+ LIST + ", " + CLEAR).create(CMD));
+ + ", " + LINKCONFIG + ", " + MAKEPATH + ", " + PUT + ", " + PUT_FILE + ","
+ + LIST + ", " + CLEAR).create(CMD));
Option zkHostOption = new Option("z", ZKHOST, true,
"ZooKeeper host address");
@@ -131,6 +136,7 @@ public class ZkCLI {
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LINKCONFIG + " -" + COLLECTION + " collection1" + " -" + CONFNAME + " myconf");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'");
+ System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT_FILE + " /solr.xml /User/myuser/solr/solr.xml");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
return;
@@ -244,6 +250,20 @@ public class ZkCLI {
}
zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes("UTF-8"),
acl, CreateMode.PERSISTENT, true);
+ } else if (line.getOptionValue(CMD).equals(PUT_FILE)) {
+ List arglist = line.getArgList();
+ if (arglist.size() != 2) {
+ System.out.println("-" + PUT_FILE + " requires two args - the path to create in ZK and the path to the local file");
+ System.exit(1);
+ }
+ InputStream is = new FileInputStream(arglist.get(1).toString());
+ try {
+ zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+
}
} finally {
if (solrPort != null) {
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java?rev=1514821&r1=1514820&r2=1514821&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java Fri Aug 16 17:48:08 2013
@@ -18,10 +18,14 @@ package org.apache.solr.cloud;
*/
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
import java.util.Collection;
import java.util.List;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -50,7 +54,8 @@ public class ZkCLITest extends SolrTestC
private String solrHome;
private SolrZkClient zkClient;
-
+
+ protected static final String SOLR_HOME = SolrTestCaseJ4.TEST_HOME();
@BeforeClass
public static void beforeClass() {
@@ -141,7 +146,7 @@ public class ZkCLITest extends SolrTestC
@Test
public void testPut() throws Exception {
- // test bootstrap_conf
+ // test put
String data = "my data";
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"put", "/data.txt", data};
@@ -151,7 +156,41 @@ public class ZkCLITest extends SolrTestC
assertArrayEquals(zkClient.getData("/data.txt", null, null, true), data.getBytes("UTF-8"));
}
-
+
+ @Test
+ public void testPutFile() throws Exception {
+ // test put file
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+ "putfile", "/solr.xml", SOLR_HOME + File.separator + "solr-stress-new.xml"};
+ ZkCLI.main(args);
+
+ String fromZk = new String(zkClient.getData("/solr.xml", null, null, true), "UTF-8");
+ File locFile = new File(SOLR_HOME + File.separator + "solr-stress-new.xml");
+ InputStream is = new FileInputStream(locFile);
+ String fromLoc;
+ try {
+ fromLoc = new String(IOUtils.toByteArray(is), "UTF-8");
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ assertEquals("Should get back what we put in ZK", fromZk, fromLoc);
+ }
+
+ @Test
+ public void testPutFileNotExists() throws Exception {
+ // test put file
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+ "putfile", "/solr.xml", SOLR_HOME + File.separator + "not-there.xml"};
+ try {
+ ZkCLI.main(args);
+ fail("Should have had a file not found exception");
+ } catch (FileNotFoundException fne) {
+ String msg = fne.getMessage();
+ assertTrue("Didn't find expected error message containing 'not-there.xml' in " + msg,
+ msg.indexOf("not-there.xml") != -1);
+ }
+ }
+
@Test
public void testList() throws Exception {
zkClient.makePath("/test", true);