You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2013/03/19 06:43:22 UTC
svn commit: r1458152 - in /lucene/dev/branches/lucene_solr_4_2: ./
dev-tools/ lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/classification/ lucene/classification/src/ lucen...
Author: markrmiller
Date: Tue Mar 19 05:43:20 2013
New Revision: 1458152
URL: http://svn.apache.org/r1458152
Log:
SOLR-4601: A Collection that is only partially created and then deleted will leave pre allocated shard information in ZooKeeper.
Modified:
lucene/dev/branches/lucene_solr_4_2/ (props changed)
lucene/dev/branches/lucene_solr_4_2/dev-tools/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/BUILD.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/CHANGES.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/LICENSE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/README.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/analysis/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/backwards/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/benchmark/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/ivy.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/classification/src/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/codecs/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/common-build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/demo/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/facet/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/grouping/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/highlighter/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/join/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/licenses/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/memory/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/misc/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/module-build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/queries/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/queryparser/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/sandbox/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/site/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/spatial/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/suggest/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_4_2/lucene/tools/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_4_2/solr/LICENSE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/README.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/cloud-dev/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/common-build.xml (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/contrib/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/core/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/example/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/scripts/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/site/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/solrj/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/testlogging.properties (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/webapp/ (props changed)
Modified: lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt?rev=1458152&r1=1458151&r2=1458152&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt Tue Mar 19 05:43:20 2013
@@ -98,6 +98,9 @@ Bug Fixes
* SOLR-4602: ZkController#unregister should cancel it's election participation
before asking the Overseer to delete the SolrCore information. (Mark Miller)
+* SOLR-4601: A Collection that is only partially created and then deleted will
+ leave pre allocated shard information in ZooKeeper. (Mark Miller)
+
Optimizations
----------------------
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1458152&r1=1458151&r2=1458152&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/Overseer.java Tue Mar 19 05:43:20 2013
@@ -459,6 +459,7 @@ public class Overseer {
}
Map<String, Slice> newSlices = new LinkedHashMap<String, Slice>();
+ boolean lastSlice = false;
for (Slice slice : coll.getSlices()) {
Replica replica = slice.getReplica(cnn);
if (replica != null) {
@@ -469,6 +470,7 @@ public class Overseer {
// if there are no replicas left for the slice remove it
if (newReplicas.size() == 0) {
slice = null;
+ lastSlice = true;
} else {
slice = new Slice(slice.getName(), newReplicas, slice.getProperties());
}
@@ -478,6 +480,15 @@ public class Overseer {
newSlices.put(slice.getName(), slice);
}
}
+
+ if (lastSlice) {
+ // remove all empty pre allocated slices
+ for (Slice slice : coll.getSlices()) {
+ if (slice.getReplicas().size() == 0) {
+ newSlices.remove(slice.getName());
+ }
+ }
+ }
// if there are no slices left in the collection, remove it?
if (newSlices.size() == 0) {
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1458152&r1=1458151&r2=1458152&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Tue Mar 19 05:43:20 2013
@@ -361,7 +361,7 @@ public class OverseerCollectionProcessor
+ repFactor
+ " on collection "
+ collectionName
- + " is higher than or equal to the number of Solr instances currently live ("
+ + " is higher than or equal to the number of Solr instances currently live or part of your " + CREATE_NODE_SET + "("
+ nodeList.size()
+ "). Its unusual to run two replica of the same slice on the same Solr-instance.");
}
@@ -432,7 +432,7 @@ public class OverseerCollectionProcessor
}
}
- private boolean collectionCmd(ClusterState clusterState, ZkNodeProps message, ModifiableSolrParams params, NamedList results) {
+ private void collectionCmd(ClusterState clusterState, ZkNodeProps message, ModifiableSolrParams params, NamedList results) {
log.info("Executing Collection Cmd : " + params);
String collectionName = message.getStr("name");
@@ -474,7 +474,6 @@ public class OverseerCollectionProcessor
}
}
- int failed = 0;
ShardResponse srsp;
do {
srsp = shardHandler.takeCompletedOrError();
@@ -483,12 +482,6 @@ public class OverseerCollectionProcessor
}
} while (srsp != null);
-
- // if all calls succeeded, return true
- if (failed > 0) {
- return false;
- }
- return true;
}
private void processResponse(NamedList results, ShardResponse srsp) {
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1458152&r1=1458151&r2=1458152&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Tue Mar 19 05:43:20 2013
@@ -141,15 +141,49 @@ public class CollectionsAPIDistributedZk
@Override
public void doTest() throws Exception {
-
+
testNodesUsedByCreate();
testCollectionsAPI();
+ deletePartiallyCreatedCollection();
testErrorHandling();
if (DEBUG) {
super.printLayout();
}
}
+
+ private void deletePartiallyCreatedCollection() throws Exception {
+ final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
+ String collectionName = "halfdeletedcollection";
+ Create createCmd = new Create();
+ createCmd.setCoreName("halfdeletedcollection_shard1_replica1");
+ createCmd.setCollection(collectionName);
+ String dataDir = SolrTestCaseJ4.dataDir.getAbsolutePath() + File.separator
+ + System.currentTimeMillis() + "halfcollection" + "_hdn";
+ createCmd.setDataDir(dataDir);
+ createCmd.setNumShards(2);
+ createNewSolrServer("", baseUrl).request(createCmd);
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.DELETE.toString());
+ params.set("name", collectionName);
+ QueryRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList<Object> resp = createNewSolrServer("", baseUrl).request(request);
+
+ checkForMissingCollection(collectionName);
+
+ // now creating that collection should work
+ params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.CREATE.toString());
+ params.set("name", collectionName);
+ params.set("numShards", 2);
+ request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ resp = createNewSolrServer("", baseUrl).request(request);
+
+ }
private void testErrorHandling() throws Exception {
final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
@@ -160,6 +194,7 @@ public class CollectionsAPIDistributedZk
params.set("action", "BADACTION");
String collectionName = "badactioncollection";
params.set("name", collectionName);
+ params.set("numShards", 2);
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
boolean gotExp = false;
@@ -175,6 +210,7 @@ public class CollectionsAPIDistributedZk
// leave out required param name
params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
+ params.set("numShards", 2);
collectionName = "collection";
// No Name
// params.set("name", collectionName);