You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/12/14 21:27:24 UTC
svn commit: r1214448 - in /lucene/dev/branches/solrcloud/solr:
core/src/java/org/apache/solr/cloud/
core/src/java/org/apache/solr/update/processor/
core/src/test/org/apache/solr/cloud/
solrj/src/java/org/apache/solr/common/cloud/
Author: markrmiller
Date: Wed Dec 14 20:27:23 2011
New Revision: 1214448
URL: http://svn.apache.org/viewvc?rev=1214448&view=rev
Log:
move getting the shard info to the cluster state - its headed there later anyhow
Added:
lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java (contents, props changed)
- copied, changed from r1214399, lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/HashPartitioner.java
Removed:
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/HashPartitioner.java
Modified:
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java
Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1214448&r1=1214447&r2=1214448&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Wed Dec 14 20:27:23 2011
@@ -19,22 +19,19 @@ package org.apache.solr.update.processor
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.solr.cloud.CloudDescriptor;
-import org.apache.solr.cloud.HashPartitioner;
-import org.apache.solr.cloud.HashPartitioner.Range;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.cloud.CloudState;
+import org.apache.solr.common.cloud.HashPartitioner;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -183,30 +180,10 @@ public class DistributedUpdateProcessor
}
private String getShard(int hash, String collection, CloudState cloudState) {
- // nocommit: we certainly don't want to do this every update request...
+ // ranges should be part of the cloud state and eventually gotten from zk
+
// get the shard names
- Map<String,Slice> slices = cloudState.getSlices(collection);
-
- if (slices == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find collection "
- + collection + " in " + cloudState);
- }
-
- Set<String> shards = slices.keySet();
- List<String> shardList = new ArrayList<String>(shards.size());
- shardList.addAll(shards);
- Collections.sort(shardList);
- hp = new HashPartitioner();
- List<Range> ranges = hp.partitionRange(shards.size());
- int cnt = 0;
- for (Range range : ranges) {
- if (hash < range.max) {
- return shardList.get(cnt);
- }
- cnt++;
- }
-
- throw new IllegalStateException("The HashPartitioner failed");
+ return cloudState.getShard(hash, collection);
}
@Override
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java?rev=1214448&r1=1214447&r2=1214448&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java Wed Dec 14 20:27:23 2011
@@ -20,7 +20,8 @@ package org.apache.solr.cloud;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.cloud.HashPartitioner.Range;
+import org.apache.solr.common.cloud.HashPartitioner;
+import org.apache.solr.common.cloud.HashPartitioner.Range;
public class TestHashPartitioner extends SolrTestCaseJ4 {
Modified: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java?rev=1214448&r1=1214447&r2=1214448&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java (original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java Wed Dec 14 20:27:23 2011
@@ -17,9 +17,19 @@ package org.apache.solr.common.cloud;
* limitations under the License.
*/
-import java.util.*;
-
-import org.apache.noggit.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.noggit.JSONWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.HashPartitioner.Range;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,6 +39,11 @@ public class CloudState implements JSONW
protected static Logger log = LoggerFactory.getLogger(CloudState.class);
private final Map<String, Map<String,Slice>> collectionStates; // Map<collectionName, Map<sliceName,Slice>>
private final Set<String> liveNodes;
+
+ private HashPartitioner hp = new HashPartitioner();
+
+ private final Map<String,RangeInfo> rangeInfos = new HashMap<String,RangeInfo>();
+
public CloudState() {
this.liveNodes = new HashSet<String>();
@@ -87,13 +102,56 @@ public class CloudState implements JSONW
return Collections.unmodifiableSet(liveNodes);
}
-// public void setLiveNodes(Set<String> liveNodes) {
-// this.liveNodes = liveNodes;
-// }
-
public boolean liveNodesContain(String name) {
return liveNodes.contains(name);
}
+
+ public RangeInfo getRanges(String collection) {
+ List<Range> ranges;
+ RangeInfo rangeInfo;
+ // TODO: store this in zk
+ // we could double check lock?
+ synchronized (rangeInfos) {
+ rangeInfo = rangeInfos.get(collection);
+ if (rangeInfo == null) {
+ rangeInfo = new RangeInfo();
+
+ Map<String,Slice> slices = getSlices(collection);
+
+ if (slices == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find collection "
+ + collection + " in " + this);
+ }
+
+ Set<String> shards = slices.keySet();
+ ArrayList<String> shardList = new ArrayList<String>(shards.size());
+ shardList.addAll(shards);
+ Collections.sort(shardList);
+
+ ranges = hp.partitionRange(shards.size());
+
+ rangeInfo.ranges = ranges;
+ rangeInfo.shardList = shardList;
+ rangeInfos.put(collection, rangeInfo);
+ }
+ }
+
+ return rangeInfo;
+ }
+
+ public String getShard(int hash, String collection) {
+ RangeInfo rangInfo = getRanges(collection);
+
+ int cnt = 0;
+ for (Range range : rangInfo.ranges) {
+ if (hash < range.max) {
+ return rangInfo.shardList.get(cnt);
+ }
+ cnt++;
+ }
+
+ throw new IllegalStateException("The HashPartitioner failed");
+ }
@Override
public String toString() {
@@ -138,6 +196,11 @@ public class CloudState implements JSONW
public void write(JSONWriter jsonWriter) {
jsonWriter.write(collectionStates);
}
+
+ class RangeInfo {
+ private List<Range> ranges;
+ private ArrayList<String> shardList;
+ }
}
Copied: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java (from r1214399, lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/HashPartitioner.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java?p2=lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java&p1=lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/HashPartitioner.java&r1=1214399&r2=1214448&rev=1214448&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/HashPartitioner.java (original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java Wed Dec 14 20:27:23 2011
@@ -1,4 +1,4 @@
-package org.apache.solr.cloud;
+package org.apache.solr.common.cloud;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more