You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/08/18 18:26:58 UTC
svn commit: r1618655 [2/2] - in /lucene/dev/branches/branch_4x: ./ solr/
solr/contrib/
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/
solr/core/ solr/core/src/java/org/apache/solr/cloud/
solr/core/src/java/org/apache/solr/c...
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java Mon Aug 18 16:26:56 2014
@@ -52,9 +52,9 @@ import java.util.Random;
import java.util.Set;
import org.apache.lucene.util.LuceneTestCase.Slow;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
@Slow
public class ShardSplitTest extends BasicDistributedZkTest {
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java Mon Aug 18 16:26:56 2014
@@ -50,9 +50,9 @@ public class SliceStateTest extends Solr
collectionStates.put("collection1", new DocCollection("collection1", slices, null, DocRouter.DEFAULT));
ZkStateReader mockZkStateReader = ClusterStateTest.getMockZkStateReader(collectionStates.keySet());
- ClusterState clusterState = new ClusterState(-1,liveNodes, collectionStates, mockZkStateReader);
+ ClusterState clusterState = new ClusterState(-1,liveNodes, collectionStates);
byte[] bytes = ZkStateReader.toJSON(clusterState);
- ClusterState loadedClusterState = ClusterState.load(-1, bytes, liveNodes, mockZkStateReader);
+ ClusterState loadedClusterState = ClusterState.load(-1, bytes, liveNodes);
assertEquals("Default state not set to active", "active", loadedClusterState.getSlice("collection1", "shard1").getState());
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Mon Aug 18 16:26:56 2014
@@ -28,11 +28,14 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.ConfigSolr;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.CoresLocator;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.util.ExternalPaths;
import org.apache.zookeeper.CreateMode;
import org.junit.AfterClass;
@@ -316,6 +319,31 @@ public class ZkControllerTest extends So
public void load() {};
@Override
+ public ConfigSolr getConfig() {
+ return new ConfigSolr() {
+
+ @Override
+ public CoresLocator getCoresLocator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected String getShardHandlerFactoryConfigPath() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isPersistent() {
+ throw new UnsupportedOperationException();
+ }};
+ }
+
+ @Override
+ public UpdateShardHandler getUpdateShardHandler() {
+ return new UpdateShardHandler(null);
+ }
+
+ @Override
public String getAdminPath() {
return "/admin/cores";
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java Mon Aug 18 16:26:56 2014
@@ -61,12 +61,14 @@ public class HdfsTestUtil {
System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache");
System.setProperty("solr.lock.type", "hdfs");
- System.setProperty("solr.hdfs.home", "/solr_hdfs_home");
System.setProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean()));
final MiniDFSCluster dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null);
dfsCluster.waitActive();
+
+ System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
+
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Mon Aug 18 16:26:56 2014
@@ -138,10 +138,9 @@ public class TestReplicationHandler exte
}
private static JettySolrRunner createJetty(SolrInstance instance) throws Exception {
- System.setProperty("solr.data.dir", instance.getDataDir());
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));
JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), "/solr", 0);
-
+ jetty.setDataDir(instance.getDataDir());
jetty.start();
return jetty;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java Mon Aug 18 16:26:56 2014
@@ -63,10 +63,9 @@ public class TestReplicationHandlerBacku
String backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM;
private static JettySolrRunner createJetty(TestReplicationHandler.SolrInstance instance) throws Exception {
- System.setProperty("solr.data.dir", instance.getDataDir());
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));
JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), "/solr", 0);
-
+ jetty.setDataDir(instance.getDataDir());
jetty.start();
return jetty;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java Mon Aug 18 16:26:56 2014
@@ -89,8 +89,6 @@ public class TestRecoveryHdfs extends So
throw new RuntimeException(e);
}
- //hdfsDataDir = hdfsUri + "/solr/shard1";
- // System.setProperty("solr.data.dir", hdfsUri + "/solr/shard1");
System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");
initCore("solrconfig-tlog.xml","schema15.xml");
@@ -99,7 +97,6 @@ public class TestRecoveryHdfs extends So
@AfterClass
public static void afterClass() throws Exception {
System.clearProperty("solr.ulog.dir");
- System.clearProperty("solr.data.dir");
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
deleteCore();
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java Mon Aug 18 16:26:56 2014
@@ -76,6 +76,7 @@ public class CollectionAdminRequest exte
protected Integer numShards;
protected Integer maxShardsPerNode;
protected Integer replicationFactor;
+ protected Boolean autoAddReplicas;
public Create() {
@@ -89,6 +90,7 @@ public class CollectionAdminRequest exte
public void setRouterField(String routerField) { this.routerField = routerField; }
public void setNumShards(Integer numShards) {this.numShards = numShards;}
public void setMaxShardsPerNode(Integer numShards) { this.maxShardsPerNode = numShards; }
+ public void setAutoAddReplicas(boolean autoAddReplicas) { this.autoAddReplicas = autoAddReplicas; }
public void setReplicationFactor(Integer repl) { this.replicationFactor = repl; }
public String getConfigName() { return configName; }
@@ -98,6 +100,7 @@ public class CollectionAdminRequest exte
public Integer getNumShards() { return numShards; }
public Integer getMaxShardsPerNode() { return maxShardsPerNode; }
public Integer getReplicationFactor() { return replicationFactor; }
+ public Boolean getAutoAddReplicas() { return autoAddReplicas; }
@Override
public SolrParams getParams() {
@@ -131,6 +134,9 @@ public class CollectionAdminRequest exte
if (asyncId != null) {
params.set("async", asyncId);
}
+ if (autoAddReplicas != null) {
+ params.set(ZkStateReader.AUTO_ADD_REPLICAS, autoAddReplicas);
+ }
return params;
}
@@ -384,6 +390,28 @@ public class CollectionAdminRequest exte
{
return createCollection(name, shards, repl, maxShards, nodeSet, conf, routerField, server, null);
}
+
+ // creates collection using a compositeId router
+ public static CollectionAdminResponse createCollection( String name,
+ Integer shards, Integer repl, Integer maxShards,
+ String nodeSet,
+ String conf,
+ String routerField,
+ Boolean autoAddReplicas,
+ SolrServer server) throws SolrServerException, IOException
+ {
+ Create req = new Create();
+ req.setCollectionName(name);
+ req.setRouterName("compositeId");
+ req.setNumShards(shards);
+ req.setReplicationFactor(repl);
+ req.setMaxShardsPerNode(maxShards);
+ req.setCreateNodeSet(nodeSet);
+ req.setConfigName(conf);
+ req.setRouterField(routerField);
+ req.setAutoAddReplicas(autoAddReplicas);
+ return req.process( server );
+ }
// creates collection using a compositeId router
public static CollectionAdminResponse createCollection( String name,
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClosableThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClosableThread.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClosableThread.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClosableThread.java Mon Aug 18 16:26:56 2014
@@ -17,6 +17,10 @@ package org.apache.solr.common.cloud;
* limitations under the License.
*/
+/**
+ * @deprecated because this class is no longer used internally and will be removed
+ */
+@Deprecated
public interface ClosableThread {
public void close();
public boolean isClosed();
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java Mon Aug 18 16:26:56 2014
@@ -46,7 +46,6 @@ public class ClusterState implements JSO
private final Map<String, DocCollection> collectionStates; // Map<collectionName, Map<sliceName,Slice>>
private Set<String> liveNodes;
- private final ZkStateReader stateReader;
/**
@@ -54,36 +53,29 @@ public class ClusterState implements JSO
*
* hashCode and equals will only depend on liveNodes and not clusterStateVersion.
*/
+ @Deprecated
public ClusterState(Set<String> liveNodes,
Map<String, DocCollection> collectionStates) {
- this(null, liveNodes, collectionStates, null);
+ this(null, liveNodes, collectionStates);
}
- /**
- * @deprecated prefer another constructor
- */
- public ClusterState(Integer zkClusterStateVersion, Set<String> liveNodes,
- Map<String, DocCollection> collectionStates) {
- this(zkClusterStateVersion, liveNodes, collectionStates, null);
- }
/**
* Use this constr when ClusterState is meant for consumption.
*/
public ClusterState(Integer zkClusterStateVersion, Set<String> liveNodes,
- Map<String, DocCollection> collectionStates, ZkStateReader stateReader) {
+ Map<String, DocCollection> collectionStates) {
this.zkClusterStateVersion = zkClusterStateVersion;
this.liveNodes = new HashSet<>(liveNodes.size());
this.liveNodes.addAll(liveNodes);
this.collectionStates = new LinkedHashMap<>(collectionStates.size());
this.collectionStates.putAll(collectionStates);
- this.stateReader = stateReader;
}
public ClusterState copyWith(Map<String,DocCollection> modified){
- ClusterState result = new ClusterState(zkClusterStateVersion, liveNodes,collectionStates,stateReader);
+ ClusterState result = new ClusterState(zkClusterStateVersion, liveNodes,collectionStates);
for (Entry<String, DocCollection> e : modified.entrySet()) {
DocCollection c = e.getValue();
if(c == null) {
@@ -100,94 +92,92 @@ public class ClusterState implements JSO
* Get the lead replica for specific collection, or null if one currently doesn't exist.
*/
public Replica getLeader(String collection, String sliceName) {
- DocCollection coll = collectionStates.get(collection);
+ DocCollection coll = getCollectionOrNull(collection);
if (coll == null) return null;
Slice slice = coll.getSlice(sliceName);
if (slice == null) return null;
return slice.getLeader();
}
-
- /**
- * Gets the replica by the core name (assuming the slice is unknown) or null if replica is not found.
- * If the slice is known, do not use this method.
- * coreNodeName is the same as replicaName
- */
- public Replica getReplica(final String collection, final String coreNodeName) {
- return getReplica(collectionStates.get(collection), coreNodeName);
- }
-
private Replica getReplica(DocCollection coll, String replicaName) {
if (coll == null) return null;
- for(Slice slice: coll.getSlices()) {
+ for (Slice slice : coll.getSlices()) {
Replica replica = slice.getReplica(replicaName);
if (replica != null) return replica;
}
return null;
}
- public boolean hasCollection(String coll){
- return collectionStates.get(coll)!=null;
+
+ public boolean hasCollection(String coll) {
+ return collectionStates.containsKey(coll) ;
}
+ /**
+ * Gets the replica by the core name (assuming the slice is unknown) or null if replica is not found.
+ * If the slice is known, do not use this method.
+ * coreNodeName is the same as replicaName
+ */
+ public Replica getReplica(final String collection, final String coreNodeName) {
+ return getReplica(collectionStates.get(collection), coreNodeName);
+ }
/**
* Get the named Slice for collection, or null if not found.
*/
public Slice getSlice(String collection, String sliceName) {
- DocCollection coll = collectionStates.get(collection);
+ DocCollection coll = getCollectionOrNull(collection);
if (coll == null) return null;
return coll.getSlice(sliceName);
}
public Map<String, Slice> getSlicesMap(String collection) {
- DocCollection coll = collectionStates.get(collection);
+ DocCollection coll = getCollectionOrNull(collection);
if (coll == null) return null;
return coll.getSlicesMap();
}
public Map<String, Slice> getActiveSlicesMap(String collection) {
- DocCollection coll = collectionStates.get(collection);
+ DocCollection coll = getCollectionOrNull(collection);
if (coll == null) return null;
return coll.getActiveSlicesMap();
}
public Collection<Slice> getSlices(String collection) {
- DocCollection coll = collectionStates.get(collection);
+ DocCollection coll = getCollectionOrNull(collection);
if (coll == null) return null;
return coll.getSlices();
}
public Collection<Slice> getActiveSlices(String collection) {
- DocCollection coll = collectionStates.get(collection);
+ DocCollection coll = getCollectionOrNull(collection);
if (coll == null) return null;
return coll.getActiveSlices();
}
- public DocCollection getCollectionOrNull(String collection) {
- return collectionStates.get(collection);
- }
/**
* Get the named DocCollection object, or throw an exception if it doesn't exist.
*/
public DocCollection getCollection(String collection) {
- DocCollection coll = collectionStates.get(collection);
- if (coll == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
- }
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection : " + collection);
return coll;
}
+
+ public DocCollection getCollectionOrNull(String coll) {
+ return collectionStates.get(coll);
+ }
+
/**
* Get collection names.
*/
public Set<String> getCollections() {
- return Collections.unmodifiableSet(collectionStates.keySet());
+ return collectionStates.keySet();
}
/**
* @return Map<collectionName, Map<sliceName,Slice>>
*/
- @Deprecated
public Map<String, DocCollection> getCollectionStates() {
return Collections.unmodifiableMap(collectionStates);
}
@@ -200,8 +190,17 @@ public class ClusterState implements JSO
}
public String getShardId(String nodeName, String coreName) {
- // System.out.println("###### getShardId(" + baseUrl + "," + coreName + ") in " + collectionStates);
- for (DocCollection coll : collectionStates.values()) {
+ return getShardId(null, nodeName, coreName);
+ }
+
+ public String getShardId(String collectionName, String nodeName, String coreName) {
+ Collection<DocCollection> states = collectionStates.values();
+ if (collectionName != null) {
+ DocCollection c = getCollectionOrNull(collectionName);
+ if (c != null) states = Collections.singletonList(c);
+ }
+
+ for (DocCollection coll : states) {
for (Slice slice : coll.getSlices()) {
for (Replica replica : slice.getReplicas()) {
// TODO: for really large clusters, we could 'index' on this
@@ -215,6 +214,25 @@ public class ClusterState implements JSO
}
return null;
}
+
+ public String getShardIdByCoreNodeName(String collectionName, String coreNodeName) {
+ Collection<DocCollection> states = collectionStates.values();
+ if (collectionName != null) {
+ DocCollection c = getCollectionOrNull(collectionName);
+ if (c != null) states = Collections.singletonList(c);
+ }
+
+ for (DocCollection coll : states) {
+ for (Slice slice : coll.getSlices()) {
+ for (Replica replica : slice.getReplicas()) {
+ if (coreNodeName.equals(replica.getName())) {
+ return slice.getName();
+ }
+ }
+ }
+ }
+ return null;
+ }
/**
* Check if node is alive.
@@ -238,7 +256,7 @@ public class ClusterState implements JSO
Stat stat = new Stat();
byte[] state = zkClient.getData(ZkStateReader.CLUSTER_STATE,
null, stat, true);
- return load(stat.getVersion(), state, liveNodes, stateReader);
+ return load(stat.getVersion(), state, liveNodes);
}
@@ -252,23 +270,24 @@ public class ClusterState implements JSO
* @param liveNodes list of live nodes
* @return the ClusterState
*/
- public static ClusterState load(Integer version, byte[] bytes, Set<String> liveNodes, ZkStateReader stateReader) {
+ public static ClusterState load(Integer version, byte[] bytes, Set<String> liveNodes) {
// System.out.println("######## ClusterState.load:" + (bytes==null ? null : new String(bytes)));
if (bytes == null || bytes.length == 0) {
- return new ClusterState(version, liveNodes, Collections.<String, DocCollection>emptyMap(),stateReader);
+ return new ClusterState(version, liveNodes, Collections.<String, DocCollection>emptyMap());
}
Map<String, Object> stateMap = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
Map<String,DocCollection> collections = new LinkedHashMap<>(stateMap.size());
for (Entry<String, Object> entry : stateMap.entrySet()) {
String collectionName = entry.getKey();
- DocCollection coll = collectionFromObjects(collectionName, (Map<String,Object>)entry.getValue());
+ DocCollection coll = collectionFromObjects(collectionName, (Map<String,Object>)entry.getValue(), version);
collections.put(collectionName, coll);
}
// System.out.println("######## ClusterState.load result:" + collections);
- return new ClusterState(version, liveNodes, collections);
+ return new ClusterState( version, liveNodes, collections);
}
-
+
+
public static Aliases load(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return new Aliases();
@@ -278,7 +297,7 @@ public class ClusterState implements JSO
return new Aliases(aliasMap);
}
- private static DocCollection collectionFromObjects(String name, Map<String,Object> objs) {
+ private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version) {
Map<String,Object> props;
Map<String,Slice> slices;
@@ -305,7 +324,7 @@ public class ClusterState implements JSO
router = DocRouter.getDocRouter((String) routerProps.get("name"));
}
- return new DocCollection(name, slices, props, router);
+ return new DocCollection(name, slices, props, router, version);
}
private static Map<String,Slice> makeSlices(Map<String,Object> genericSlices) {
@@ -362,6 +381,8 @@ public class ClusterState implements JSO
return true;
}
+
+
/**
* Internal API used only by ZkStateReader
*/
@@ -369,4 +390,9 @@ public class ClusterState implements JSO
this.liveNodes = liveNodes;
}
+ public DocCollection getCommonCollection(String name){
+ return collectionStates.get(name);
+
+ }
+
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java Mon Aug 18 16:26:56 2014
@@ -17,39 +17,69 @@ package org.apache.solr.common.cloud;
* limitations under the License.
*/
-import org.noggit.JSONUtil;
-import org.noggit.JSONWriter;
-
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.noggit.JSONUtil;
+import org.noggit.JSONWriter;
+
/**
* Models a Collection in zookeeper (but that Java name is obviously taken, hence "DocCollection")
*/
public class DocCollection extends ZkNodeProps {
public static final String DOC_ROUTER = "router";
public static final String SHARDS = "shards";
+ private int version;
private final String name;
private final Map<String, Slice> slices;
private final Map<String, Slice> activeSlices;
private final DocRouter router;
+ private final Integer replicationFactor;
+ private final Integer maxShardsPerNode;
+ private final boolean autoAddReplicas;
+
+
+ public DocCollection(String name, Map<String, Slice> slices, Map<String, Object> props, DocRouter router) {
+ this(name, slices, props, router, -1);
+ }
+
/**
* @param name The name of the collection
* @param slices The logical shards of the collection. This is used directly and a copy is not made.
* @param props The properties of the slice. This is used directly and a copy is not made.
*/
- public DocCollection(String name, Map<String, Slice> slices, Map<String, Object> props, DocRouter router) {
- super( props==null ? props = new HashMap<>() : props);
+ public DocCollection(String name, Map<String, Slice> slices, Map<String, Object> props, DocRouter router, int zkVersion) {
+ super( props==null ? props = new HashMap<String,Object>() : props);
+ this.version = zkVersion;
this.name = name;
this.slices = slices;
this.activeSlices = new HashMap<>();
+ Object replicationFactorObject = (Object) props.get(ZkStateReader.REPLICATION_FACTOR);
+ if (replicationFactorObject != null) {
+ this.replicationFactor = Integer.parseInt(replicationFactorObject.toString());
+ } else {
+ this.replicationFactor = null;
+ }
+ Object maxShardsPerNodeObject = (Object) props.get(ZkStateReader.MAX_SHARDS_PER_NODE);
+ if (maxShardsPerNodeObject != null) {
+ this.maxShardsPerNode = Integer.parseInt(maxShardsPerNodeObject.toString());
+ } else {
+ this.maxShardsPerNode = null;
+ }
+ Object autoAddReplicasObject = (Object) props.get(ZkStateReader.AUTO_ADD_REPLICAS);
+ if (autoAddReplicasObject != null) {
+ this.autoAddReplicas = Boolean.parseBoolean(autoAddReplicasObject.toString());
+ } else {
+ this.autoAddReplicas = false;
+ }
Iterator<Map.Entry<String, Slice>> iter = slices.entrySet().iterator();
@@ -104,6 +134,30 @@ public class DocCollection extends ZkNod
return activeSlices;
}
+ public int getVersion(){
+ return version;
+ }
+
+ /**
+ * @return replication factor for this collection or null if no
+ * replication factor exists.
+ */
+ public Integer getReplicationFactor() {
+ return replicationFactor;
+ }
+
+ public boolean getAutoAddReplicas() {
+ return autoAddReplicas;
+ }
+
+ public int getMaxShardsPerNode() {
+ if (maxShardsPerNode == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, ZkStateReader.MAX_SHARDS_PER_NODE + " is not in the cluster state.");
+ }
+ return maxShardsPerNode;
+ }
+
+
public DocRouter getRouter() {
return router;
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java Mon Aug 18 16:26:56 2014
@@ -17,11 +17,11 @@ package org.apache.solr.common.cloud;
* the License.
*/
+import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
-import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -59,7 +59,7 @@ import org.slf4j.LoggerFactory;
* ZooKeeper. This class handles synchronous connects and reconnections.
*
*/
-public class SolrZkClient {
+public class SolrZkClient implements Closeable {
// These should *only* be used for debugging or monitoring purposes
public static final AtomicLong numOpens = new AtomicLong();
public static final AtomicLong numCloses = new AtomicLong();
@@ -87,6 +87,11 @@ public class SolrZkClient {
return zkClientTimeout;
}
+ // expert: for tests
+ public SolrZkClient() {
+
+ }
+
public SolrZkClient(String zkServerAddress, int zkClientTimeout) {
this(zkServerAddress, zkClientTimeout, new DefaultConnectionStrategy(), null);
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Mon Aug 18 16:26:56 2014
@@ -10,7 +10,7 @@ package org.apache.solr.common.cloud;
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ * Unless required byOCP applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -32,6 +32,7 @@ import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Closeable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
@@ -49,7 +50,7 @@ import java.util.concurrent.ThreadFactor
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-public class ZkStateReader {
+public class ZkStateReader implements Closeable {
private static Logger log = LoggerFactory.getLogger(ZkStateReader.class);
public static final String BASE_URL_PROP = "base_url";
@@ -73,6 +74,9 @@ public class ZkStateReader {
public static final String CLUSTER_STATE = "/clusterstate.json";
public static final String CLUSTER_PROPS = "/clusterprops.json";
+ public static final String REPLICATION_FACTOR = "replicationFactor";
+ public static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
+ public static final String AUTO_ADD_REPLICAS = "autoAddReplicas";
public static final String ROLES = "/roles.json";
@@ -88,8 +92,8 @@ public class ZkStateReader {
public static final String LEGACY_CLOUD = "legacyCloud";
public static final String URL_SCHEME = "urlScheme";
-
- private volatile ClusterState clusterState;
+
+ protected volatile ClusterState clusterState;
private static final long SOLRCLOUD_UPDATE_DELAY = Long.parseLong(System.getProperty("solrcloud.update.delay", "5000"));
@@ -98,7 +102,6 @@ public class ZkStateReader {
public static final String SHARD_LEADERS_ZKNODE = "leaders";
-
//
// convenience methods... should these go somewhere else?
@@ -128,10 +131,10 @@ public class ZkStateReader {
throw new RuntimeException(e); // should never happen w/o using real IO
}
}
-
+
/**
* Returns config set name for collection.
- *
+ *
* @param collection to return config set name for
*/
public String readConfigName(String collection) {
@@ -248,6 +251,23 @@ public class ZkStateReader {
public Aliases getAliases() {
return aliases;
}
+
+ /*public Boolean checkValid(String coll, int version){
+ DocCollection collection = clusterState.getCollectionOrNull(coll);
+ if(collection ==null) return null;
+ if(collection.getVersion() < version){
+ log.info("server older than client {}<{}",collection.getVersion(),version);
+ DocCollection nu = getExternCollectionFresh(this, coll);
+ if(nu.getVersion()> collection.getVersion()){
+ updateExternCollection(nu);
+ collection = nu;
+ }
+ }
+ if(collection.getVersion() == version) return Boolean.TRUE;
+ log.info("wrong version from client {}!={} ",version, collection.getVersion());
+ return Boolean.FALSE;
+
+ }*/
public synchronized void createClusterStateWatchersAndUpdate() throws KeeperException,
InterruptedException {
@@ -280,9 +300,14 @@ public class ZkStateReader {
byte[] data = zkClient.getData(CLUSTER_STATE, thisWatch, stat ,
true);
Set<String> ln = ZkStateReader.this.clusterState.getLiveNodes();
- ClusterState clusterState = ClusterState.load(stat.getVersion(), data, ln,ZkStateReader.this);
+ ClusterState clusterState = ClusterState.load(stat.getVersion(), data, ln);
// update volatile
ZkStateReader.this.clusterState = clusterState;
+
+// HashSet<String> all = new HashSet<>(colls);;
+// all.addAll(clusterState.getAllInternalCollections());
+// all.remove(null);
+
}
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.SESSIONEXPIRED
@@ -322,14 +347,13 @@ public class ZkStateReader {
synchronized (ZkStateReader.this.getUpdateLock()) {
List<String> liveNodes = zkClient.getChildren(
LIVE_NODES_ZKNODE, this, true);
- log.info("Updating live nodes... ({})", liveNodes.size());
+ log.debug("Updating live nodes... ({})", liveNodes.size());
Set<String> liveNodesSet = new HashSet<>();
liveNodesSet.addAll(liveNodes);
- ClusterState clusterState = new ClusterState(
- ZkStateReader.this.clusterState.getZkClusterStateVersion(),
- liveNodesSet, ZkStateReader.this.clusterState
- .getCollectionStates());
- ZkStateReader.this.clusterState = clusterState;
+
+ ClusterState clusterState = ZkStateReader.this.clusterState;
+
+ clusterState.setLiveNodes(liveNodesSet);
}
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.SESSIONEXPIRED
@@ -354,7 +378,7 @@ public class ZkStateReader {
liveNodeSet.addAll(liveNodes);
ClusterState clusterState = ClusterState.load(zkClient, liveNodeSet, ZkStateReader.this);
this.clusterState = clusterState;
-
+
zkClient.exists(ALIASES,
new Watcher() {
@@ -400,8 +424,8 @@ public class ZkStateReader {
}
updateAliases();
}
-
-
+
+
// load and publish a new CollectionInfo
private synchronized void updateClusterState(boolean immediate,
final boolean onlyLiveNodes) throws KeeperException,
@@ -417,32 +441,29 @@ public class ZkStateReader {
liveNodesSet.addAll(liveNodes);
if (!onlyLiveNodes) {
- log.info("Updating cloud state from ZooKeeper... ");
+ log.debug("Updating cloud state from ZooKeeper... ");
clusterState = ClusterState.load(zkClient, liveNodesSet,this);
} else {
- log.info("Updating live nodes from ZooKeeper... ({})", liveNodesSet.size());
+ log.debug("Updating live nodes from ZooKeeper... ({})", liveNodesSet.size());
clusterState = this.clusterState;
clusterState.setLiveNodes(liveNodesSet);
- /*clusterState = new ClusterState(
- ZkStateReader.this.clusterState.getZkClusterStateVersion(), liveNodesSet,
- ZkStateReader.this.clusterState.getCollectionStates());*/
}
this.clusterState = clusterState;
}
} else {
if (clusterStateUpdateScheduled) {
- log.info("Cloud state update for ZooKeeper already scheduled");
+ log.debug("Cloud state update for ZooKeeper already scheduled");
return;
}
- log.info("Scheduling cloud state update from ZooKeeper...");
+ log.debug("Scheduling cloud state update from ZooKeeper...");
clusterStateUpdateScheduled = true;
updateCloudExecutor.schedule(new Runnable() {
@Override
public void run() {
- log.info("Updating cluster state from ZooKeeper...");
+ log.debug("Updating cluster state from ZooKeeper...");
synchronized (getUpdateLock()) {
clusterStateUpdateScheduled = false;
ClusterState clusterState;
@@ -453,12 +474,14 @@ public class ZkStateReader {
liveNodesSet.addAll(liveNodes);
if (!onlyLiveNodes) {
- log.info("Updating cloud state from ZooKeeper... ");
+ log.debug("Updating cloud state from ZooKeeper... ");
clusterState = ClusterState.load(zkClient, liveNodesSet,ZkStateReader.this);
} else {
- log.info("Updating live nodes from ZooKeeper... ");
- clusterState = new ClusterState(ZkStateReader.this.clusterState.getZkClusterStateVersion(), liveNodesSet, ZkStateReader.this.clusterState.getCollectionStates());
+ log.debug("Updating live nodes from ZooKeeper... ");
+ clusterState = ZkStateReader.this.clusterState;
+ clusterState.setLiveNodes(liveNodesSet);
+
}
ZkStateReader.this.clusterState = clusterState;
@@ -485,7 +508,7 @@ public class ZkStateReader {
}
}, SOLRCLOUD_UPDATE_DELAY, TimeUnit.MILLISECONDS);
}
-
+
}
/**
@@ -610,9 +633,6 @@ public class ZkStateReader {
public SolrZkClient getZkClient() {
return zkClient;
}
- public Set<String> getAllCollections(){
- return clusterState.getCollections();
- }
public void updateAliases() throws KeeperException, InterruptedException {
byte[] data = zkClient.getData(ALIASES, null, null, true);
@@ -624,7 +644,7 @@ public class ZkStateReader {
public Map getClusterProps(){
Map result = null;
try {
- if(getZkClient().exists(ZkStateReader.CLUSTER_PROPS,true)){
+ if(getZkClient().exists(ZkStateReader.CLUSTER_PROPS, true)){
result = (Map) ZkStateReader.fromJSON(getZkClient().getData(ZkStateReader.CLUSTER_PROPS, null, new Stat(), true)) ;
} else {
result= new LinkedHashMap();
@@ -658,5 +678,5 @@ public class ZkStateReader {
throw new IllegalStateException("JVM Does not seem to support UTF-8", e);
}
}
-
+
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTestBase.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTestBase.java Mon Aug 18 16:26:56 2014
@@ -17,7 +17,6 @@
package org.apache.solr.client.solrj;
-
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.BeforeClass;
@@ -26,28 +25,33 @@ import org.junit.BeforeClass;
*
* This lets us try various SolrServer implementations with the same tests.
*
- *
- * @since solr 1.3
*/
-abstract public class SolrExampleTestBase extends AbstractSolrTestCase
-{
+abstract public class SolrExampleTestBase extends AbstractSolrTestCase {
@Override
- public String getSolrHome() { return "../../../example/solr/"; }
-
+ public String getSolrHome() {
+ return "../../../example/solr/";
+ }
+
@BeforeClass
public static void beforeClass() throws Exception {
-
+
}
-
+
@Override
- public void setUp() throws Exception
- {
+ public void setUp() throws Exception {
ignoreException("maxWarmingSearchers");
super.setUp();
// this sets the property for jetty starting SolrDispatchFilter
- System.setProperty( "solr.solr.home", this.getSolrHome() );
- System.setProperty( "solr.data.dir", this.initCoreDataDir.getCanonicalPath() );
+ System.setProperty("solr.solr.home", this.getSolrHome());
+ System.setProperty("solr.data.dir", this.initCoreDataDir.getCanonicalPath());
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ System.clearProperty("solr.solr.home");
+ System.clearProperty("solr.data.dir");
+ super.tearDown();
}
/**
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java Mon Aug 18 16:26:56 2014
@@ -313,7 +313,7 @@ public class TestLBHttpSolrServer extend
public void startJetty() throws Exception {
jetty = new JettySolrRunner(getHomeDir(), "/solr", port, "bad_solrconfig.xml", null, true, null, sslConfig);
- System.setProperty("solr.data.dir", getDataDir());
+ jetty.setDataDir(getDataDir());
jetty.start();
int newPort = jetty.getLocalPort();
if (port != 0 && newPort != port) {
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java Mon Aug 18 16:26:56 2014
@@ -88,6 +88,7 @@ public class JettyWebappTest extends Sol
server.stop();
} catch( Exception ex ) {}
System.clearProperty("tests.shardhandler.randomSeed");
+ System.clearProperty("solr.data.dir");
super.tearDown();
}
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java Mon Aug 18 16:26:56 2014
@@ -37,7 +37,6 @@ import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServer;
@@ -288,8 +287,13 @@ public abstract class BaseDistributedSea
super.tearDown();
}
+ protected JettySolrRunner createControlJetty() throws Exception {
+ JettySolrRunner jetty = createJetty(new File(getSolrHome()), testDir + "/control/data", null, getSolrConfigFile(), getSchemaFile());
+ return jetty;
+ }
+
protected void createServers(int numShards) throws Exception {
- controlJetty = createJetty(new File(getSolrHome()), testDir + "/control/data", null, getSolrConfigFile(), getSchemaFile());
+ controlJetty = createControlJetty();
controlClient = createNewSolrServer(controlJetty.getLocalPort());
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java Mon Aug 18 16:26:56 2014
@@ -55,13 +55,15 @@ abstract public class SolrJettyTestBase
ignoreException("maxWarmingSearchers");
- // this sets the property for jetty starting SolrDispatchFilter
- System.setProperty( "solr.data.dir", createTempDir().getCanonicalPath() );
-
context = context==null ? "/solr" : context;
SolrJettyTestBase.context = context;
jetty = new JettySolrRunner(solrHome, context, 0, configFile, schemaFile, stopAtShutdown, extraServlets, sslConfig);
+ // this sets the property for jetty starting SolrDispatchFilter
+ if (System.getProperty("solr.data.dir") == null && System.getProperty("solr.hdfs.home") == null) {
+ jetty.setDataDir(createTempDir().getCanonicalPath());
+ }
+
jetty.start();
port = jetty.getLocalPort();
log.info("Jetty Assigned Port#" + port);
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Mon Aug 18 16:26:56 2014
@@ -18,10 +18,10 @@ package org.apache.solr.cloud;
*/
import static org.apache.solr.cloud.OverseerCollectionProcessor.CREATE_NODE_SET;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import java.io.File;
import java.io.IOException;
@@ -279,33 +279,44 @@ public abstract class AbstractFullDistri
protected void createServers(int numServers) throws Exception {
System.setProperty("collection", "control_collection");
- String numShards = System.getProperty(ZkStateReader.NUM_SHARDS_PROP);
// we want hashes by default for the control, so set to 1 shard as opposed to leaving unset
- // System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ String oldNumShards = System.getProperty(ZkStateReader.NUM_SHARDS_PROP);
System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "1");
+
+ try {
+
+ File controlJettyDir = createTempDir();
+ setupJettySolrHome(controlJettyDir);
+
+ controlJetty = createJetty(controlJettyDir, useJettyDataDir ? getDataDir(testDir
+ + "/control/data") : null); // don't pass shard name... let it default to
+ // "shard1"
- File controlJettyDir = createTempDir();
- setupJettySolrHome(controlJettyDir);
-
- controlJetty = createJetty(controlJettyDir, testDir + "/control/data"); // don't pass shard name... let it default to "shard1"
- System.clearProperty("collection");
- if(numShards != null) {
- System.setProperty(ZkStateReader.NUM_SHARDS_PROP, numShards);
- } else {
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
- }
- controlClient = createNewSolrServer(controlJetty.getLocalPort());
- if (sliceCount <= 0) {
- // for now, just create the cloud client for the control if we don't create the normal cloud client.
- // this can change if more tests need it.
- controlClientCloud = createCloudClient("control_collection");
- controlClientCloud.connect();
- waitForCollection(controlClientCloud.getZkStateReader(), "control_collection", 0);
- // NOTE: we are skipping creation of the chaos monkey by returning here
- cloudClient = controlClientCloud; // temporary - some code needs/uses cloudClient
- return;
+ controlClient = createNewSolrServer(controlJetty.getLocalPort());
+
+ if (sliceCount <= 0) {
+ // for now, just create the cloud client for the control if we don't
+ // create the normal cloud client.
+ // this can change if more tests need it.
+ controlClientCloud = createCloudClient("control_collection");
+ controlClientCloud.connect();
+ waitForCollection(controlClientCloud.getZkStateReader(),
+ "control_collection", 0);
+ // NOTE: we are skipping creation of the chaos monkey by returning here
+ cloudClient = controlClientCloud; // temporary - some code needs/uses
+ // cloudClient
+ return;
+ }
+
+ } finally {
+ System.clearProperty("collection");
+ if (oldNumShards != null) {
+ System.setProperty(ZkStateReader.NUM_SHARDS_PROP, oldNumShards);
+ } else {
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ }
}
@@ -1481,8 +1492,8 @@ public abstract class AbstractFullDistri
numShards = StrUtils.splitSmart(shardNames,',').size();
}
Integer replicationFactor = (Integer) collectionProps.get(REPLICATION_FACTOR);
- if(numShards==null){
- numShards = (Integer) OverseerCollectionProcessor.COLL_PROPS.get(REPLICATION_FACTOR);
+ if(replicationFactor==null){
+ replicationFactor = (Integer) OverseerCollectionProcessor.COLL_PROPS.get(REPLICATION_FACTOR);
}
if (confSetName != null) {
@@ -1712,6 +1723,25 @@ public abstract class AbstractFullDistri
fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
}
+ protected void checkForMissingCollection(String collectionName)
+ throws Exception {
+ // check for a collection - we poll the state
+ long timeoutAt = System.currentTimeMillis() + 45000;
+ boolean found = true;
+ while (System.currentTimeMillis() < timeoutAt) {
+ getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+ if (!clusterState.hasCollection(collectionName)) {
+ found = false;
+ break;
+ }
+ Thread.sleep(100);
+ }
+ if (found) {
+ fail("Found collection that should be gone " + collectionName);
+ }
+ }
+
protected NamedList<Object> invokeCollectionApi(String... args) throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
SolrRequest request = new QueryRequest(params);
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java Mon Aug 18 16:26:56 2014
@@ -35,6 +35,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.Filter;
+
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
@@ -230,8 +231,15 @@ public class ChaosMonkey {
}
}
- public static void kill(CloudJettyRunner cjetty) throws Exception {
- FilterHolder filterHolder = cjetty.jetty.getDispatchFilter();
+
+ public static void kill(List<JettySolrRunner> jettys) throws Exception {
+ for (JettySolrRunner jetty : jettys) {
+ kill(jetty);
+ }
+ }
+
+ public static void kill(JettySolrRunner jetty) throws Exception {
+ FilterHolder filterHolder = jetty.getDispatchFilter();
if (filterHolder != null) {
Filter filter = filterHolder.getFilter();
if (filter != null) {
@@ -244,9 +252,8 @@ public class ChaosMonkey {
}
}
- IpTables.blockPort(cjetty.jetty.getLocalPort());
+ IpTables.blockPort(jetty.getLocalPort());
- JettySolrRunner jetty = cjetty.jetty;
monkeyLog("kill shard! " + jetty.getLocalPort());
jetty.stop();
@@ -258,6 +265,10 @@ public class ChaosMonkey {
}
}
+ public static void kill(CloudJettyRunner cjetty) throws Exception {
+ kill(cjetty.jetty);
+ }
+
public void stopShard(String slice) throws Exception {
List<CloudJettyRunner> jetties = shardToJetty.get(slice);
for (CloudJettyRunner jetty : jetties) {
@@ -556,10 +567,22 @@ public class ChaosMonkey {
return starts.get();
}
+ public static void stop(List<JettySolrRunner> jettys) throws Exception {
+ for (JettySolrRunner jetty : jettys) {
+ stop(jetty);
+ }
+ }
+
public static void stop(JettySolrRunner jetty) throws Exception {
stopJettySolrRunner(jetty);
}
+ public static void start(List<JettySolrRunner> jettys) throws Exception {
+ for (JettySolrRunner jetty : jettys) {
+ start(jetty);
+ }
+ }
+
public static boolean start(JettySolrRunner jetty) throws Exception {
IpTables.unblockPort(jetty.getLocalPort());
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/CloudInspectUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/CloudInspectUtil.java?rev=1618655&r1=1618654&r2=1618655&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/CloudInspectUtil.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/CloudInspectUtil.java Mon Aug 18 16:26:56 2014
@@ -228,4 +228,5 @@ public class CloudInspectUtil {
return true;
}
+
}