You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/29 12:14:56 UTC
svn commit: r1562403 [7/7] - in /lucene/dev/branches/lucene5376: ./
dev-tools/ dev-tools/idea/lucene/suggest/
dev-tools/idea/solr/core/src/java/ lucene/ lucene/analysis/
lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis...
Modified: lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java (original)
+++ lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java Wed Jan 29 11:14:53 2014
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.*;
+import java.util.Map.Entry;
import java.nio.ByteBuffer;
/**
@@ -65,6 +66,7 @@ public class JavaBinCodec {
SOLRINPUTDOC = 16,
SOLRINPUTDOC_CHILDS = 17,
ENUM_FIELD_VALUE = 18,
+ MAP_ENTRY = 19,
// types that combine tag + length (or other info) in a single byte
TAG_AND_LEN = (byte) (1 << 5),
STR = (byte) (1 << 5),
@@ -227,6 +229,8 @@ public class JavaBinCodec {
return readSolrInputDocument(dis);
case ENUM_FIELD_VALUE:
return readEnumFieldValue(dis);
+ case MAP_ENTRY:
+ return readMapEntry(dis);
}
throw new RuntimeException("Unknown type " + tagByte);
@@ -286,6 +290,10 @@ public class JavaBinCodec {
writeEnumFieldValue((EnumFieldValue) val);
return true;
}
+ if (val instanceof Map.Entry) {
+ writeMapEntry((Map.Entry)val);
+ return true;
+ }
return false;
}
@@ -480,6 +488,12 @@ public class JavaBinCodec {
writeInt(enumFieldValue.toInt());
writeStr(enumFieldValue.toString());
}
+
+ public void writeMapEntry(Entry<Object,Object> val) throws IOException {
+ writeTag(MAP_ENTRY);
+ writeVal(val.getKey());
+ writeVal(val.getValue());
+ }
/**
* read {@link EnumFieldValue} (int+string) from input stream
@@ -491,6 +505,33 @@ public class JavaBinCodec {
String stringValue = (String) readVal(dis);
return new EnumFieldValue(intValue, stringValue);
}
+
+
+ public Map.Entry<Object,Object> readMapEntry(DataInputInputStream dis) throws IOException {
+ final Object key = readVal(dis);
+ final Object value = readVal(dis);
+ return new Map.Entry<Object,Object>() {
+
+ @Override
+ public Object getKey() {
+ return key;
+ }
+
+ @Override
+ public Object getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return "MapEntry[" + key.toString() + ":" + value.toString() + "]";
+ }
+
+ @Override
+ public Object setValue(Object value) {
+ throw new UnsupportedOperationException();
+ }};
+ }
/**
* write the string as tag+length, with length being the number of UTF-8 bytes
Modified: lucene/dev/branches/lucene5376/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java (original)
+++ lucene/dev/branches/lucene5376/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java Wed Jan 29 11:14:53 2014
@@ -85,7 +85,7 @@ public class TestUpdateRequestCodec exte
final List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
JavaBinUpdateRequestCodec.StreamingUpdateHandler handler = new JavaBinUpdateRequestCodec.StreamingUpdateHandler() {
@Override
- public void update(SolrInputDocument document, UpdateRequest req) {
+ public void update(SolrInputDocument document, UpdateRequest req, Integer commitWithin, Boolean overwrite) {
Assert.assertNotNull(req.getParams());
docs.add(document);
}
@@ -136,7 +136,7 @@ public class TestUpdateRequestCodec exte
final List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
JavaBinUpdateRequestCodec.StreamingUpdateHandler handler = new JavaBinUpdateRequestCodec.StreamingUpdateHandler() {
@Override
- public void update(SolrInputDocument document, UpdateRequest req) {
+ public void update(SolrInputDocument document, UpdateRequest req, Integer commitWithin, Boolean overwrite) {
Assert.assertNotNull(req.getParams());
docs.add(document);
}
Modified: lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original)
+++ lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Wed Jan 29 11:14:53 2014
@@ -349,6 +349,7 @@ public abstract class AbstractFullDistri
getClass().getName() + "-jetty" + cnt + "-" + System.currentTimeMillis());
jettyDir.mkdirs();
setupJettySolrHome(jettyDir);
+ log.info("create jetty " + i);
JettySolrRunner j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty"
+ cnt) : null, null, "solrconfig.xml", null);
jettys.add(j);
@@ -502,8 +503,11 @@ public abstract class AbstractFullDistri
return solrHome;
}
- protected void updateMappingsFromZk(List<JettySolrRunner> jettys,
- List<SolrServer> clients) throws Exception {
+ protected void updateMappingsFromZk(List<JettySolrRunner> jettys, List<SolrServer> clients) throws Exception {
+ updateMappingsFromZk(jettys, clients, false);
+ }
+
+ protected void updateMappingsFromZk(List<JettySolrRunner> jettys, List<SolrServer> clients, boolean allowOverSharding) throws Exception {
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
zkStateReader.updateClusterState(true);
cloudJettys.clear();
@@ -522,7 +526,7 @@ public abstract class AbstractFullDistri
int port = new URI(((HttpSolrServer) client).getBaseURL())
.getPort();
- if (replica.getNodeName().contains(":" + port + "_")) {
+ if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
CloudSolrServerClient csc = new CloudSolrServerClient();
csc.solrClient = client;
csc.port = port;
@@ -579,9 +583,13 @@ public abstract class AbstractFullDistri
for (Slice slice : coll.getSlices()) {
// check that things look right
List<CloudJettyRunner> jetties = shardToJetty.get(slice.getName());
- assertNotNull("Test setup problem: We found no jetties for shard: " + slice.getName()
- + " just:" + shardToJetty.keySet(), jetties);
- assertEquals("slice:" + slice.getName(), slice.getReplicas().size(), jetties.size());
+ if (!allowOverSharding) {
+ assertNotNull("Test setup problem: We found no jetties for shard: "
+ + slice.getName() + " just:" + shardToJetty.keySet(), jetties);
+
+ assertEquals("slice:" + slice.getName(), slice.getReplicas().size(),
+ jetties.size());
+ }
}
}
@@ -1137,7 +1145,7 @@ public abstract class AbstractFullDistri
protected void checkShardConsistency(boolean checkVsControl, boolean verbose)
throws Exception {
- updateMappingsFromZk(jettys, clients);
+ updateMappingsFromZk(jettys, clients, true);
Set<String> theShards = shardToJetty.keySet();
String failMessage = null;
@@ -1202,45 +1210,53 @@ public abstract class AbstractFullDistri
String msg = "document count mismatch. control=" + controlDocs + " sum(shards)="+ cnt + " cloudClient="+cloudClientDocs;
log.error(msg);
- // re-execute the query getting ids
- q = params("q","*:*","rows","100000", "fl","id", "tests","checkShardConsistency(vsControl)/getIds"); // add a tag to aid in debugging via logs
- controlDocList = controlClient.query(q).getResults();
- if (controlDocs != controlDocList.getNumFound()) {
- log.error("Something changed! control now " + controlDocList.getNumFound());
- };
-
- cloudDocList = cloudClient.query(q).getResults();
- if (cloudClientDocs != cloudDocList.getNumFound()) {
- log.error("Something changed! cloudClient now " + cloudDocList.getNumFound());
- };
-
- Set<Map> differences = showDiff(controlDocList, cloudDocList,"controlDocList","cloudDocList");
-
- // get versions for the mismatched ids
- boolean foundId = false;
- StringBuilder ids = new StringBuilder("id:(");
- for (Map doc : differences) {
- ids.append(" "+doc.get("id"));
- foundId = true;
- }
- ids.append(")");
-
- if (foundId) {
- // get versions for those ids that don't match
- q = params("q", ids.toString(), "rows", "100000", "fl", "id,_version_",
- "sort", "id asc", "tests",
- "checkShardConsistency(vsControl)/getVers"); // add a tag to aid in
- // debugging via logs
-
- SolrDocumentList a = controlClient.query(q).getResults();
- SolrDocumentList b = cloudClient.query(q).getResults();
-
- log.error("controlClient :" + a + "\n\tcloudClient :" + b);
- }
+ compareResults(controlDocs, cloudClientDocs);
fail(msg);
}
}
+
+ protected void compareResults(long controlDocs, long cloudClientDocs)
+ throws SolrServerException {
+ SolrParams q;
+ SolrDocumentList controlDocList;
+ SolrDocumentList cloudDocList;
+ // re-execute the query getting ids
+ q = params("q","*:*","rows","100000", "fl","id", "tests","checkShardConsistency(vsControl)/getIds"); // add a tag to aid in debugging via logs
+ controlDocList = controlClient.query(q).getResults();
+ if (controlDocs != controlDocList.getNumFound()) {
+ log.error("Something changed! control now " + controlDocList.getNumFound());
+ };
+
+ cloudDocList = cloudClient.query(q).getResults();
+ if (cloudClientDocs != cloudDocList.getNumFound()) {
+ log.error("Something changed! cloudClient now " + cloudDocList.getNumFound());
+ };
+
+ Set<Map> differences = showDiff(controlDocList, cloudDocList,"controlDocList","cloudDocList");
+
+ // get versions for the mismatched ids
+ boolean foundId = false;
+ StringBuilder ids = new StringBuilder("id:(");
+ for (Map doc : differences) {
+ ids.append(" "+doc.get("id"));
+ foundId = true;
+ }
+ ids.append(")");
+
+ if (foundId) {
+ // get versions for those ids that don't match
+ q = params("q", ids.toString(), "rows", "100000", "fl", "id,_version_",
+ "sort", "id asc", "tests",
+ "checkShardConsistency(vsControl)/getVers"); // add a tag to aid in
+ // debugging via logs
+
+ SolrDocumentList a = controlClient.query(q).getResults();
+ SolrDocumentList b = cloudClient.query(q).getResults();
+
+ log.error("controlClient :" + a + "\n\tcloudClient :" + b);
+ }
+ }
protected SolrServer getClient(String nodeName) {
for (CloudJettyRunner cjetty : cloudJettys) {
Modified: lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java?rev=1562403&r1=1562402&r2=1562403&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java (original)
+++ lucene/dev/branches/lucene5376/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java Wed Jan 29 11:14:53 2014
@@ -79,6 +79,8 @@ public class ChaosMonkey {
private boolean aggressivelyKillLeaders;
private Map<String,CloudJettyRunner> shardToLeaderJetty;
private volatile long startTime;
+
+ private List<CloudJettyRunner> deadPool = new ArrayList<CloudJettyRunner>();
private Thread monkeyThread;
@@ -319,58 +321,31 @@ public class ChaosMonkey {
public CloudJettyRunner getRandomJetty(String slice, boolean aggressivelyKillLeaders) throws KeeperException, InterruptedException {
-
- int numRunning = 0;
- int numRecovering = 0;
int numActive = 0;
- for (CloudJettyRunner cloudJetty : shardToJetty.get(slice)) {
- boolean running = true;
-
- // get latest cloud state
- zkStateReader.updateClusterState(true);
-
- Slice theShards = zkStateReader.getClusterState().getSlicesMap(collection)
- .get(slice);
-
- ZkNodeProps props = theShards.getReplicasMap().get(cloudJetty.coreNodeName);
- if (props == null) {
- throw new RuntimeException("shard name " + cloudJetty.coreNodeName + " not found in " + theShards.getReplicasMap().keySet());
- }
-
- String state = props.getStr(ZkStateReader.STATE_PROP);
- String nodeName = props.getStr(ZkStateReader.NODE_NAME_PROP);
-
-
- if (!cloudJetty.jetty.isRunning()
- || !state.equals(ZkStateReader.ACTIVE)
- || !zkStateReader.getClusterState().liveNodesContain(nodeName)) {
- running = false;
- }
-
- if (cloudJetty.jetty.isRunning()
- && state.equals(ZkStateReader.RECOVERING)
- && zkStateReader.getClusterState().liveNodesContain(nodeName)) {
- numRecovering++;
- }
-
- if (cloudJetty.jetty.isRunning()
- && state.equals(ZkStateReader.ACTIVE)
- && zkStateReader.getClusterState().liveNodesContain(nodeName)) {
- numActive++;
- }
-
- if (running) {
+ numActive = checkIfKillIsLegal(slice, numActive);
+
+ // TODO: stale state makes this a tough call
+ if (numActive < 2) {
+ // we cannot kill anyone
+ monkeyLog("only one active node in shard - monkey cannot kill :(");
+ return null;
+ }
+
+ // let's check the deadpool count
+ int numRunning = 0;
+ for (CloudJettyRunner cjetty : shardToJetty.get(slice)) {
+ if (!deadPool.contains(cjetty)) {
numRunning++;
}
}
- // TODO: stale state makes this a tough call
- if (numActive < 2) {
+ if (numRunning < 2) {
// we cannot kill anyone
monkeyLog("only one active node in shard - monkey cannot kill :(");
return null;
}
+
Random random = LuceneTestCase.random();
int chance = random.nextInt(10);
CloudJettyRunner cjetty;
@@ -439,6 +414,33 @@ public class ChaosMonkey {
return cjetty;
}
+
+ private int checkIfKillIsLegal(String slice, int numActive)
+ throws KeeperException, InterruptedException {
+ for (CloudJettyRunner cloudJetty : shardToJetty.get(slice)) {
+
+ // get latest cloud state
+ zkStateReader.updateClusterState(true);
+
+ Slice theShards = zkStateReader.getClusterState().getSlicesMap(collection)
+ .get(slice);
+
+ ZkNodeProps props = theShards.getReplicasMap().get(cloudJetty.coreNodeName);
+ if (props == null) {
+ throw new RuntimeException("shard name " + cloudJetty.coreNodeName + " not found in " + theShards.getReplicasMap().keySet());
+ }
+
+ String state = props.getStr(ZkStateReader.STATE_PROP);
+ String nodeName = props.getStr(ZkStateReader.NODE_NAME_PROP);
+
+ if (cloudJetty.jetty.isRunning()
+ && state.equals(ZkStateReader.ACTIVE)
+ && zkStateReader.getClusterState().liveNodesContain(nodeName)) {
+ numActive++;
+ }
+ }
+ return numActive;
+ }
public SolrServer getRandomClient(String slice) throws KeeperException, InterruptedException {
// get latest cloud state
@@ -473,7 +475,6 @@ public class ChaosMonkey {
stop = false;
monkeyThread = new Thread() {
- private List<CloudJettyRunner> deadPool = new ArrayList<CloudJettyRunner>();
@Override
public void run() {