You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/11/19 12:12:47 UTC
svn commit: r1543382 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/update/
solr/core/src/test/org/apache/solr/cloud/ solr/solrj/
solr/solrj/src/java/org/apache/solr/common/cloud/
Author: shalin
Date: Tue Nov 19 11:12:47 2013
New Revision: 1543382
URL: http://svn.apache.org/r1543382
Log:
SOLR-5320: Added support for tri-level compositeId routing. Also merged javadoc fixes and forbidden-api fixes from r1542305, r1542321
Added:
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java
- copied, changed from r1542272, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1543382&r1=1543381&r2=1543382&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Tue Nov 19 11:12:47 2013
@@ -215,7 +215,7 @@ public class SolrIndexSplitter {
}
public static String getRouteKey(String idString) {
- int idx = idString.indexOf(CompositeIdRouter.separator);
+ int idx = idString.indexOf(CompositeIdRouter.SEPARATOR);
if (idx <= 0) return null;
String part1 = idString.substring(0, idx);
int commaIdx = part1.indexOf(CompositeIdRouter.bitsSeparator);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java?rev=1543382&r1=1543381&r2=1543382&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java Tue Nov 19 11:12:47 2013
@@ -102,6 +102,10 @@ public class ShardRoutingTest extends Ab
hash of x is 3e9a9b1b high bits=0 shard=shard3
hash of y is 477d9216 high bits=1 shard=shard4
hash of z is c1f69a17 high bits=3 shard=shard2
+
+ hash of f1 is 313bf6b1
+ hash of f2 is ff143f8
+
***/
}
@@ -143,28 +147,31 @@ public class ShardRoutingTest extends Ab
doAddDoc("c!doc2");
doAddDoc("d!doc3");
doAddDoc("e!doc4");
+ doAddDoc("f1!f2!doc5");
doRTG("b!doc1");
doRTG("c!doc2");
doRTG("d!doc3");
doRTG("e!doc4");
+ doRTG("f1!f2!doc5");
doRTG("b!doc1,c!doc2");
doRTG("d!doc3,e!doc4");
commit();
- doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*");
- doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*", "shards","shard1,shard2,shard3,shard4");
- doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*", shardKeys,"b!,c!,d!,e!");
+ doQuery("b!doc1,c!doc2,d!doc3,e!doc4,f1!f2!doc5", "q","*:*");
+ doQuery("b!doc1,c!doc2,d!doc3,e!doc4,f1!f2!doc5", "q","*:*", "shards","shard1,shard2,shard3,shard4");
+ doQuery("b!doc1,c!doc2,d!doc3,e!doc4,f1!f2!doc5", "q","*:*", shardKeys,"b!,c!,d!,e!,f1!f2!");
doQuery("b!doc1", "q","*:*", shardKeys,"b!");
doQuery("c!doc2", "q","*:*", shardKeys,"c!");
- doQuery("d!doc3", "q","*:*", shardKeys,"d!");
+ doQuery("d!doc3,f1!f2!doc5", "q","*:*", shardKeys,"d!");
doQuery("e!doc4", "q","*:*", shardKeys,"e!");
+ doQuery("f1!f2!doc5,d!doc3", "q","*:*", shardKeys,"f1/8!");
// try using shards parameter
doQuery("b!doc1", "q","*:*", "shards",bucket1);
doQuery("c!doc2", "q","*:*", "shards",bucket2);
- doQuery("d!doc3", "q","*:*", "shards",bucket3);
+ doQuery("d!doc3,f1!f2!doc5", "q","*:*", "shards",bucket3);
doQuery("e!doc4", "q","*:*", "shards",bucket4);
@@ -174,26 +181,27 @@ public class ShardRoutingTest extends Ab
doQuery("b!doc1,c!doc2", "q","*:*", shardKeys,"b,c"); // query shards that would contain *documents* "b" and "c" (i.e. not prefixes). The upper bits are the same, so the shards should be the same.
doQuery("b!doc1,c!doc2", "q","*:*", shardKeys,"b/1!"); // top bit of hash(b)==1, so shard1 and shard2
- doQuery("d!doc3,e!doc4", "q","*:*", shardKeys,"d/1!"); // top bit of hash(b)==0, so shard3 and shard4
+ doQuery("d!doc3,e!doc4,f1!f2!doc5", "q","*:*", shardKeys,"d/1!"); // top bit of hash(b)==0, so shard3 and shard4
doQuery("b!doc1,c!doc2", "q","*:*", shardKeys,"b!,c!");
- doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*", shardKeys,"foo/0!");
+ doQuery("b!doc1,f1!f2!doc5,c!doc2,d!doc3,e!doc4", "q","*:*", shardKeys,"foo/0!");
// test targeting deleteByQuery at only certain shards
doDBQ("*:*", shardKeys,"b!");
commit();
- doQuery("c!doc2,d!doc3,e!doc4", "q","*:*");
+ doQuery("c!doc2,d!doc3,e!doc4,f1!f2!doc5", "q","*:*");
doAddDoc("b!doc1");
- doDBQ("*:*", shardKeys,"c!");
+ doDBQ("*:*", shardKeys,"f1!");
commit();
- doQuery("b!doc1,d!doc3,e!doc4", "q","*:*");
- doAddDoc("c!doc2");
+ doQuery("b!doc1,c!doc2,e!doc4", "q","*:*");
+ doAddDoc("f1!f2!doc5");
+ doAddDoc("d!doc3");
doDBQ("*:*", shardKeys,"c!");
commit();
- doQuery("b!doc1,d!doc3,e!doc4", "q","*:*");
+ doQuery("b!doc1,f1!f2!doc5,d!doc3,e!doc4", "q","*:*");
doAddDoc("c!doc2");
doDBQ("*:*", shardKeys,"d!,e!");
@@ -201,15 +209,12 @@ public class ShardRoutingTest extends Ab
doQuery("b!doc1,c!doc2", "q","*:*");
doAddDoc("d!doc3");
doAddDoc("e!doc4");
+ doAddDoc("f1!f2!doc5");
commit();
}
-
-
-
-
public void doTestNumRequests() throws Exception {
log.info("### STARTING doTestNumRequests");
@@ -263,6 +268,11 @@ public class ShardRoutingTest extends Ab
leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!,d!") );
nEnd = getNumRequests();
assertEquals(5, nEnd - nStart); // original + 2 phase distrib search * 2 shards.
+
+ nStart = getNumRequests();
+ leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!,f1!f2!") );
+ nEnd = getNumRequests();
+ assertEquals(5, nEnd - nStart);
}
public void doAtomicUpdate() throws Exception {
@@ -279,9 +289,10 @@ public class ShardRoutingTest extends Ab
Object val = ((Map)rsp.getResponse().get("doc")).get("foo_i");
assertEquals((Integer)expectedVal, val);
}
+
}
- long getNumRequests() {
+ long getNumRequests() {
long n = controlJetty.getDebugFilter().getTotalRequests();
for (JettySolrRunner jetty : jettys) {
n += jetty.getDebugFilter().getTotalRequests();
Copied: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java (from r1542272, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java?p2=lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java&p1=lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java&r1=1542272&r2=1543382&rev=1543382&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java Tue Nov 19 11:12:47 2013
@@ -80,8 +80,6 @@ public class TriLevelCompositeIdRoutingT
// for now, we know how ranges will be distributed to shards.
// may have to look it up in clusterstate if that assumption changes.
- Random r = new Random();
-
for (int i=0;i < NUM_DOCS;i++) {
int appId = r.nextInt(NUM_APPS) + 1;
int userId = r.nextInt(NUM_USERS) + 1;
@@ -113,8 +111,6 @@ public class TriLevelCompositeIdRoutingT
// may have to look it up in clusterstate if that assumption changes.
del("*:*");
- Random r = new Random();
-
for (int i=0;i < NUM_DOCS;i++) {
int appId = r.nextInt(NUM_APPS) + 1;
int userId = r.nextInt(NUM_USERS) + 1;
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java?rev=1543382&r1=1543381&r2=1543382&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java Tue Nov 19 11:12:47 2013
@@ -29,35 +29,18 @@ import java.util.List;
//
// user!uniqueid
+// app!user!uniqueid
// user/4!uniqueid
+// app/2!user/4!uniqueid
//
public class CompositeIdRouter extends HashBasedRouter {
public static final String NAME = "compositeId";
- public static final int separator = '!';
+ public static final String SEPARATOR = "!";
// separator used to optionally specify number of bits to allocate toward first part.
public static final int bitsSeparator = '/';
private int bits = 16;
- private int mask1 = 0xffff0000;
- private int mask2 = 0x0000ffff;
-
- protected void setBits(int firstBits) {
- this.bits = firstBits;
- // java can't shift 32 bits
- mask1 = firstBits==0 ? 0 : (-1 << (32-firstBits));
- mask2 = firstBits==32 ? 0 : (-1 >>> firstBits);
- }
-
- protected int getBits(String firstPart, int commaIdx) {
- int v = 0;
- for (int idx = commaIdx + 1; idx<firstPart.length(); idx++) {
- char ch = firstPart.charAt(idx);
- if (ch < '0' || ch > '9') return -1;
- v = v * 10 + (ch - '0');
- }
- return v > 32 ? -1 : v;
- }
@Override
public int sliceHash(String id, SolrInputDocument doc, SolrParams params, DocCollection collection) {
@@ -68,55 +51,26 @@ public class CompositeIdRouter extends H
throw new SolrException (SolrException.ErrorCode.BAD_REQUEST, "No value for :"+shardFieldName + ". Unable to identify shard");
id = o.toString();
}
- int idx = id.indexOf(separator);
- if (idx < 0) {
+ if (id.indexOf(SEPARATOR) < 0) {
return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
}
- String part1 = id.substring(0, idx);
- int commaIdx = part1.indexOf(bitsSeparator);
- int m1 = mask1;
- int m2 = mask2;
-
- if (commaIdx > 0) {
- int firstBits = getBits(part1, commaIdx);
- if (firstBits >= 0) {
- m1 = firstBits==0 ? 0 : (-1 << (32-firstBits));
- m2 = firstBits==32 ? 0 : (-1 >>> firstBits);
- part1 = part1.substring(0, commaIdx);
- }
- }
-
- String part2 = id.substring(idx+1);
- int hash1 = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0);
- int hash2 = Hash.murmurhash3_x86_32(part2, 0, part2.length(), 0);
- return (hash1 & m1) | (hash2 & m2);
+ return new KeyParser(id).getHash();
}
+
+ /**
+ * Get Range for a given CompositeId based route key
+ * @param routeKey to return Range for
+ * @return Range for given routeKey
+ */
public Range keyHashRange(String routeKey) {
- int idx = routeKey.indexOf(separator);
- if (idx < 0) {
+ if (routeKey.indexOf(SEPARATOR) < 0) {
int hash = sliceHash(routeKey, null, null, null);
return new Range(hash, hash);
}
- String part1 = routeKey.substring(0, idx);
- int commaIdx = part1.indexOf(bitsSeparator);
- int m1 = mask1;
- int m2 = mask2;
-
- if (commaIdx > 0) {
- int firstBits = getBits(part1, commaIdx);
- if (firstBits >= 0) {
- m1 = firstBits==0 ? 0 : (-1 << (32-firstBits));
- m2 = firstBits==32 ? 0 : (-1 >>> firstBits);
- part1 = part1.substring(0, commaIdx);
- }
- }
- int hash = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0);
- int min = hash & m1;
- int max = min | m2;
- return new Range(min, max);
+ return new KeyParser(routeKey).getRange();
}
@Override
@@ -128,41 +82,12 @@ public class CompositeIdRouter extends H
}
String id = shardKey;
- int idx = shardKey.indexOf(separator);
- if (idx < 0) {
+ if (shardKey.indexOf(SEPARATOR) < 0) {
// shardKey is a simple id, so don't do a range
return Collections.singletonList(hashToSlice(Hash.murmurhash3_x86_32(id, 0, id.length(), 0), collection));
}
- int m1 = mask1;
- int m2 = mask2;
-
- String part1 = id.substring(0,idx);
- int bitsSepIdx = part1.indexOf(bitsSeparator);
- if (bitsSepIdx > 0) {
- int firstBits = getBits(part1, bitsSepIdx);
- if (firstBits >= 0) {
- m1 = firstBits==0 ? 0 : (-1 << (32-firstBits));
- m2 = firstBits==32 ? 0 : (-1 >>> firstBits);
- part1 = part1.substring(0, bitsSepIdx);
- }
- }
-
- // If the upper bits are 0xF0000000, the range we want to cover is
- // 0xF0000000 0xFfffffff
-
- int hash1 = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0);
- int upperBits = hash1 & m1;
- int lowerBound = upperBits;
- int upperBound = upperBits | m2;
-
- if (m1 == 0) {
- // no bits used from first part of key.. the code above will produce 0x000000000->0xffffffff which only works on unsigned space, but we're using signed space.
- lowerBound = Integer.MIN_VALUE;
- upperBound = Integer.MAX_VALUE;
- }
-
- Range completeRange = new Range(lowerBound, upperBound);
+ Range completeRange = new KeyParser(id).getRange();
List<Slice> targetSlices = new ArrayList<Slice>(1);
for (Slice slice : collection.getActiveSlices()) {
@@ -219,17 +144,18 @@ public class CompositeIdRouter extends H
// Round to avoid splitting hash domains across ranges if such rounding is not significant.
// With default bits==16, one would need to create more than 4000 shards before this
// becomes false by default.
+ int mask = 0x0000ffff;
boolean round = rangeStep >= (1<<bits)*16;
while (end < max) {
targetEnd = targetStart + rangeStep;
end = targetEnd;
- if (round && ((end & mask2) != mask2)) {
+ if (round && ((end & mask) != mask)) {
// round up or down?
int increment = 1 << bits; // 0x00010000
- long roundDown = (end | mask2) - increment ;
- long roundUp = (end | mask2) + increment;
+ long roundDown = (end | mask) - increment ;
+ long roundUp = (end | mask) + increment;
if (end - roundDown < roundUp - end && roundDown > start) {
end = roundDown;
} else {
@@ -249,4 +175,120 @@ public class CompositeIdRouter extends H
return ranges;
}
+ /**
+ * Helper class to calculate parts, masks etc for an id.
+ */
+ static class KeyParser {
+ String key;
+ int[] numBits;
+ int[] hashes;
+ int[] masks;
+ boolean triLevel;
+ int pieces;
+
+ public KeyParser(String key) {
+ String[] parts = key.split(SEPARATOR);
+ this.key = key;
+ pieces = parts.length;
+ hashes = new int[pieces];
+ numBits = new int[2];
+ if(key.endsWith("!"))
+ pieces++;
+ if(pieces == 3) {
+ numBits[0] = 8;
+ numBits[1] = 8;
+ triLevel = true;
+ } else {
+ numBits[0] = 16;
+ triLevel = false;
+
+ }
+
+ for(int i=0;i<parts.length;i++) {
+ int commaIdx = parts[i].indexOf(bitsSeparator);
+
+ if(commaIdx > 0) {
+ numBits[i] = getNumBits(parts[i], commaIdx);
+ parts[i] = parts[i].substring(0, commaIdx);
+ }
+ hashes[i] = Hash.murmurhash3_x86_32(parts[i], 0, parts[i].length(), 0);
+ }
+ masks = getMasks();
+ }
+
+ Range getRange() {
+ int lowerBound;
+ int upperBound;
+
+ if(triLevel) {
+ lowerBound = hashes[0] & masks[0] | hashes[1] & masks[1];
+ upperBound = lowerBound | masks[2];
+ } else {
+ lowerBound = hashes[0] & masks[0];
+ upperBound = lowerBound | masks[1];
+ }
+ // If the upper bits are 0xF0000000, the range we want to cover is
+ // 0xF0000000 0xFfffffff
+
+ if ((masks[0] == 0 && !triLevel) || (masks[0] == 0 && masks[1] == 0 && triLevel)) {
+ // no bits used from first part of key.. the code above will produce 0x000000000->0xffffffff
+ // which only works on unsigned space, but we're using signed space.
+ lowerBound = Integer.MIN_VALUE;
+ upperBound = Integer.MAX_VALUE;
+ }
+ Range r = new Range(lowerBound, upperBound);
+ return r;
+ }
+
+ /**
+ * Get bit masks for routing based on routing level
+ */
+ private int[] getMasks() {
+ int[] masks;
+ if(triLevel)
+ masks = getBitMasks(numBits[0], numBits[1]);
+ else
+ masks = getBitMasks(numBits[0]);
+
+ return masks;
+ }
+
+ private int[] getBitMasks(int firstBits, int secondBits) {
+ // java can't shift 32 bits
+ int[] masks = new int[3];
+ masks[0] = firstBits==0 ? 0 : (-1 << (32-firstBits));
+ masks[1] = (firstBits + secondBits)==0 ? 0 : (-1 << (32 - firstBits - secondBits));
+ masks[1] = masks[0] ^ masks[1];
+ masks[2] = (firstBits + secondBits) == 32 ? 0 : ~(masks[0] | masks[1]);
+ return masks;
+ }
+
+ private int getNumBits(String firstPart, int commaIdx) {
+ int v = 0;
+ for (int idx = commaIdx + 1; idx<firstPart.length(); idx++) {
+ char ch = firstPart.charAt(idx);
+ if (ch < '0' || ch > '9') return -1;
+ v = v * 10 + (ch - '0');
+ }
+ return v > 32 ? -1 : v;
+ }
+
+ private int[] getBitMasks(int firstBits) {
+ // java can't shift 32 bits
+ int[] masks;
+ masks = new int[2];
+ masks[0] = firstBits==0 ? 0 : (-1 << (32-firstBits));
+ masks[1] = firstBits==32 ? 0 : (-1 >>> firstBits);
+ return masks;
+ }
+
+ int getHash() {
+ int result = hashes[0] & masks[0];
+
+ for(int i=1;i<pieces;i++)
+ result = result | (hashes[i] & masks[i]);
+ return result;
+ }
+
+ }
}