You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [38/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/...
Modified: lucene/dev/branches/LUCENE-2878/solr/example/solr/collection1/conf/velocity/richtext-doc.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/example/solr/collection1/conf/velocity/richtext-doc.vm?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/example/solr/collection1/conf/velocity/richtext-doc.vm (original)
+++ lucene/dev/branches/LUCENE-2878/solr/example/solr/collection1/conf/velocity/richtext-doc.vm Fri Jan 18 18:30:54 2013
@@ -1,3 +1,59 @@
+## Mimetype to extension map for detecting file type and show icon
+## List of types match the icons in /solr/img/filetypes
+#set($extMap = {"application/x-7z-compressed": "7z",
+ "application/postscript": "ai",
+ "application/pgp-signature": "asc",
+ "application/octet-stream": "bin",
+ "application/x-bzip2": "bz2",
+ "text/x-c": "c",
+ "application/vnd.ms-htmlhelp": "chm",
+ "application/java-vm": "class",
+ "text/css": "css",
+ "text/csv": "csv",
+ "application/x-debian-package": "deb",
+ "application/msword": "doc",
+ "message/rfc822": "eml",
+ "image/gif": "gif",
+ "application/winhlp": "hlp",
+ "text/html": "html",
+ "application/java-archive": "jar",
+ "text/x-java-source": "java",
+ "image/jpeg": "jpeg",
+ "application/javascript": "js",
+ "application/vnd.oasis.opendocument.chart": "odc",
+ "application/vnd.oasis.opendocument.formula": "odf",
+ "application/vnd.oasis.opendocument.graphics": "odg",
+ "application/vnd.oasis.opendocument.image": "odi",
+ "application/vnd.oasis.opendocument.presentation": "odp",
+ "application/vnd.oasis.opendocument.spreadsheet": "ods",
+ "application/vnd.oasis.opendocument.text": "odt",
+ "application/pdf": "pdf",
+ "application/pgp-encrypted": "pgp",
+ "image/png": "png",
+ "application/vnd.ms-powerpoint": "ppt",
+ "audio/x-pn-realaudio": "ram",
+ "application/x-rar-compressed": "rar",
+ "application/vnd.rn-realmedia": "rm",
+ "application/rtf": "rtf",
+ "application/x-shockwave-flash": "swf",
+ "application/vnd.sun.xml.calc": "sxc",
+ "application/vnd.sun.xml.draw": "sxd",
+ "application/vnd.sun.xml.impress": "sxi",
+ "application/vnd.sun.xml.writer": "sxw",
+ "application/x-tar": "tar",
+ "application/x-tex": "tex",
+ "text/plain": "txt",
+ "text/x-vcard": "vcf",
+ "application/vnd.visio": "vsd",
+ "audio/x-wav": "wav",
+ "audio/x-ms-wma": "wma",
+ "video/x-ms-wmv": "wmv",
+ "application/vnd.ms-excel": "xls",
+ "application/xml": "xml",
+ "application/x-xpinstall": "xpi",
+ "application/zip": "zip"})
+
+
#if($doc.getFieldValue('title'))
#set($title = $esc.html($doc.getFirstValue('title')))
#else
@@ -13,10 +69,16 @@
#set($supportedtypes = "7z;ai;aiff;asc;audio;bin;bz2;c;cfc;cfm;chm;class;conf;cpp;cs;css;csv;deb;divx;doc;dot;eml;enc;file;gif;gz;hlp;htm;html;image;iso;jar;java;jpeg;jpg;js;lua;m;mm;mov;mp3;mpg;odc;odf;odg;odi;odp;ods;odt;ogg;pdf;pgp;php;pl;png;ppt;ps;py;ram;rar;rb;rm;rpm;rtf;sig;sql;swf;sxc;sxd;sxi;sxw;tar;tex;tgz;txt;vcf;video;vsd;wav;wma;wmv;xls;xml;xpi;xvid;zip")
#set($ct = $list.get($doc.getFirstValue('content_type').split(";"),0))
#set($filename = $doc.getFieldValue('resourcename'))
-#set($filetype = $mimeToExt.get($ct))
+
+#set($filetype = false)
+#set($filetype = $extMap.get($ct))
+##TODO: falling back to file extension is convenient, except when you don't have an icon for that extension
+## example "application/vnd.openxmlformats-officedocument.wordprocessingml.document" document
+## with a .docx extension. It'd be nice to fall back to an "unknown" or the existing "file" type
+## We sort of do this below, but only if the filename has no extension (anything after the last dot).
#if(!$filetype)#set($filetype = $filename.substring($filename.lastIndexOf(".")).substring(1))#end
-#if(!$filetype)#set($filetype = "file")#end
-#if(!$supportedtypes.contains($filetype))#set($filetype = "file")#end
+##if(!$filetype)#set($filetype = "file")#end
+##if(!$supportedtypes.contains($filetype))#set($filetype = "file")#end
<div class="result-title">
## Small file type icons from http://www.splitbrain.org/projects/file_icons (public domain)
<img src="#{url_root}/img/filetypes/${filetype}.png" align="center">
@@ -29,7 +91,7 @@
#if($ct) ($ct)#end
</div>
#if($doc.getFieldValue('author'))<div>Author: #field('author')</div>#end
-#if($doc.getFieldValue('last_modified'))<div>Last-modified: #field('last_modified')</div>#end
+#if($doc.getFieldValue('last_modified'))<div>last-modified: #field('last_modified')</div>#end
<div class="result-body">#field('content')</div>
<div class="mlt">
#set($mlt = $mltResults.get($docId))
Modified: lucene/dev/branches/LUCENE-2878/solr/licenses/start.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/licenses/start.jar.sha1?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/licenses/start.jar.sha1 (original)
+++ lucene/dev/branches/LUCENE-2878/solr/licenses/start.jar.sha1 Fri Jan 18 18:30:54 2013
@@ -1 +1 @@
-b90a67a4f1f6db9e9cea94663e82c320d70e8982
+5f657368cd61ab70babe98bcd6709e199a4829d9
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/ivy.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/ivy.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/ivy.xml Fri Jan 18 18:30:54 2013
@@ -21,7 +21,6 @@
<dependencies>
<dependency org="org.apache.zookeeper" name="zookeeper" rev="3.4.5" transitive="false"/>
- <dependency org="org.slf4j" name="log4j-over-slf4j" rev="1.6.4" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpcore" rev="4.1.4" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.1.3" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpmime" rev="4.1.3" transitive="false"/>
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/CharArr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/CharArr.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/CharArr.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/CharArr.java Fri Jan 18 18:30:54 2013
@@ -65,14 +65,17 @@ public class CharArr implements CharSequ
public int getStart() { return start; }
public int getEnd() { return end; }
public int size() { return end-start; }
+ @Override
public int length() { return size(); }
public int capacity() { return buf.length; }
+ @Override
public char charAt(int index) {
return buf[start+index];
}
+ @Override
public CharArr subSequence(int start, int end) {
return new CharArr(buf, this.start+start, this.start+end);
}
@@ -157,6 +160,7 @@ public class CharArr implements CharSequ
}
+ @Override
public String toString() {
return new String(buf, start, size());
}
@@ -189,15 +193,18 @@ public class CharArr implements CharSequ
}
//////////////// Appendable methods /////////////
+ @Override
public final Appendable append(CharSequence csq) throws IOException {
return append(csq, 0, csq.length());
}
+ @Override
public Appendable append(CharSequence csq, int start, int end) throws IOException {
write(csq.subSequence(start, end).toString());
return null;
}
+ @Override
public final Appendable append(char c) {
write(c);
return this;
@@ -209,28 +216,38 @@ class NullCharArr extends CharArr {
public NullCharArr() {
super(new char[1],0,0);
}
+ @Override
public void unsafeWrite(char b) {}
+ @Override
public void unsafeWrite(char b[], int off, int len) {}
+ @Override
public void unsafeWrite(int b) {}
+ @Override
public void write(char b) {}
+ @Override
public void write(char b[], int off, int len) {}
+ @Override
public void reserve(int num) {}
+ @Override
protected void resize(int len) {}
+ @Override
public Appendable append(CharSequence csq, int start, int end) throws IOException {
return this;
}
+ @Override
public char charAt(int index) {
return 0;
}
+ @Override
public void write(String s, int stringOffset, int len) {
}
}
@@ -246,11 +263,13 @@ class CharArrReader extends CharArr {
this.in = in;
}
+ @Override
public int read() throws IOException {
if (start>=end) fill();
return start>=end ? -1 : buf[start++];
}
+ @Override
public int read(CharBuffer cb) throws IOException {
// empty the buffer and then read direct
int sz = size();
@@ -260,6 +279,7 @@ class CharArrReader extends CharArr {
return sz>0 ? sz : -1;
}
+ @Override
public int fill() throws IOException {
if (start>=end) {
reset();
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/JSONParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/JSONParser.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/JSONParser.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/noggit/JSONParser.java Fri Jan 18 18:30:54 2013
@@ -624,6 +624,7 @@ public class JSONParser {
}
}
+ @Override
public String toString() {
return "start="+start+",end="+end+",state="+state+"valstate="+valstate;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java Fri Jan 18 18:30:54 2013
@@ -67,27 +67,33 @@ public class BinaryRequestWriter extends
new JavaBinUpdateRequestCodec().marshal(request, baos);
return new ContentStream() {
+ @Override
public String getName() {
return null;
}
+ @Override
public String getSourceInfo() {
return "javabin";
}
+ @Override
public String getContentType() {
return "application/javabin";
}
+ @Override
public Long getSize() // size if we know it, otherwise null
{
return new Long(baos.size());
}
+ @Override
public InputStream getStream() {
return new ByteArrayInputStream(baos.getbuf(), 0, baos.size());
}
+ @Override
public Reader getReader() {
throw new RuntimeException("No reader available . this is a binarystream");
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Fri Jan 18 18:30:54 2013
@@ -20,6 +20,7 @@ package org.apache.solr.client.solrj.imp
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -162,9 +163,10 @@ public class CloudSolrServer extends Sol
}
@Override
- public NamedList<Object> request(SolrRequest request) throws SolrServerException, IOException {
+ public NamedList<Object> request(SolrRequest request)
+ throws SolrServerException, IOException {
connect();
-
+
// TODO: if you can hash here, you could favor the shard leader
ClusterState clusterState = zkStateReader.getClusterState();
@@ -175,96 +177,111 @@ public class CloudSolrServer extends Sol
sendToLeaders = true;
replicas = new ArrayList<String>();
}
-
+
SolrParams reqParams = request.getParams();
if (reqParams == null) {
reqParams = new ModifiableSolrParams();
}
- String collection = reqParams.get("collection", defaultCollection);
-
- if (collection == null) {
- throw new SolrServerException("No collection param specified on request and no default collection has been set.");
- }
-
- // Extract each comma separated collection name and store in a List.
- List<String> collectionList = StrUtils.splitSmart(collection, ",", true);
-
- // TODO: not a big deal because of the caching, but we could avoid looking at every shard
- // when getting leaders if we tweaked some things
-
- // Retrieve slices from the cloud state and, for each collection specified,
- // add it to the Map of slices.
- Map<String,Slice> slices = new HashMap<String,Slice>();
- for (int i = 0; i < collectionList.size(); i++) {
- String coll= collectionList.get(i);
- ClientUtils.appendMap(coll, slices, clusterState.getSlices(coll));
- }
-
- Set<String> liveNodes = clusterState.getLiveNodes();
-
- List<String> theUrlList;
- synchronized (cachLock) {
- List<String> leaderUrlList = leaderUrlLists.get(collection);
- List<String> urlList = urlLists.get(collection);
- List<String> replicasList = replicasLists.get(collection);
-
- if ((sendToLeaders && leaderUrlList == null) || (!sendToLeaders
- && urlList == null)
- || clusterState.hashCode() != this.lastClusterStateHashCode) {
- // build a map of unique nodes
- // TODO: allow filtering by group, role, etc
- Map<String,ZkNodeProps> nodes = new HashMap<String,ZkNodeProps>();
- List<String> urlList2 = new ArrayList<String>();
- for (Slice slice : slices.values()) {
- for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
- ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
- String node = coreNodeProps.getNodeName();
- if (!liveNodes.contains(coreNodeProps.getNodeName())
- || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue;
- if (nodes.put(node, nodeProps) == null) {
- if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) {
- String url = coreNodeProps.getCoreUrl();
- urlList2.add(url);
- } else if (sendToLeaders) {
- String url = coreNodeProps.getCoreUrl();
- replicas.add(url);
+ List<String> theUrlList = new ArrayList<String>();
+ if (request.getPath().equals("/admin/collections")) {
+ Set<String> liveNodes = clusterState.getLiveNodes();
+ for (String liveNode : liveNodes) {
+ int splitPointBetweenHostPortAndContext = liveNode.indexOf("_");
+ theUrlList.add("http://"
+ + liveNode.substring(0, splitPointBetweenHostPortAndContext) + "/"
+ + liveNode.substring(splitPointBetweenHostPortAndContext + 1));
+ }
+ } else {
+ String collection = reqParams.get("collection", defaultCollection);
+
+ if (collection == null) {
+ throw new SolrServerException(
+ "No collection param specified on request and no default collection has been set.");
+ }
+
+ // Extract each comma separated collection name and store in a List.
+ List<String> collectionList = StrUtils.splitSmart(collection, ",", true);
+
+ // TODO: not a big deal because of the caching, but we could avoid looking
+ // at every shard
+ // when getting leaders if we tweaked some things
+
+ // Retrieve slices from the cloud state and, for each collection
+ // specified,
+ // add it to the Map of slices.
+ Map<String,Slice> slices = new HashMap<String,Slice>();
+ for (String collectionName : collectionList) {
+ ClientUtils.addSlices(slices, collectionName,
+ clusterState.getSlices(collectionName), true);
+ }
+ Set<String> liveNodes = clusterState.getLiveNodes();
+
+ synchronized (cachLock) {
+ List<String> leaderUrlList = leaderUrlLists.get(collection);
+ List<String> urlList = urlLists.get(collection);
+ List<String> replicasList = replicasLists.get(collection);
+
+ if ((sendToLeaders && leaderUrlList == null)
+ || (!sendToLeaders && urlList == null)
+ || clusterState.hashCode() != this.lastClusterStateHashCode) {
+ // build a map of unique nodes
+ // TODO: allow filtering by group, role, etc
+ Map<String,ZkNodeProps> nodes = new HashMap<String,ZkNodeProps>();
+ List<String> urlList2 = new ArrayList<String>();
+ for (Slice slice : slices.values()) {
+ for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
+ ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
+ String node = coreNodeProps.getNodeName();
+ if (!liveNodes.contains(coreNodeProps.getNodeName())
+ || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue;
+ if (nodes.put(node, nodeProps) == null) {
+ if (!sendToLeaders
+ || (sendToLeaders && coreNodeProps.isLeader())) {
+ String url = coreNodeProps.getCoreUrl();
+ urlList2.add(url);
+ } else if (sendToLeaders) {
+ String url = coreNodeProps.getCoreUrl();
+ replicas.add(url);
+ }
}
}
}
+
+ if (sendToLeaders) {
+ this.leaderUrlLists.put(collection, urlList2);
+ leaderUrlList = urlList2;
+ this.replicasLists.put(collection, replicas);
+ replicasList = replicas;
+ } else {
+ this.urlLists.put(collection, urlList2);
+ urlList = urlList2;
+ }
+ this.lastClusterStateHashCode = clusterState.hashCode();
}
+
if (sendToLeaders) {
- this.leaderUrlLists.put(collection, urlList2);
- leaderUrlList = urlList2;
- this.replicasLists.put(collection, replicas);
- replicasList = replicas;
+ theUrlList = new ArrayList<String>(leaderUrlList.size());
+ theUrlList.addAll(leaderUrlList);
} else {
- this.urlLists.put(collection, urlList2);
- urlList = urlList2;
+ theUrlList = new ArrayList<String>(urlList.size());
+ theUrlList.addAll(urlList);
+ }
+ Collections.shuffle(theUrlList, rand);
+ if (sendToLeaders) {
+ ArrayList<String> theReplicas = new ArrayList<String>(
+ replicasList.size());
+ theReplicas.addAll(replicasList);
+ Collections.shuffle(theReplicas, rand);
+ // System.out.println("leaders:" + theUrlList);
+ // System.out.println("replicas:" + theReplicas);
+ theUrlList.addAll(theReplicas);
}
- this.lastClusterStateHashCode = clusterState.hashCode();
- }
-
- if (sendToLeaders) {
- theUrlList = new ArrayList<String>(leaderUrlList.size());
- theUrlList.addAll(leaderUrlList);
- } else {
- theUrlList = new ArrayList<String>(urlList.size());
- theUrlList.addAll(urlList);
- }
- Collections.shuffle(theUrlList, rand);
- if (sendToLeaders) {
- ArrayList<String> theReplicas = new ArrayList<String>(
- replicasList.size());
- theReplicas.addAll(replicasList);
- Collections.shuffle(theReplicas, rand);
- // System.out.println("leaders:" + theUrlList);
- // System.out.println("replicas:" + theReplicas);
- theUrlList.addAll(theReplicas);
}
}
-
- // System.out.println("########################## MAKING REQUEST TO " + theUrlList);
-
+
+ // System.out.println("########################## MAKING REQUEST TO " +
+ // theUrlList);
+
LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, theUrlList);
LBHttpSolrServer.Rsp rsp = lbServer.request(req);
return rsp.getResponse();
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java Fri Jan 18 18:30:54 2013
@@ -114,6 +114,7 @@ public class ConcurrentUpdateSolrServer
class Runner implements Runnable {
final Lock runnerLock = new ReentrantLock();
+ @Override
public void run() {
runnerLock.lock();
@@ -136,6 +137,7 @@ public class ConcurrentUpdateSolrServer
EntityTemplate template = new EntityTemplate(new ContentProducer() {
+ @Override
public void writeTo(OutputStream out) throws IOException {
try {
if (isXml) {
@@ -243,6 +245,7 @@ public class ConcurrentUpdateSolrServer
}
}
+ @Override
public NamedList<Object> request(final SolrRequest request)
throws SolrServerException, IOException {
if (!(request instanceof UpdateRequest)) {
@@ -369,6 +372,18 @@ public class ConcurrentUpdateSolrServer
Thread.currentThread().interrupt();
}
}
+
+ public void setConnectionTimeout(int timeout) {
+ HttpClientUtil.setConnectionTimeout(server.getHttpClient(), timeout);
+ }
+
+ /**
+ * set soTimeout (read timeout) on the underlying HttpConnectionManager. This is desirable for queries, but probably
+ * not for indexing.
+ */
+ public void setSoTimeout(int timeout) {
+ HttpClientUtil.setSoTimeout(server.getHttpClient(), timeout);
+ }
public void shutdownNow() {
server.shutdown();
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java Fri Jan 18 18:30:54 2013
@@ -238,6 +238,7 @@ public class HttpClientUtil {
private static class UseCompressionResponseInterceptor implements
HttpResponseInterceptor {
+ @Override
public void process(final HttpResponse response, final HttpContext context)
throws HttpException, IOException {
@@ -266,10 +267,12 @@ public class HttpClientUtil {
super(entity);
}
+ @Override
public InputStream getContent() throws IOException, IllegalStateException {
return new GZIPInputStream(wrappedEntity.getContent());
}
+ @Override
public long getContentLength() {
return -1;
}
@@ -281,6 +284,7 @@ public class HttpClientUtil {
super(entity);
}
+ @Override
public InputStream getContent() throws IOException, IllegalStateException {
return new InflaterInputStream(wrappedEntity.getContent());
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java Fri Jan 18 18:30:54 2013
@@ -346,6 +346,7 @@ public class HttpSolrServer extends Solr
method.addHeader("User-Agent", AGENT);
InputStream respBody = null;
+ boolean shouldClose = true;
try {
// Execute the method.
@@ -378,6 +379,8 @@ public class HttpSolrServer extends Solr
// no processor specified, return raw stream
NamedList<Object> rsp = new NamedList<Object>();
rsp.add("stream", respBody);
+ // Only case where stream should not be closed
+ shouldClose = false;
return rsp;
}
String charset = EntityUtils.getContentCharSet(response.getEntity());
@@ -413,7 +416,7 @@ public class HttpSolrServer extends Solr
throw new SolrServerException(
"IOException occured when talking to server at: " + getBaseURL(), e);
} finally {
- if (respBody != null && processor!=null) {
+ if (respBody != null && shouldClose) {
try {
respBody.close();
} catch (Throwable t) {} // ignore
@@ -563,16 +566,19 @@ public class HttpSolrServer extends Solr
UpdateRequest req = new UpdateRequest();
req.setDocIterator(new Iterator<SolrInputDocument>() {
+ @Override
public boolean hasNext() {
return beanIterator.hasNext();
}
+ @Override
public SolrInputDocument next() {
Object o = beanIterator.next();
if (o == null) return null;
return getBinder().toSolrInputDocument(o);
}
+ @Override
public void remove() {
beanIterator.remove();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java Fri Jan 18 18:30:54 2013
@@ -85,7 +85,7 @@ public class LBHttpSolrServer extends So
private final AtomicInteger counter = new AtomicInteger(-1);
private static final SolrQuery solrQuery = new SolrQuery("*:*");
- private static final BinaryResponseParser binaryParser = new BinaryResponseParser();
+ private final ResponseParser parser;
static {
solrQuery.setRows(0);
@@ -189,6 +189,7 @@ public class LBHttpSolrServer extends So
public LBHttpSolrServer(HttpClient httpClient, ResponseParser parser, String... solrServerUrl)
throws MalformedURLException {
clientIsInternal = (httpClient == null);
+ this.parser = parser;
if (httpClient == null) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_USE_RETRY, false);
@@ -210,7 +211,7 @@ public class LBHttpSolrServer extends So
}
protected HttpSolrServer makeServer(String server) throws MalformedURLException {
- return new HttpSolrServer(server, httpClient, binaryParser);
+ return new HttpSolrServer(server, httpClient, parser);
}
@@ -566,6 +567,7 @@ public class LBHttpSolrServer extends So
private static Runnable getAliveCheckRunner(final WeakReference<LBHttpSolrServer> lbRef) {
return new Runnable() {
+ @Override
public void run() {
LBHttpSolrServer lb = lbRef.get();
if (lb != null && lb.zombieServers != null) {
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java Fri Jan 18 18:30:54 2013
@@ -42,6 +42,7 @@ public class CoreAdminRequest extends So
{
protected String core = null;
protected String other = null;
+ protected boolean isIndexInfoNeeded = true;
protected CoreAdminParams.CoreAdminAction action = null;
//a create core request
@@ -50,6 +51,7 @@ public class CoreAdminRequest extends So
protected String configName = null;
protected String schemaName = null;
protected String dataDir = null;
+ protected String ulogDir = null;
protected String collection;
private Integer numShards;
private String shardId;
@@ -63,6 +65,7 @@ public class CoreAdminRequest extends So
public void setSchemaName(String schema) { this.schemaName = schema; }
public void setConfigName(String config) { this.configName = config; }
public void setDataDir(String dataDir) { this.dataDir = dataDir; }
+ public void setUlogDir(String ulogDir) { this.ulogDir = ulogDir; }
public void setCollection(String collection) { this.collection = collection; }
public void setNumShards(int numShards) {this.numShards = numShards;}
public void setShardId(String shardId) {this.shardId = shardId;}
@@ -72,6 +75,7 @@ public class CoreAdminRequest extends So
public String getSchemaName() { return schemaName; }
public String getConfigName() { return configName; }
public String getDataDir() { return dataDir; }
+ public String getUlogDir() { return ulogDir; }
public String getCollection() { return collection; }
public String getShardId() { return shardId; }
public String getRoles() { return roles; }
@@ -98,6 +102,9 @@ public class CoreAdminRequest extends So
if (dataDir != null) {
params.set( CoreAdminParams.DATA_DIR, dataDir);
}
+ if (ulogDir != null) {
+ params.set( CoreAdminParams.ULOG_DIR, ulogDir);
+ }
if (collection != null) {
params.set( CoreAdminParams.COLLECTION, collection);
}
@@ -377,6 +384,10 @@ public class CoreAdminRequest extends So
{
this.other = otherCoreName;
}
+
+ public final void setIndexInfoNeeded(boolean isIndexInfoNeeded) {
+ this.isIndexInfoNeeded = isIndexInfoNeeded;
+ }
//---------------------------------------------------------------------------------------
//
@@ -400,6 +411,7 @@ public class CoreAdminRequest extends So
ModifiableSolrParams params = new ModifiableSolrParams();
params.set( CoreAdminParams.ACTION, action.toString() );
params.set( CoreAdminParams.CORE, core );
+ params.set(CoreAdminParams.INDEX_INFO, (isIndexInfoNeeded ? "true" : "false"));
if (other != null) {
params.set(CoreAdminParams.OTHER, other);
}
@@ -471,11 +483,21 @@ public class CoreAdminRequest extends So
return CoreAdminRequest.createCore(name, instanceDir, server, null, null);
}
- public static CoreAdminResponse createCore( String name, String instanceDir, SolrServer server, String configFile, String schemaFile ) throws SolrServerException, IOException
+ public static CoreAdminResponse createCore( String name, String instanceDir, SolrServer server, String configFile, String schemaFile ) throws SolrServerException, IOException {
+ return createCore(name, instanceDir, server, configFile, schemaFile, null, null);
+ }
+
+ public static CoreAdminResponse createCore( String name, String instanceDir, SolrServer server, String configFile, String schemaFile, String dataDir, String tlogDir ) throws SolrServerException, IOException
{
CoreAdminRequest.Create req = new CoreAdminRequest.Create();
req.setCoreName( name );
req.setInstanceDir(instanceDir);
+ if (dataDir != null) {
+ req.setDataDir(dataDir);
+ }
+ if (tlogDir != null) {
+ req.setUlogDir(tlogDir);
+ }
if(configFile != null){
req.setConfigName(configFile);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java Fri Jan 18 18:30:54 2013
@@ -98,26 +98,32 @@ public class RequestWriter {
return contentStream;
}
+ @Override
public String getName() {
return getDelegate().getName();
}
+ @Override
public String getSourceInfo() {
return getDelegate().getSourceInfo();
}
+ @Override
public String getContentType() {
return getUpdateContentType();
}
+ @Override
public Long getSize() {
return getDelegate().getSize();
}
+ @Override
public InputStream getStream() throws IOException {
return getDelegate().getStream();
}
+ @Override
public Reader getReader() throws IOException {
return getDelegate().getReader();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java Fri Jan 18 18:30:54 2013
@@ -103,6 +103,7 @@ public class DocumentAnalysisResponse ex
*
* @return An iterator over the document analyses map.
*/
+ @Override
public Iterator<Map.Entry<String, DocumentAnalysis>> iterator() {
return documentAnalysisByKey.entrySet().iterator();
}
@@ -152,6 +153,7 @@ public class DocumentAnalysisResponse ex
*
* @return An iterator over the field analyses map.
*/
+ @Override
public Iterator<Map.Entry<String, FieldAnalysis>> iterator() {
return fieldAnalysisByFieldName.entrySet().iterator();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java Fri Jan 18 18:30:54 2013
@@ -23,8 +23,13 @@ import java.io.Writer;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.ParseException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.TimeZone;
import java.nio.ByteBuffer;
import org.apache.solr.common.SolrDocument;
@@ -32,7 +37,11 @@ import org.apache.solr.common.SolrInputD
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.*;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.DateUtil;
+import org.apache.solr.common.util.XML;
/**
@@ -104,41 +113,57 @@ public class ClientUtils
// currently only supports a single value
for (Entry<Object,Object> entry : ((Map<Object,Object>)v).entrySet()) {
update = entry.getKey().toString();
- Object fieldVal = entry.getValue();
- v = fieldVal;
+ v = entry.getValue();
+ if (v instanceof Collection) {
+ Collection values = (Collection) v;
+ for (Object value : values) {
+ writeVal(writer, boost, name, value, update);
+ boost = 1.0f;
+ }
+ } else {
+ writeVal(writer, boost, name, v, update);
+ boost = 1.0f;
+ }
}
+ } else {
+ writeVal(writer, boost, name, v, update);
+ // only write the boost for the first multi-valued field
+ // otherwise, the used boost is the product of all the boost values
+ boost = 1.0f;
}
+ }
+ }
+ writer.write("</doc>");
+ }
- if (v instanceof Date) {
- v = DateUtil.getThreadLocalDateFormat().format( (Date)v );
- } else if (v instanceof byte[]) {
- byte[] bytes = (byte[]) v;
- v = Base64.byteArrayToBase64(bytes, 0,bytes.length);
- } else if (v instanceof ByteBuffer) {
- ByteBuffer bytes = (ByteBuffer) v;
- v = Base64.byteArrayToBase64(bytes.array(), bytes.position(),bytes.limit() - bytes.position());
- }
+ private static void writeVal(Writer writer, float boost, String name, Object v, String update) throws IOException {
+ if (v instanceof Date) {
+ v = DateUtil.getThreadLocalDateFormat().format( (Date)v );
+ } else if (v instanceof byte[]) {
+ byte[] bytes = (byte[]) v;
+ v = Base64.byteArrayToBase64(bytes, 0, bytes.length);
+ } else if (v instanceof ByteBuffer) {
+ ByteBuffer bytes = (ByteBuffer) v;
+ v = Base64.byteArrayToBase64(bytes.array(), bytes.position(),bytes.limit() - bytes.position());
+ }
- if (update == null) {
- if( boost != 1.0f ) {
- XML.writeXML(writer, "field", v.toString(), "name", name, "boost", boost );
- } else if (v != null) {
- XML.writeXML(writer, "field", v.toString(), "name", name );
- }
- } else {
- if( boost != 1.0f ) {
- XML.writeXML(writer, "field", v.toString(), "name", name, "boost", boost, "update", update);
- } else if (v != null) {
- XML.writeXML(writer, "field", v.toString(), "name", name, "update", update);
- }
+ if (update == null) {
+ if( boost != 1.0f ) {
+ XML.writeXML(writer, "field", v.toString(), "name", name, "boost", boost);
+ } else if (v != null) {
+ XML.writeXML(writer, "field", v.toString(), "name", name );
+ }
+ } else {
+ if( boost != 1.0f ) {
+ XML.writeXML(writer, "field", v.toString(), "name", name, "boost", boost, "update", update);
+ } else {
+ if (v == null) {
+ XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true);
+ } else {
+ XML.writeXML(writer, "field", v.toString(), "name", name, "update", update);
}
-
- // only write the boost for the first multi-valued field
- // otherwise, the used boost is the product of all the boost values
- boost = 1.0f;
}
}
- writer.write("</doc>");
}
@@ -242,15 +267,13 @@ public class ClientUtils
catch (IOException e) {throw new RuntimeException(e);} // can't happen
return sb.toString();
}
-
- public static void appendMap(String collection, Map<String,Slice> map1, Map<String,Slice> map2) {
- if (map1==null)
- map1 = new HashMap<String,Slice>();
- if (map2!=null) {
- Set<Entry<String,Slice>> entrySet = map2.entrySet();
- for (Entry<String,Slice> entry : entrySet) {
- map1.put(collection + "_" + entry.getKey(), entry.getValue());
- }
+
+ /** Constructs a slices map from a collection of slices and handles disambiguation if multiple collections are being queried simultaneously */
+ public static void addSlices(Map<String,Slice> target, String collectionName, Collection<Slice> slices, boolean multiCollection) {
+ for (Slice slice : slices) {
+ String key = slice.getName();
+ if (multiCollection) key = collectionName + "_" + key;
+ target.put(key, slice);
}
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java Fri Jan 18 18:30:54 2013
@@ -64,6 +64,7 @@ public class SolrDocument implements Map
/**
* Remove all fields from the document
*/
+ @Override
public void clear()
{
_fields.clear();
@@ -202,6 +203,7 @@ public class SolrDocument implements Map
/**
* Iterate of String->Object keys
*/
+ @Override
public Iterator<Entry<String, Object>> iterator() {
return _fields.entrySet().iterator();
}
@@ -217,23 +219,35 @@ public class SolrDocument implements Map
{
return new Map<String,Collection<Object>>() {
/** Get the field Value */
+ @Override
public Collection<Object> get(Object key) {
return getFieldValues( (String)key );
}
// Easily Supported methods
+ @Override
public boolean containsKey(Object key) { return _fields.containsKey( key ); }
+ @Override
public Set<String> keySet() { return _fields.keySet(); }
+ @Override
public int size() { return _fields.size(); }
+ @Override
public boolean isEmpty() { return _fields.isEmpty(); }
// Unsupported operations. These are not necessary for JSTL
+ @Override
public void clear() { throw new UnsupportedOperationException(); }
+ @Override
public boolean containsValue(Object value) {throw new UnsupportedOperationException();}
+ @Override
public Set<java.util.Map.Entry<String, Collection<Object>>> entrySet() {throw new UnsupportedOperationException();}
+ @Override
public void putAll(Map<? extends String, ? extends Collection<Object>> t) {throw new UnsupportedOperationException();}
+ @Override
public Collection<Collection<Object>> values() {throw new UnsupportedOperationException();}
+ @Override
public Collection<Object> put(String key, Collection<Object> value) {throw new UnsupportedOperationException();}
+ @Override
public Collection<Object> remove(Object key) {throw new UnsupportedOperationException();}
@Override
public String toString() {return _fields.toString();}
@@ -246,23 +260,35 @@ public class SolrDocument implements Map
public Map<String,Object> getFieldValueMap() {
return new Map<String,Object>() {
/** Get the field Value */
+ @Override
public Object get(Object key) {
return getFirstValue( (String)key );
}
// Easily Supported methods
+ @Override
public boolean containsKey(Object key) { return _fields.containsKey( key ); }
+ @Override
public Set<String> keySet() { return _fields.keySet(); }
+ @Override
public int size() { return _fields.size(); }
+ @Override
public boolean isEmpty() { return _fields.isEmpty(); }
// Unsupported operations. These are not necessary for JSTL
+ @Override
public void clear() { throw new UnsupportedOperationException(); }
+ @Override
public boolean containsValue(Object value) {throw new UnsupportedOperationException();}
+ @Override
public Set<java.util.Map.Entry<String, Object>> entrySet() {throw new UnsupportedOperationException();}
+ @Override
public void putAll(Map<? extends String, ? extends Object> t) {throw new UnsupportedOperationException();}
+ @Override
public Collection<Object> values() {throw new UnsupportedOperationException();}
+ @Override
public Collection<Object> put(String key, Object value) {throw new UnsupportedOperationException();}
+ @Override
public Collection<Object> remove(Object key) {throw new UnsupportedOperationException();}
@Override
public String toString() {return _fields.toString();}
@@ -273,46 +299,57 @@ public class SolrDocument implements Map
// MAP interface
//---------------------------------------------------
+ @Override
public boolean containsKey(Object key) {
return _fields.containsKey(key);
}
+ @Override
public boolean containsValue(Object value) {
return _fields.containsValue(value);
}
+ @Override
public Set<Entry<String, Object>> entrySet() {
return _fields.entrySet();
}
//TODO: Shouldn't the input parameter here be a String? The _fields map requires a String.
+ @Override
public Object get(Object key) {
return _fields.get(key);
}
+ @Override
public boolean isEmpty() {
return _fields.isEmpty();
}
+ @Override
public Set<String> keySet() {
return _fields.keySet();
}
+ @Override
public Object put(String key, Object value) {
return _fields.put(key, value);
}
+ @Override
public void putAll(Map<? extends String, ? extends Object> t) {
_fields.putAll( t );
}
+ @Override
public Object remove(Object key) {
return _fields.remove(key);
}
+ @Override
public int size() {
return _fields.size();
}
+ @Override
public Collection<Object> values() {
return _fields.values();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java Fri Jan 18 18:30:54 2013
@@ -48,6 +48,7 @@ public class SolrInputDocument implement
/**
* Remove all fields and boosts from the document
*/
+ @Override
public void clear()
{
if( _fields != null ) {
@@ -166,6 +167,7 @@ public class SolrInputDocument implement
return _fields.get( field );
}
+ @Override
public Iterator<SolrInputField> iterator() {
return _fields.values().iterator();
}
@@ -198,46 +200,57 @@ public class SolrInputDocument implement
// MAP interface
//---------------------------------------------------
+ @Override
public boolean containsKey(Object key) {
return _fields.containsKey(key);
}
+ @Override
public boolean containsValue(Object value) {
return _fields.containsValue(value);
}
+ @Override
public Set<Entry<String, SolrInputField>> entrySet() {
return _fields.entrySet();
}
+ @Override
public SolrInputField get(Object key) {
return _fields.get(key);
}
+ @Override
public boolean isEmpty() {
return _fields.isEmpty();
}
+ @Override
public Set<String> keySet() {
return _fields.keySet();
}
+ @Override
public SolrInputField put(String key, SolrInputField value) {
return _fields.put(key, value);
}
+ @Override
public void putAll(Map<? extends String, ? extends SolrInputField> t) {
_fields.putAll( t );
}
+ @Override
public SolrInputField remove(Object key) {
return _fields.remove(key);
}
+ @Override
public int size() {
return _fields.size();
}
+ @Override
public Collection<SolrInputField> values() {
return _fields.values();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/SolrInputField.java Fri Jan 18 18:30:54 2013
@@ -182,6 +182,7 @@ public class SolrInputField implements I
this.name = name;
}
+ @Override
@SuppressWarnings("unchecked")
public Iterator<Object> iterator() {
if( value instanceof Collection ) {
@@ -190,15 +191,18 @@ public class SolrInputField implements I
return new Iterator<Object>() {
boolean nxt = (value!=null);
+ @Override
public boolean hasNext() {
return nxt;
}
+ @Override
public Object next() {
nxt = false;
return value;
}
+ @Override
public void remove() {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java Fri Jan 18 18:30:54 2013
@@ -18,6 +18,7 @@ package org.apache.solr.common.cloud;
*/
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,7 +31,7 @@ import java.util.Set;
import org.apache.noggit.JSONWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.HashPartitioner.Range;
+import org.apache.solr.common.cloud.DocRouter.Range;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
@@ -39,29 +40,23 @@ import org.slf4j.LoggerFactory;
/**
* Immutable state of the cloud. Normally you can get the state by using
* {@link ZkStateReader#getClusterState()}.
+ * @lucene.experimental
*/
public class ClusterState implements JSONWriter.Writable {
private static Logger log = LoggerFactory.getLogger(ClusterState.class);
private Integer zkClusterStateVersion;
- private final Map<String, Map<String,Slice>> collectionStates; // Map<collectionName, Map<sliceName,Slice>>
+ private final Map<String, DocCollection> collectionStates; // Map<collectionName, Map<sliceName,Slice>>
private final Set<String> liveNodes;
-
- private final HashPartitioner hp = new HashPartitioner();
-
- private final Map<String,RangeInfo> rangeInfos = new HashMap<String,RangeInfo>();
- private final Map<String,Map<String,ZkNodeProps>> leaders = new HashMap<String,Map<String,ZkNodeProps>>();
-
-
/**
* Use this constr when ClusterState is meant for publication.
*
* hashCode and equals will only depend on liveNodes and not clusterStateVersion.
*/
public ClusterState(Set<String> liveNodes,
- Map<String, Map<String,Slice>> collectionStates) {
+ Map<String, DocCollection> collectionStates) {
this(null, liveNodes, collectionStates);
}
@@ -69,87 +64,75 @@ public class ClusterState implements JSO
* Use this constr when ClusterState is meant for consumption.
*/
public ClusterState(Integer zkClusterStateVersion, Set<String> liveNodes,
- Map<String, Map<String,Slice>> collectionStates) {
+ Map<String, DocCollection> collectionStates) {
this.zkClusterStateVersion = zkClusterStateVersion;
this.liveNodes = new HashSet<String>(liveNodes.size());
this.liveNodes.addAll(liveNodes);
- this.collectionStates = new HashMap<String, Map<String,Slice>>(collectionStates.size());
+ this.collectionStates = new HashMap<String, DocCollection>(collectionStates.size());
this.collectionStates.putAll(collectionStates);
- addRangeInfos(collectionStates.keySet());
- getShardLeaders();
}
- private void getShardLeaders() {
- Set<Entry<String,Map<String,Slice>>> collections = collectionStates.entrySet();
- for (Entry<String,Map<String,Slice>> collection : collections) {
- Map<String,Slice> state = collection.getValue();
- Set<Entry<String,Slice>> slices = state.entrySet();
- for (Entry<String,Slice> sliceEntry : slices) {
- Slice slice = sliceEntry.getValue();
- Map<String,Replica> shards = slice.getReplicasMap();
- Set<Entry<String,Replica>> shardsEntries = shards.entrySet();
- for (Entry<String,Replica> shardEntry : shardsEntries) {
- ZkNodeProps props = shardEntry.getValue();
- if (props.containsKey(ZkStateReader.LEADER_PROP)) {
- Map<String,ZkNodeProps> leadersForCollection = leaders.get(collection.getKey());
- if (leadersForCollection == null) {
- leadersForCollection = new HashMap<String,ZkNodeProps>();
- leaders.put(collection.getKey(), leadersForCollection);
- }
- leadersForCollection.put(sliceEntry.getKey(), props);
- break; // we found the leader for this shard
- }
- }
- }
- }
- }
/**
- * Get properties of a shard leader for specific collection.
+ * Get the lead replica for specific collection, or null if one currently doesn't exist.
*/
- public ZkNodeProps getLeader(String collection, String shard) {
- Map<String,ZkNodeProps> collectionLeaders = leaders.get(collection);
- if (collectionLeaders == null) return null;
- return collectionLeaders.get(shard);
+ public Replica getLeader(String collection, String sliceName) {
+ DocCollection coll = collectionStates.get(collection);
+ if (coll == null) return null;
+ Slice slice = coll.getSlice(sliceName);
+ if (slice == null) return null;
+ return slice.getLeader();
}
/**
- * Get shard properties or null if shard is not found.
- */
- public Replica getShardProps(final String collection, final String coreNodeName) {
- Map<String, Slice> slices = getSlices(collection);
- if (slices == null) return null;
- for(Slice slice: slices.values()) {
- if(slice.getReplicasMap().get(coreNodeName)!=null) {
- return slice.getReplicasMap().get(coreNodeName);
- }
- }
- return null;
+ * Gets the replica by the core name (assuming the slice is unknown) or null if replica is not found.
+ * If the slice is known, do not use this method.
+ * coreNodeName is the same as replicaName
+ */
+ public Replica getReplica(final String collection, final String coreNodeName) {
+ return getReplica(collectionStates.get(collection), coreNodeName);
}
- private void addRangeInfos(Set<String> collections) {
- for (String collection : collections) {
- addRangeInfo(collection);
+ private Replica getReplica(DocCollection coll, String replicaName) {
+ if (coll == null) return null;
+ for(Slice slice: coll.getSlices()) {
+ Replica replica = slice.getReplica(replicaName);
+ if (replica != null) return replica;
}
+ return null;
}
+
/**
- * Get the index Slice for collection.
+ * Get the named Slice for collection, or null if not found.
*/
- public Slice getSlice(String collection, String slice) {
- if (collectionStates.containsKey(collection)
- && collectionStates.get(collection).containsKey(slice))
- return collectionStates.get(collection).get(slice);
- return null;
+ public Slice getSlice(String collection, String sliceName) {
+ DocCollection coll = collectionStates.get(collection);
+ if (coll == null) return null;
+ return coll.getSlice(sliceName);
+ }
+
+ public Map<String, Slice> getSlicesMap(String collection) {
+ DocCollection coll = collectionStates.get(collection);
+ if (coll == null) return null;
+ return coll.getSlicesMap();
+ }
+
+ public Collection<Slice> getSlices(String collection) {
+ DocCollection coll = collectionStates.get(collection);
+ if (coll == null) return null;
+ return coll.getSlices();
}
/**
- * Get all slices for collection.
+ * Get the named DocCollection object, or throw an exception if it doesn't exist.
*/
- public Map<String, Slice> getSlices(String collection) {
- if(!collectionStates.containsKey(collection))
- return null;
- return Collections.unmodifiableMap(collectionStates.get(collection));
+ public DocCollection getCollection(String collection) {
+ DocCollection coll = collectionStates.get(collection);
+ if (coll == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
+ }
+ return coll;
}
/**
@@ -162,7 +145,7 @@ public class ClusterState implements JSO
/**
* @return Map<collectionName, Map<sliceName,Slice>>
*/
- public Map<String, Map<String, Slice>> getCollectionStates() {
+ public Map<String, DocCollection> getCollectionStates() {
return Collections.unmodifiableMap(collectionStates);
}
@@ -174,17 +157,14 @@ public class ClusterState implements JSO
}
/**
- * Get shardId for core.
- * @param coreNodeName in the form of nodeName_coreName
+ * Get the slice/shardId for a core.
+ * @param coreNodeName in the form of nodeName_coreName (the name of the replica)
*/
public String getShardId(String coreNodeName) {
- for (Entry<String, Map<String, Slice>> states: collectionStates.entrySet()){
- for(Entry<String, Slice> slices: states.getValue().entrySet()) {
- for(Entry<String, Replica> shards: slices.getValue().getReplicasMap().entrySet()){
- if(coreNodeName.equals(shards.getKey())) {
- return slices.getKey();
- }
- }
+ // System.out.println("###### getShardId("+coreNodeName+") in " + collectionStates);
+ for (DocCollection coll : collectionStates.values()) {
+ for (Slice slice : coll.getSlices()) {
+ if (slice.getReplicasMap().containsKey(coreNodeName)) return slice.getName();
}
}
return null;
@@ -196,56 +176,6 @@ public class ClusterState implements JSO
public boolean liveNodesContain(String name) {
return liveNodes.contains(name);
}
-
- public RangeInfo getRanges(String collection) {
- // TODO: store this in zk
- RangeInfo rangeInfo = rangeInfos.get(collection);
-
- return rangeInfo;
- }
-
- private RangeInfo addRangeInfo(String collection) {
- List<Range> ranges;
- RangeInfo rangeInfo;
- rangeInfo = new RangeInfo();
-
- Map<String,Slice> slices = getSlices(collection);
-
- if (slices == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find collection "
- + collection + " in " + this);
- }
-
- Set<String> shards = slices.keySet();
- ArrayList<String> shardList = new ArrayList<String>(shards.size());
- shardList.addAll(shards);
- Collections.sort(shardList);
-
- ranges = hp.partitionRange(shards.size(), Integer.MIN_VALUE, Integer.MAX_VALUE);
-
- rangeInfo.ranges = ranges;
- rangeInfo.shardList = shardList;
- rangeInfos.put(collection, rangeInfo);
- return rangeInfo;
- }
-
- /**
- * Get shard id for hash. This is used when determining which Slice the
- * document is to be submitted to.
- */
- public String getShard(int hash, String collection) {
- RangeInfo rangInfo = getRanges(collection);
-
- int cnt = 0;
- for (Range range : rangInfo.ranges) {
- if (range.includes(hash)) {
- return rangInfo.shardList.get(cnt);
- }
- cnt++;
- }
-
- throw new IllegalStateException("The HashPartitioner failed");
- }
@Override
public String toString() {
@@ -278,35 +208,60 @@ public class ClusterState implements JSO
* @return the ClusterState
*/
public static ClusterState load(Integer version, byte[] bytes, Set<String> liveNodes) {
+ // System.out.println("######## ClusterState.load:" + (bytes==null ? null : new String(bytes)));
if (bytes == null || bytes.length == 0) {
- return new ClusterState(version, liveNodes, Collections.<String, Map<String,Slice>>emptyMap());
+ return new ClusterState(version, liveNodes, Collections.<String, DocCollection>emptyMap());
}
- // System.out.println("########## Loading ClusterState:" + new String(bytes));
- LinkedHashMap<String, Object> stateMap = (LinkedHashMap<String, Object>) ZkStateReader.fromJSON(bytes);
- HashMap<String,Map<String, Slice>> state = new HashMap<String,Map<String,Slice>>();
-
- for(String collectionName: stateMap.keySet()){
- Map<String, Object> collection = (Map<String, Object>)stateMap.get(collectionName);
- Map<String, Slice> slices = new LinkedHashMap<String,Slice>();
-
- for (Entry<String,Object> sliceEntry : collection.entrySet()) {
- Slice slice = new Slice(sliceEntry.getKey(), null, (Map<String,Object>)sliceEntry.getValue());
- slices.put(slice.getName(), slice);
+ Map<String, Object> stateMap = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
+ Map<String,DocCollection> collections = new LinkedHashMap<String,DocCollection>(stateMap.size());
+ for (Entry<String, Object> entry : stateMap.entrySet()) {
+ String collectionName = entry.getKey();
+ DocCollection coll = collectionFromObjects(collectionName, (Map<String,Object>)entry.getValue());
+ collections.put(collectionName, coll);
+ }
+
+ // System.out.println("######## ClusterState.load result:" + collections);
+ return new ClusterState(version, liveNodes, collections);
+ }
+
+ private static DocCollection collectionFromObjects(String name, Map<String,Object> objs) {
+ Map<String,Object> props;
+ Map<String,Slice> slices;
+
+ Map<String,Object> sliceObjs = (Map<String,Object>)objs.get(DocCollection.SHARDS);
+ if (sliceObjs == null) {
+ // legacy format from 4.0... there was no separate "shards" level to contain the collection shards.
+ slices = makeSlices(objs);
+ props = Collections.emptyMap();
+ } else {
+ slices = makeSlices(sliceObjs);
+ props = new HashMap<String, Object>(objs);
+ objs.remove(DocCollection.SHARDS);
+ }
+
+ DocRouter router = DocRouter.getDocRouter(props.get(DocCollection.DOC_ROUTER));
+ return new DocCollection(name, slices, props, router);
+ }
+
+ private static Map<String,Slice> makeSlices(Map<String,Object> genericSlices) {
+ if (genericSlices == null) return Collections.emptyMap();
+ Map<String,Slice> result = new LinkedHashMap<String, Slice>(genericSlices.size());
+ for (Map.Entry<String,Object> entry : genericSlices.entrySet()) {
+ String name = entry.getKey();
+ Object val = entry.getValue();
+ if (val instanceof Slice) {
+ result.put(name, (Slice)val);
+ } else if (val instanceof Map) {
+ result.put(name, new Slice(name, null, (Map<String,Object>)val));
}
- state.put(collectionName, slices);
}
- return new ClusterState(version, liveNodes, state);
+ return result;
}
@Override
public void write(JSONWriter jsonWriter) {
jsonWriter.write(collectionStates);
}
-
- private class RangeInfo {
- private List<Range> ranges;
- private ArrayList<String> shardList;
- }
/**
* The version of clusterstate.json in ZooKeeper.
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java Fri Jan 18 18:30:54 2013
@@ -66,6 +66,7 @@ class ConnectionManager implements Watch
connected = false;
}
+ @Override
public synchronized void process(WatchedEvent event) {
if (log.isInfoEnabled()) {
log.info("Watcher " + this + " name:" + name + " got event " + event
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java Fri Jan 18 18:30:54 2013
@@ -24,11 +24,12 @@ import java.util.Map;
public class Replica extends ZkNodeProps {
private final String name;
+ private final String nodeName;
public Replica(String name, Map<String,Object> propMap) {
super(propMap);
this.name = name;
- String nodeName = (String)propMap.get(ZkStateReader.NODE_NAME_PROP);
+ nodeName = (String)propMap.get(ZkStateReader.NODE_NAME_PROP);
assert nodeName == null || name.startsWith(nodeName);
}
@@ -36,9 +37,13 @@ public class Replica extends ZkNodeProps
return name;
}
+ /** The name of the node this replica resides on */
+ public String getNodeName() {
+ return nodeName;
+ }
+
@Override
public String toString() {
return name + ':' + JSONUtil.toJSON(propMap, -1); // small enough, keep it on one line (i.e. no indent)
}
-
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Slice.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Slice.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Slice.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/Slice.java Fri Jan 18 18:30:54 2013
@@ -34,8 +34,8 @@ public class Slice extends ZkNodeProps {
public static String LEADER = "leader"; // FUTURE: do we want to record the leader as a slice property in the JSON (as opposed to isLeader as a replica property?)
private final String name;
- private final HashPartitioner.Range range;
- private final Integer replicationFactor;
+ private final DocRouter.Range range;
+ private final Integer replicationFactor; // FUTURE: optional per-slice override of the collection replicationFactor
private final Map<String,Replica> replicas;
private final Replica leader;
@@ -49,15 +49,21 @@ public class Slice extends ZkNodeProps {
this.name = name;
Object rangeObj = propMap.get(RANGE);
- HashPartitioner.Range tmpRange = null;
- if (rangeObj instanceof HashPartitioner.Range) {
- tmpRange = (HashPartitioner.Range)rangeObj;
+ DocRouter.Range tmpRange = null;
+ if (rangeObj instanceof DocRouter.Range) {
+ tmpRange = (DocRouter.Range)rangeObj;
} else if (rangeObj != null) {
- HashPartitioner hp = new HashPartitioner();
- tmpRange = hp.fromString(rangeObj.toString());
+ // Doesn't support custom implementations of Range, but currently not needed.
+ tmpRange = DocRouter.DEFAULT.fromString(rangeObj.toString());
}
range = tmpRange;
+ /** debugging. this isn't an error condition for custom sharding.
+ if (range == null) {
+ System.out.println("###### NO RANGE for " + name + " props=" + props);
+ }
+ **/
+
replicationFactor = null; // future
// add the replicas *after* the other properties (for aesthetics, so it's easy to find slice properties in the JSON output)
@@ -121,6 +127,14 @@ public class Slice extends ZkNodeProps {
return leader;
}
+ public Replica getReplica(String replicaName) {
+ return replicas.get(replicaName);
+ }
+
+ public DocRouter.Range getRange() {
+ return range;
+ }
+
@Override
public String toString() {
return name + ':' + JSONUtil.toJSON(propMap);
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java Fri Jan 18 18:30:54 2013
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.List;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
@@ -70,11 +71,16 @@ public class SolrZkClient {
private volatile SolrZooKeeper keeper;
- private ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor();
+ private ZkCmdExecutor zkCmdExecutor;
private volatile boolean isClosed = false;
private ZkClientConnectionStrategy zkClientConnectionStrategy;
+ private int zkClientTimeout;
+ public int getZkClientTimeout() {
+ return zkClientTimeout;
+ }
+
public SolrZkClient(String zkServerAddress, int zkClientTimeout) {
this(zkServerAddress, zkClientTimeout, new DefaultConnectionStrategy(), null);
}
@@ -91,6 +97,9 @@ public class SolrZkClient {
public SolrZkClient(String zkServerAddress, int zkClientTimeout,
ZkClientConnectionStrategy strat, final OnReconnect onReconnect, int clientConnectTimeout) {
this.zkClientConnectionStrategy = strat;
+ this.zkClientTimeout = zkClientTimeout;
+ // we must retry at least as long as the session timeout
+ zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout);
connManager = new ConnectionManager("ZooKeeperConnection Watcher:"
+ zkServerAddress, this, zkServerAddress, zkClientTimeout, strat, onReconnect);
try {
@@ -461,6 +470,28 @@ public class SolrZkClient {
}
/**
+ * Returns the baseURL corrisponding to a given node's nodeName --
+ * NOTE: does not (currently) imply that the nodeName (or resulting
+ * baseURL) exists in the cluster.
+ * @lucene.experimental
+ */
+ public String getBaseUrlForNodeName(final String nodeName) {
+ final int _offset = nodeName.indexOf("_");
+ if (_offset < 0) {
+ throw new IllegalArgumentException("nodeName does not contain expected '_' seperator: " + nodeName);
+ }
+ final String hostAndPort = nodeName.substring(0,_offset);
+ try {
+ final String path = URLDecoder.decode(nodeName.substring(1+_offset),
+ "UTF-8");
+ return "http://" + hostAndPort + (path.isEmpty() ? "" : ("/" + path));
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalStateException("JVM Does not seem to support UTF-8", e);
+ }
+ }
+
+
+ /**
* Fills string with printout of current ZooKeeper layout.
*/
public void printLayout(String path, int indent, StringBuilder string)
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java Fri Jan 18 18:30:54 2013
@@ -52,6 +52,7 @@ public class SolrZooKeeper extends ZooKe
*/
public void pauseCnxn(final long ms) {
final Thread t = new Thread() {
+ @Override
public void run() {
try {
final ClientCnxn cnxn = getConnection();
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java Fri Jan 18 18:30:54 2013
@@ -27,11 +27,13 @@ import org.apache.zookeeper.data.ACL;
public class ZkCmdExecutor {
- private long retryDelay = 1000L;
- private int retryCount = 15;
+ private long retryDelay = 1300L; // 300 ms over for padding
+ private int retryCount;
private List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE;
- public ZkCmdExecutor() {
+ public ZkCmdExecutor(int timeoutms) {
+ double timeouts = timeoutms / 1000.0;
+ this.retryCount = Math.round(0.5f * ((float)Math.sqrt(8.0f * timeouts + 1.0f) - 1.0f));
}
public List<ACL> getAcl() {
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java Fri Jan 18 18:30:54 2013
@@ -39,6 +39,8 @@ public class ZkNodeProps implements JSON
*/
public ZkNodeProps(Map<String,Object> propMap) {
this.propMap = propMap;
+ // TODO: store an unmodifiable map, but in a way that guarantees not to wrap more than once.
+ // Always wrapping introduces a memory leak.
}
@@ -70,14 +72,14 @@ public class ZkNodeProps implements JSON
* Get property keys.
*/
public Set<String> keySet() {
- return Collections.unmodifiableSet(propMap.keySet());
+ return propMap.keySet();
}
/**
* Get all properties as map.
*/
public Map<String, Object> getProperties() {
- return Collections.unmodifiableMap(propMap);
+ return propMap;
}
/** Returns a shallow writable copy of the properties */
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Fri Jan 18 18:30:54 2013
@@ -127,18 +127,21 @@ public class ZkStateReader {
private boolean closeClient = false;
- private ZkCmdExecutor cmdExecutor = new ZkCmdExecutor();
+ private ZkCmdExecutor cmdExecutor;
public ZkStateReader(SolrZkClient zkClient) {
this.zkClient = zkClient;
+ initZkCmdExecutor(zkClient.getZkClientTimeout());
}
-
+
public ZkStateReader(String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout) throws InterruptedException, TimeoutException, IOException {
closeClient = true;
+ initZkCmdExecutor(zkClientTimeout);
zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout, zkClientConnectTimeout,
// on reconnect, reload cloud info
new OnReconnect() {
+ @Override
public void command() {
try {
ZkStateReader.this.createClusterStateWatchersAndUpdate();
@@ -158,6 +161,11 @@ public class ZkStateReader {
});
}
+ private void initZkCmdExecutor(int zkClientTimeout) {
+ // we must retry at least as long as the session timeout
+ cmdExecutor = new ZkCmdExecutor(zkClientTimeout);
+ }
+
// load and publish a new CollectionInfo
public void updateClusterState(boolean immediate) throws KeeperException, InterruptedException {
updateClusterState(immediate, false);
@@ -186,7 +194,7 @@ public class ZkStateReader {
if (EventType.None.equals(event.getType())) {
return;
}
- log.info("A cluster state change has occurred - updating... ({})", ZkStateReader.this.clusterState == null ? 0 : ZkStateReader.this.clusterState.getLiveNodes().size());
+ log.info("A cluster state change: {}, has occurred - updating... (live nodes size: {})", (event) , ZkStateReader.this.clusterState == null ? 0 : ZkStateReader.this.clusterState.getLiveNodes().size());
try {
// delayed approach
@@ -317,6 +325,7 @@ public class ZkStateReader {
clusterStateUpdateScheduled = true;
updateCloudExecutor.schedule(new Runnable() {
+ @Override
public void run() {
log.info("Updating cluster state from ZooKeeper...");
synchronized (getUpdateLock()) {
@@ -391,25 +400,28 @@ public class ZkStateReader {
public String getLeaderUrl(String collection, String shard, int timeout)
throws InterruptedException, KeeperException {
- ZkCoreNodeProps props = new ZkCoreNodeProps(getLeaderProps(collection,
+ ZkCoreNodeProps props = new ZkCoreNodeProps(getLeaderRetry(collection,
shard, timeout));
return props.getCoreUrl();
}
/**
- * Get shard leader properties.
+ * Get shard leader properties, with retry if none exist.
*/
- public ZkNodeProps getLeaderProps(String collection, String shard) throws InterruptedException {
- return getLeaderProps(collection, shard, 1000);
+ public Replica getLeaderRetry(String collection, String shard) throws InterruptedException {
+ return getLeaderRetry(collection, shard, 1000);
}
-
- public ZkNodeProps getLeaderProps(String collection, String shard, int timeout) throws InterruptedException {
+
+ /**
+ * Get shard leader properties, with retry if none exist.
+ */
+ public Replica getLeaderRetry(String collection, String shard, int timeout) throws InterruptedException {
long timeoutAt = System.currentTimeMillis() + timeout;
while (System.currentTimeMillis() < timeoutAt) {
if (clusterState != null) {
- final ZkNodeProps nodeProps = clusterState.getLeader(collection, shard);
- if (nodeProps != null && getClusterState().liveNodesContain((String) nodeProps.get(ZkStateReader.NODE_NAME_PROP))) {
- return nodeProps;
+ Replica replica = clusterState.getLeader(collection, shard);
+ if (replica != null && getClusterState().liveNodesContain(replica.getNodeName())) {
+ return replica;
}
}
Thread.sleep(50);
@@ -450,7 +462,7 @@ public class ZkStateReader {
if (clusterState == null) {
return null;
}
- Map<String,Slice> slices = clusterState.getSlices(collection);
+ Map<String,Slice> slices = clusterState.getSlicesMap(collection);
if (slices == null) {
throw new ZooKeeperException(ErrorCode.BAD_REQUEST,
"Could not find collection in zk: " + collection + " "
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java Fri Jan 18 18:30:54 2013
@@ -28,6 +28,9 @@ public interface CoreAdminParams
/** What Core are we talking about **/
public final static String CORE = "core";
+ /** Should the STATUS request include index info **/
+ public final static String INDEX_INFO = "indexInfo";
+
/** Persistent -- should it save the cores state? **/
public final static String PERSISTENT = "persistent";
@@ -36,6 +39,8 @@ public interface CoreAdminParams
/** If you rename something, what is the new name **/
public final static String DATA_DIR = "dataDir";
+
+ public final static String ULOG_DIR = "ulogDir";
/** Name of the other core in actions involving 2 cores **/
public final static String OTHER = "other";
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/MoreLikeThisParams.java Fri Jan 18 18:30:54 2013
@@ -31,6 +31,7 @@ public interface MoreLikeThisParams
public final static String SIMILARITY_FIELDS = PREFIX + "fl";
public final static String MIN_TERM_FREQ = PREFIX + "mintf";
+ public final static String MAX_DOC_FREQ = PREFIX + "maxdf";
public final static String MIN_DOC_FREQ = PREFIX + "mindf";
public final static String MIN_WORD_LEN = PREFIX + "minwl";
public final static String MAX_WORD_LEN = PREFIX + "maxwl";
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java Fri Jan 18 18:30:54 2013
@@ -45,4 +45,7 @@ public interface ShardParams {
/** Should things fail if there is an error? (true/false) */
public static final String SHARDS_TOLERANT = "shards.tolerant";
+
+ /** Should things fail if there is an error? (true/false) */
+ public static final String SHARD_KEYS = "shard.keys";
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java Fri Jan 18 18:30:54 2013
@@ -186,6 +186,7 @@ public abstract class ContentStreamBase
* Base reader implementation. If the contentType declares a
* charset use it, otherwise use "utf-8".
*/
+ @Override
public Reader getReader() throws IOException {
String charset = getCharsetFromContentType( getContentType() );
return charset == null
@@ -197,6 +198,7 @@ public abstract class ContentStreamBase
// Getters / Setters for overrideable attributes
//------------------------------------------------------------------
+ @Override
public String getContentType() {
return contentType;
}
@@ -205,6 +207,7 @@ public abstract class ContentStreamBase
this.contentType = contentType;
}
+ @Override
public String getName() {
return name;
}
@@ -213,6 +216,7 @@ public abstract class ContentStreamBase
this.name = name;
}
+ @Override
public Long getSize() {
return size;
}
@@ -221,6 +225,7 @@ public abstract class ContentStreamBase
this.size = size;
}
+ @Override
public String getSourceInfo() {
return sourceInfo;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java Fri Jan 18 18:30:54 2013
@@ -51,7 +51,7 @@ public class ExecutorUtil {
while (!shutdown) {
try {
// Wait a while for existing tasks to terminate
- shutdown = pool.awaitTermination(60, TimeUnit.SECONDS);
+ shutdown = pool.awaitTermination(30, TimeUnit.SECONDS);
} catch (InterruptedException ie) {
// Preserve interrupt status
Thread.currentThread().interrupt();