You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/05 22:05:03 UTC
svn commit: r1555629 [6/7] - in /lucene/dev/branches/lucene5376: ./
dev-tools/ dev-tools/maven/solr/contrib/map-reduce/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/
lucene/analysis/com...
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java Sun Jan 5 21:04:58 2014
@@ -110,8 +110,8 @@ public class TestFacets extends ServerBa
put(o, "grouping", s);
}
- put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
- put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
+ put(o, "facets", "[{dim: dateFacet, topN: 10}]");
+ put(o, "retrieveFields", "[id, date, price, {field: body, highlight: " + (snippets ? "snippets" : "whole") + "}]");
return send("search", o);
}
@@ -188,19 +188,19 @@ public class TestFacets extends ServerBa
long indexGen = getLong(send("addDocument", "{fields: {author: Tom}}"), "indexGen");
// Initial query:
- JSONObject o = send("search", String.format(Locale.ROOT, "{query: MatchAllDocsQuery, facets: [{path: [author], topN: 10}], searcher: {indexGen: %d}}", indexGen));
+ JSONObject o = send("search", String.format(Locale.ROOT, "{query: MatchAllDocsQuery, facets: [{dim: author, topN: 10}], searcher: {indexGen: %d}}", indexGen));
assertEquals(6, o.get("totalHits"));
- assertEquals("[[\"top\",0],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
+ assertEquals("[[\"top\",6],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
// Now, single drill down:
- o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, values: [Bob]}], query: MatchAllDocsQuery, facets: [{path: [author], topN: 10}], searcher: {indexGen: %d}}", indexGen));
+ o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, value: Bob}], query: MatchAllDocsQuery, facets: [{dim: author, topN: 10}], searcher: {indexGen: %d}}", indexGen));
assertEquals(1, o.get("totalHits"));
- assertEquals("[[\"top\",0],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
+ assertEquals("[[\"top\",6],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
- // Multi drill down:
- o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, values: [Bob, Lisa]}], query: MatchAllDocsQuery, facets: [{path: [author], topN: 10}], searcher: {indexGen: %d}}", indexGen));
+ // Multi (OR'd) drill down:
+ o = send("search", String.format(Locale.ROOT, "{drillDowns: [{field: author, value: Bob}, {field: author, value: Lisa}], query: MatchAllDocsQuery, facets: [{dim: author, topN: 10}], searcher: {indexGen: %d}}", indexGen));
assertEquals(3, o.get("totalHits"));
- assertEquals("[[\"top\",0],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
+ assertEquals("[[\"top\",6],[\"Tom\",3],[\"Lisa\",2],[\"Bob\",1]]", getArray(o, "facets[0].counts").toString());
}
public void testRangeFacets() throws Exception {
@@ -209,7 +209,7 @@ public class TestFacets extends ServerBa
for(int i=0;i<100;i++) {
gen = getLong(send("addDocument", "{fields: {longField: " + i + "}}"), "indexGen");
}
- JSONObject o = send("search", "{facets: [{path: longField, numericRanges: [{label: All, min: 0, max: 99, minInclusive: true, maxInclusive: true}, {label: Half, min: 0, max: 49, minInclusive: true, maxInclusive: true}]}], searcher: {indexGen: " + gen + "}}");
+ JSONObject o = send("search", "{facets: [{dim: longField, numericRanges: [{label: All, min: 0, max: 99, minInclusive: true, maxInclusive: true}, {label: Half, min: 0, max: 49, minInclusive: true, maxInclusive: true}]}], searcher: {indexGen: " + gen + "}}");
assertEquals("All", getString(o, "facets[0].counts[1][0]"));
assertEquals(100, getInt(o, "facets[0].counts[1][1]"));
assertEquals("Half", getString(o, "facets[0].counts[2][0]"));
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java Sun Jan 5 21:04:58 2014
@@ -199,7 +199,9 @@ public class TestGrouping extends Server
if (sortField != null) {
JSONObject sort = new JSONObject();
o.put("sort", sort);
- sort.put("doDocScores", true);
+ if (groupField == null) {
+ sort.put("doDocScores", true);
+ }
JSONArray sortFields = new JSONArray();
sort.put("fields", sortFields);
@@ -220,7 +222,7 @@ public class TestGrouping extends Server
put(o, "grouping", s);
}
- put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
+ put(o, "facets", "[{dim: 'dateFacet', topN: 10}]");
put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
return send("search", o);
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java Sun Jan 5 21:04:58 2014
@@ -116,7 +116,7 @@ public class TestHighlight extends Serve
put(o, "grouping", s);
}
- put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
+ put(o, "facets", "[{dim: 'dateFacet', topN: 10}]");
put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
return send("search", o);
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java Sun Jan 5 21:04:58 2014
@@ -132,11 +132,7 @@ public class TestIndexing extends Server
sb.append(',');
}
if (i == 57) {
- JSONArray broken = new JSONArray();
- broken.add("2013");
- broken.add("");
- broken.add("17");
- o.put("dateFacet", broken);
+ o.put("foobar", 17);
}
JSONObject o2 = new JSONObject();
o2.put("fields", o);
@@ -188,8 +184,8 @@ public class TestIndexing extends Server
put(o, "grouping", s);
}
- put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
- put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
+ put(o, "facets", "[{dim: dateFacet, topN: 10}]");
+ put(o, "retrieveFields", "[id, date, price, {field: body, highlight: " + (snippets ? "snippets" : "whole") + "}]");
return send("search", o);
}
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java Sun Jan 5 21:04:58 2014
@@ -25,6 +25,7 @@ public class TestNumericFields extends S
@BeforeClass
public static void initClass() throws Exception {
+ curIndexName = "index";
startServer();
createAndStartIndex();
registerFields();
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java Sun Jan 5 21:04:58 2014
@@ -167,6 +167,7 @@ public class TestSearch extends ServerBa
public void testNumericRangeQuery() throws Exception {
curIndexName = "nrq";
for(String type : new String[] {"int", "long", "float", "double"}) {
+ _TestUtil.rmDir(new File("nrq"));
send("createIndex", "{rootDir: nrq}");
send("startIndex", "{}");
send("registerFields", String.format(Locale.ROOT, "{fields: {nf: {type: %s, index: true}}}", type));
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java Sun Jan 5 21:04:58 2014
@@ -29,6 +29,7 @@ public class TestServer extends ServerBa
@BeforeClass
public static void initClass() throws Exception {
+ curIndexName = "index";
startServer();
createAndStartIndex();
registerFields();
@@ -118,8 +119,8 @@ public class TestServer extends ServerBa
put(o, "grouping", s);
}
- put(o, "facets", "[{path: 'dateFacet', topN: 10}]");
- put(o, "retrieveFields", "['id', 'date', 'price', {field: 'body', highlight: " + (snippets ? "snippets" : "whole") + "}]");
+ put(o, "facets", "[{dim: dateFacet, topN: 10}]");
+ put(o, "retrieveFields", "[id, date, price, {field: body, highlight: " + (snippets ? "snippets" : "whole") + "}]");
return send("search", o);
}
Modified: lucene/dev/branches/lucene5376/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/CHANGES.txt?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5376/solr/CHANGES.txt Sun Jan 5 21:04:58 2014
@@ -83,7 +83,7 @@ Detailed Change List
New Features
----------------------
-* SOLR-5308: A new 'migrate' collection API to split all documents with a
+* SOLR-5308: SOLR-5601: A new 'migrate' collection API to split all documents with a
route key into another collection (shalin)
* SOLR-5441: Expose number of transaction log files and their size via JMX.
@@ -132,18 +132,16 @@ New Features
* SOLR-5556: Allow class of CollectionsHandler and InfoHandler to be specified
in solr.xml. (Gregory Chanan, Alan Woodward)
+* SOLR-5581: Give ZkCLI the ability to get files. (Gregory Chanan via Mark Miller)
+
+* SOLR-5536: Add ValueSource collapse criteria to CollapsingQParsingPlugin (Joel Bernstein)
+
Bug Fixes
----------------------
* SOLR-5438: DebugComponent throws NPE when used with grouping.
(Tomás Fernández Löbbe via shalin)
-* SOLR-5442: Python client cannot parse proxied response when served by Tomcat.
- (Patrick Hunt, Gregory Chanan, Vamsee Yarlagadda, Romain Rigaux, Mark Miller)
-
-* SOLR-5445: Proxied responses should propagate all headers rather than the
- first one for each key. (Patrick Hunt, Mark Miller)
-
* SOLR-4612: Admin UI - Analysis Screen contains empty table-columns (steffkes)
* SOLR-5451: SyncStrategy closes it's http connection manager before the
@@ -155,15 +153,9 @@ Bug Fixes
* SOLR-5461: Request proxying should only set con.setDoOutput(true) if the
request is a post. (Mark Miller)
-* SOLR-5479: SolrCmdDistributor retry logic stops if a leader for the request
- cannot be found in 1 second. (Mark Miller)
-
* SOLR-5481: SolrCmdDistributor should not let the http client do it's own
retries. (Mark Miller)
-* SOLR-4709: The core reload after replication if config files have changed
- can fail due to a race condition. (Mark Miller, Hossman))
-
* LUCENE-5347: Fixed Solr's Zookeeper Client to copy files to Zookeeper using
binary transfer. Previously data was read with default encoding and stored
in zookeeper as UTF-8. This bug was found after upgrading to forbidden-apis
@@ -173,13 +165,6 @@ Bug Fixes
a delta-import is run first before any full-imports.
(Sebastien Lorber, Arcadius Ahouansou via shalin)
-* SOLR-5496: We should share an http connection manager across non search
- HttpClients and ensure all http connection managers get shutdown.
- (Mark Miller)
-
-* SOLR-5503: Retry 'forward to leader' requests less aggressively - rather
- than on IOException and status 500, ConnectException. (Mark Miller)
-
* SOLR-5494: CoreContainer#remove throws NPE rather than returning null when
a SolrCore does not exist in core discovery mode. (Mark Miller)
@@ -200,20 +185,19 @@ Bug Fixes
* SOLR-5540: HdfsLockFactory should explicitly create the lock parent directory
if necessary. (Mark Miller)
-* SOLR-5532: SolrJ Content-Type validation is too strict for some
- webcontainers / proxies. (Jakob Furrer, hossman, Shawn Heisey, Uwe Schindler,
- Mark Miller)
-
-* SOLR-5547: Creating a collection alias using SolrJ's CollectionAdminRequest
- sets the alias name and the collections to alias to the same value.
- (Aaron Schram, Mark Miller)
-
* SOLR-5543: Core swaps resulted in duplicate core entries in solr.xml when
using solr.xml persistence. (Bill Bell, Alan Woodward)
* SOLR-5562: ConcurrentUpdateSolrServer constructor ignores supplied httpclient.
(Kyle Halliday via Mark Miller)
-
+
+* SOLR-5567: ZkController getHostAddress duplicates url prefix.
+ (Kyle Halliday, Alexey Serba, shalin)
+
+* SOLR-5577: Likely ZooKeeper expiration should not slow down updates a given
+ amount, but instead cut off updates after a given time.
+ (Mark Miller, Christine Poerschke)
+
Optimizations
----------------------
@@ -227,8 +211,11 @@ Optimizations
* SOLR-5512: Optimize DocValuesFacets. (Robert Muir)
-* SOLR-5576: Improve concurrency when registering and waiting for all
- SolrCore's to register a DOWN state. (Christine Poerschke via Mark Miller)
+* SOLR-2960: fix DIH XPathEntityProcessor to add the correct number of "null"
+ placeholders for multi-valued fields (Michael Watts via James Dyer)
+
+* SOLR-5214: Reduce memory usage for shard splitting by merging segments one
+ at a time. (Christine Poerschke via shalin)
Other Changes
---------------------
@@ -271,6 +258,12 @@ Other Changes
* SOLR-5574: CoreContainer shutdown publishes all nodes as down and waits to
see that and then again publishes all nodes as down. (Mark Miller)
+* SOLR-5590: Upgrade HttpClient/HttpComponents to 4.3.x.
+ (Karl Wright via Shawn Heisey)
+
+* pull request #11: change the default of hl.phraseLimit to 5000.
+ (Michael Della Bitta via Robert Muir)
+
================== 4.6.1 ==================
Versions of Major Components
@@ -292,7 +285,68 @@ Bug Fixes
* SOLR-5416: CollapsingQParserPlugin breaks Tag/Exclude Faceting (David Boychuck, Joel Bernstein)
+* SOLR-5442: Python client cannot parse proxied response when served by Tomcat.
+ (Patrick Hunt, Gregory Chanan, Vamsee Yarlagadda, Romain Rigaux, Mark Miller)
+
+* SOLR-5445: Proxied responses should propagate all headers rather than the
+ first one for each key. (Patrick Hunt, Mark Miller)
+
+* SOLR-5479: SolrCmdDistributor retry logic stops if a leader for the request
+ cannot be found in 1 second. (Mark Miller)
+
+* SOLR-5532: SolrJ Content-Type validation is too strict for some
+ webcontainers / proxies. (Jakob Furrer, hossman, Shawn Heisey, Uwe Schindler,
+ Mark Miller)
+
+* SOLR-5547: Creating a collection alias using SolrJ's CollectionAdminRequest
+ sets the alias name and the collections to alias to the same value.
+ (Aaron Schram, Mark Miller)
+
+* SOLR-5580: NPE when creating a core with both explicit shard and coreNodeName.
+ (YouPeng Yang, Mark Miller)
+
+* SOLR-5552: Leader recovery process can select the wrong leader if all replicas
+ for a shard are down and trying to recover as well as lose updates that should
+ have been recovered. (Timothy Potter, Mark Miller)
+
+* SOLR-5569 A replica should not try and recover from a leader until it has
+ published that it is ACTIVE. (Mark Miller)
+
+* SOLR-5568 A SolrCore cannot decide to be the leader just because the cluster
+ state says no other SolrCore's are active. (Mark Miller)
+
+* SOLR-5496: We should share an http connection manager across non search
+ HttpClients and ensure all http connection managers get shutdown.
+ (Mark Miller)
+
+* SOLR-5583: ConcurrentUpdateSolrServer#blockUntilFinished may wait forever if
+ the executor service is shutdown. (Mark Miller)
+
+* SOLR-5586: All ZkCmdExecutor's should be initialized with the zk client
+ timeout. (Mark Miller)
+
+* SOLR-5587: ElectionContext implementations should use
+ ZkCmdExecutor#ensureExists to ensure their election paths are properly
+ created. (Mark Miller)
+
+* SOLR-4709: The core reload after replication if config files have changed
+ can fail due to a race condition. (Mark Miller, Hossman))
+
+* SOLR-5503: Retry 'forward to leader' requests less aggressively - rather
+ than on IOException and status 500, ConnectException. (Mark Miller)
+
+* SOLR-5588: PeerSync doesn't count all connect failures as success.
+ (Mark Miller)
+
+* SOLR-5564: hl.maxAlternateFieldLength should apply to original field when
+ fallback is attempted (janhoy)
+
+Optimizations
+----------------------
+* SOLR-5576: Improve concurrency when registering and waiting for all
+ SolrCore's to register a DOWN state. (Christine Poerschke via Mark Miller)
+
================== 4.6.0 ==================
Versions of Major Components
Modified: lucene/dev/branches/lucene5376/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/build.xml?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/build.xml (original)
+++ lucene/dev/branches/lucene5376/solr/build.xml Sun Jan 5 21:04:58 2014
@@ -612,7 +612,7 @@
<property name="analysis-common.res.dir" value="../lucene/analysis/common/src/resources/org/apache/lucene/analysis"/>
<property name="analysis-kuromoji.res.dir" value="../lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis"/>
- <property name="analysis.conf.dest" value="${example}/solr/conf/lang"/>
+ <property name="analysis.conf.dest" value="${example}/solr/collection1/conf/lang"/>
<target name="sync-analyzers"
description="Committers' Helper: synchronizes analysis resources (e.g. stoplists) to the example">
@@ -625,6 +625,9 @@
<!-- catalan -->
<copy verbose="true" file="${analysis-common.res.dir}/ca/stopwords.txt"
tofile="${analysis.conf.dest}/stopwords_ca.txt"/>
+ <!-- kurdish -->
+ <copy verbose="true" file="${analysis-common.res.dir}/ckb/stopwords.txt"
+ tofile="${analysis.conf.dest}/stopwords_ckb.txt"/>
<!-- czech -->
<copy verbose="true" file="${analysis-common.res.dir}/cz/stopwords.txt"
tofile="${analysis.conf.dest}/stopwords_cz.txt"/>
Modified: lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java Sun Jan 5 21:04:58 2014
@@ -296,7 +296,7 @@ public class XPathRecordReader {
for (Node n : childNodes) {
// For the multivalue child nodes where we could have, but
// didnt, collect text. Push a null string into values.
- if (!childrenFound.contains(n)) n.putNulls(values);
+ if (!childrenFound.contains(n)) n.putNulls(values, valuesAddedinThisFrame);
}
}
return;
@@ -429,18 +429,28 @@ public class XPathRecordReader {
* pushing a null string onto every multiValued fieldName's List of values
* where a value has not been provided from the stream.
*/
- private void putNulls(Map<String, Object> values) {
+ private void putNulls(Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
if (attributes != null) {
for (Node n : attributes) {
- if (n.multiValued)
- putText(values, null, n.fieldName, true);
+ if (n.multiValued) {
+ putANull(n.fieldName, values, valuesAddedinThisFrame);
+ }
}
}
- if (hasText && multiValued)
- putText(values, null, fieldName, true);
+ if (hasText && multiValued) {
+ putANull(fieldName, values, valuesAddedinThisFrame);
+ }
if (childNodes != null) {
- for (Node childNode : childNodes)
- childNode.putNulls(values);
+ for (Node childNode : childNodes) {
+ childNode.putNulls(values, valuesAddedinThisFrame);
+ }
+ }
+ }
+
+ private void putANull(String thisFieldName, Map<String, Object> values, Set<String> valuesAddedinThisFrame) {
+ putText(values, null, thisFieldName, true);
+ if( valuesAddedinThisFrame != null) {
+ valuesAddedinThisFrame.add(thisFieldName);
}
}
Modified: lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java Sun Jan 5 21:04:58 2014
@@ -93,6 +93,128 @@ public class TestXPathEntityProcessor ex
assertEquals("2", l.get(1));
assertEquals("ü", l.get(2));
}
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ @Test
+ public void testMultiValuedWithMultipleDocuments() throws Exception {
+ Map entityAttrs = createMap("name", "e", "url", "testdata.xml", XPathEntityProcessor.FOR_EACH, "/documents/doc");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "id", "xpath", "/documents/doc/id", DataImporter.MULTI_VALUED, "false"));
+ fields.add(createMap("column", "a", "xpath", "/documents/doc/a", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataA", "xpath", "/documents/doc/sec1/s1dataA", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataB", "xpath", "/documents/doc/sec1/s1dataB", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataC", "xpath", "/documents/doc/sec1/s1dataC", DataImporter.MULTI_VALUED, "true"));
+
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(textMultipleDocuments), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ {
+ assertEquals("1", result.get(0).get("id"));
+ List a = (List)result.get(0).get("a");
+ List s1dataA = (List)result.get(0).get("s1dataA");
+ List s1dataB = (List)result.get(0).get("s1dataB");
+ List s1dataC = (List)result.get(0).get("s1dataC");
+ assertEquals(2, a.size());
+ assertEquals("id1-a1", a.get(0));
+ assertEquals("id1-a2", a.get(1));
+ assertEquals(3, s1dataA.size());
+ assertEquals("id1-s1dataA-1", s1dataA.get(0));
+ assertNull(s1dataA.get(1));
+ assertEquals("id1-s1dataA-3", s1dataA.get(2));
+ assertEquals(3, s1dataB.size());
+ assertEquals("id1-s1dataB-1", s1dataB.get(0));
+ assertEquals("id1-s1dataB-2", s1dataB.get(1));
+ assertEquals("id1-s1dataB-3", s1dataB.get(2));
+ assertEquals(3, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ assertNull(s1dataC.get(1));
+ assertNull(s1dataC.get(2));
+ }
+ {
+ assertEquals("2", result.get(1).get("id"));
+ List a = (List)result.get(1).get("a");
+ List s1dataA = (List)result.get(1).get("s1dataA");
+ List s1dataB = (List)result.get(1).get("s1dataB");
+ List s1dataC = (List)result.get(1).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertNull(s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertEquals("id2-s1dataB-1", s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ }
+ {
+ assertEquals("3", result.get(2).get("id"));
+ List a = (List)result.get(2).get("a");
+ List s1dataA = (List)result.get(2).get("s1dataA");
+ List s1dataB = (List)result.get(2).get("s1dataB");
+ List s1dataC = (List)result.get(2).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertEquals("id3-s1dataA-1", s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertNull(s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ }
+ {
+ assertEquals("4", result.get(3).get("id"));
+ List a = (List)result.get(3).get("a");
+ List s1dataA = (List)result.get(3).get("s1dataA");
+ List s1dataB = (List)result.get(3).get("s1dataB");
+ List s1dataC = (List)result.get(3).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertEquals("id4-s1dataA-1", s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertEquals("id4-s1dataB-1", s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertEquals("id4-s1dataC-1", s1dataC.get(0));
+ }
+ {
+ assertEquals("5", result.get(4).get("id"));
+ List a = (List)result.get(4).get("a");
+ List s1dataA = (List)result.get(4).get("s1dataA");
+ List s1dataB = (List)result.get(4).get("s1dataB");
+ List s1dataC = (List)result.get(4).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertNull(s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertNull(s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertEquals("id5-s1dataC-1", s1dataC.get(0));
+ }
+ {
+ assertEquals("6", result.get(5).get("id"));
+ List a = (List)result.get(5).get("a");
+ List s1dataA = (List)result.get(5).get("s1dataA");
+ List s1dataB = (List)result.get(5).get("s1dataB");
+ List s1dataC = (List)result.get(5).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(3, s1dataA.size());
+ assertEquals("id6-s1dataA-1", s1dataA.get(0));
+ assertEquals("id6-s1dataA-2", s1dataA.get(1));
+ assertNull(s1dataA.get(2));
+ assertEquals(3, s1dataB.size());
+ assertEquals("id6-s1dataB-1", s1dataB.get(0));
+ assertEquals("id6-s1dataB-2", s1dataB.get(1));
+ assertEquals("id6-s1dataB-3", s1dataB.get(2));
+ assertEquals(3, s1dataC.size());
+ assertEquals("id6-s1dataC-1", s1dataC.get(0));
+ assertNull(s1dataC.get(1));
+ assertEquals("id6-s1dataC-3", s1dataC.get(2));
+ }
+ }
@Test
public void testMultiValuedFlatten() throws Exception {
@@ -305,4 +427,68 @@ public class TestXPathEntityProcessor ex
private static final String testXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE root [\n<!ENTITY uuml \"ü\" >\n]>\n<root><a>1</a><a>2</a><a>ü</a></root>";
private static final String testXmlFlatten = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root><a>1<b>B</b>2</a></root>";
+
+ private static final String textMultipleDocuments =
+ "<?xml version=\"1.0\" ?>" +
+ "<documents>" +
+ " <doc>" +
+ " <id>1</id>" +
+ " <a>id1-a1</a>" +
+ " <a>id1-a2</a>" +
+ " <sec1>" +
+ " <s1dataA>id1-s1dataA-1</s1dataA>" +
+ " <s1dataB>id1-s1dataB-1</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataB>id1-s1dataB-2</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataA>id1-s1dataA-3</s1dataA>" +
+ " <s1dataB>id1-s1dataB-3</s1dataB>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>2</id>" +
+ " <sec1>" +
+ " <s1dataB>id2-s1dataB-1</s1dataB>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>3</id>" +
+ " <sec1>" +
+ " <s1dataA>id3-s1dataA-1</s1dataA>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>4</id>" +
+ " <sec1>" +
+ " <s1dataA>id4-s1dataA-1</s1dataA>" +
+ " <s1dataB>id4-s1dataB-1</s1dataB>" +
+ " <s1dataC>id4-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>5</id>" +
+ " <sec1>" +
+ " <s1dataC>id5-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>6</id>" +
+ " <sec1>" +
+ " <s1dataA>id6-s1dataA-1</s1dataA>" +
+ " <s1dataB>id6-s1dataB-1</s1dataB>" +
+ " <s1dataC>id6-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataA>id6-s1dataA-2</s1dataA>" +
+ " <s1dataB>id6-s1dataB-2</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataB>id6-s1dataB-3</s1dataB>" +
+ " <s1dataC>id6-s1dataC-3</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ "</documents>"
+ ;
}
Modified: lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java Sun Jan 5 21:04:58 2014
@@ -17,9 +17,19 @@ package org.apache.solr.handler.dataimpo
* limitations under the License.
*/
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
import org.apache.solr.cloud.AbstractZkTestCase;
import org.apache.solr.cloud.ZkTestServer;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.junit.After;
@@ -29,20 +39,13 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
protected static ZkTestServer zkServer;
protected static String zkDir;
+ private static CoreContainer cc;
+
private String dateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
@BeforeClass
@@ -61,13 +64,12 @@ public class TestZKPropertiesWriter exte
"dataimport-solrconfig.xml", "dataimport-schema.xml");
//initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
- createDefaultCoreContainer(getFile("dih/solr").getAbsolutePath());
+ cc = createDefaultCoreContainer(getFile("dih/solr").getAbsolutePath());
}
@Before
public void beforeDihZKTest() throws Exception {
- clearIndex();
- assertU(commit());
+
}
@After
@@ -78,13 +80,12 @@ public class TestZKPropertiesWriter exte
@AfterClass
public static void dihZk_afterClass() throws Exception {
+ cc.shutdown();
+
zkServer.shutdown();
zkServer = null;
zkDir = null;
-
- // wait just a bit for any zk client threads to outlast timeout
- Thread.sleep(2000);
}
@Test
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java Sun Jan 5 21:04:58 2014
@@ -56,7 +56,7 @@ public class DistributedQueue {
public DistributedQueue(SolrZkClient zookeeper, String dir, List<ACL> acl) {
this.dir = dir;
- ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(30);
+ ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zookeeper.getZkClientTimeout());
try {
cmdExecutor.ensureExists(dir, zookeeper);
} catch (KeeperException e) {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Sun Jan 5 21:04:58 2014
@@ -10,6 +10,7 @@ import org.apache.solr.common.cloud.Clus
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -93,6 +94,15 @@ class ShardLeaderElectionContextBase ext
this.zkClient = zkStateReader.getZkClient();
this.shardId = shardId;
this.collection = collection;
+
+ try {
+ new ZkCmdExecutor(zkStateReader.getZkClient().getZkClientTimeout()).ensureExists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection, zkClient);
+ } catch (KeeperException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
}
@Override
@@ -424,6 +434,14 @@ final class OverseerElectionContext exte
super(zkNodeName, "/overseer_elect", "/overseer_elect/leader", null, zkClient);
this.overseer = overseer;
this.zkClient = zkClient;
+ try {
+ new ZkCmdExecutor(zkClient.getZkClientTimeout()).ensureExists("/overseer_elect", zkClient);
+ } catch (KeeperException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
}
@Override
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Sun Jan 5 21:04:58 2014
@@ -69,7 +69,7 @@ public class LeaderElector {
public LeaderElector(SolrZkClient zkClient) {
this.zkClient = zkClient;
- zkCmdExecutor = new ZkCmdExecutor((int) (zkClient.getZkClientTimeout()/1000.0 + 3000));
+ zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout());
}
// for tests
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java Sun Jan 5 21:04:58 2014
@@ -442,18 +442,10 @@ public class Overseer {
// if coreNodeName is null, auto assign one
coreNodeName = Assign.assignNode(collection, state);
}
- message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
- } else {
- //probably, this core was removed explicitly
- if (sliceName !=null && collectionExists && !"true".equals(state.getCollection(collection).getStr("autoCreated"))) {
- Slice slice = state.getSlice(collection, sliceName);
- if (slice.getReplica(coreNodeName) == null) {
- log.info("core_deleted . Just return");
- return state;
- }
- }
-
+ message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP,
+ coreNodeName);
}
+
// use the provided non null shardId
if (sliceName == null) {
//get shardId from ClusterState
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Sun Jan 5 21:04:58 2014
@@ -1191,7 +1191,7 @@ public class OverseerCollectionProcessor
// refresh cluster state
clusterState = zkStateReader.getClusterState();
Slice tempSourceSlice = clusterState.getCollection(tempSourceCollectionName).getSlices().iterator().next();
- Replica tempSourceLeader = clusterState.getLeader(tempSourceCollectionName, tempSourceSlice.getName());
+ Replica tempSourceLeader = zkStateReader.getLeaderRetry(tempSourceCollectionName, tempSourceSlice.getName(), 60000);
String tempCollectionReplica1 = tempSourceCollectionName + "_" + tempSourceSlice.getName() + "_replica1";
String coreNodeName = waitForCoreNodeName(clusterState.getCollection(tempSourceCollectionName),
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java Sun Jan 5 21:04:58 2014
@@ -3,6 +3,7 @@ package org.apache.solr.cloud;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
@@ -48,6 +49,8 @@ public class ZkCLI {
private static final String MAKEPATH = "makepath";
private static final String PUT = "put";
private static final String PUT_FILE = "putfile";
+ private static final String GET = "get";
+ private static final String GET_FILE = "getfile";
private static final String DOWNCONFIG = "downconfig";
private static final String ZK_CLI_NAME = "ZkCLI";
private static final String HELP = "help";
@@ -92,7 +95,7 @@ public class ZkCLI {
.withDescription(
"cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG
+ ", " + LINKCONFIG + ", " + MAKEPATH + ", " + PUT + ", " + PUT_FILE + ","
- + LIST + ", " + CLEAR).create(CMD));
+ + GET + "," + GET_FILE + ", " + LIST + ", " + CLEAR).create(CMD));
Option zkHostOption = new Option("z", ZKHOST, true,
"ZooKeeper host address");
@@ -137,6 +140,8 @@ public class ZkCLI {
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT_FILE + " /solr.xml /User/myuser/solr/solr.xml");
+ System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET + " /solr.xml");
+ System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET_FILE + " /solr.xml solr.xml.file");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
return;
@@ -264,6 +269,22 @@ public class ZkCLI {
IOUtils.closeQuietly(is);
}
+ } else if (line.getOptionValue(CMD).equals(GET)) {
+ List arglist = line.getArgList();
+ if (arglist.size() != 1) {
+ System.out.println("-" + GET + " requires one arg - the path to get");
+ System.exit(1);
+ }
+ byte [] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
+ System.out.println(new String(data, "UTF-8"));
+ } else if (line.getOptionValue(CMD).equals(GET_FILE)) {
+ List arglist = line.getArgList();
+ if (arglist.size() != 2) {
+ System.out.println("-" + GET_FILE + "requires two args - the path to get and the file to save it to");
+ System.exit(1);
+ }
+ byte [] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
+ FileUtils.writeByteArrayToFile(new File(arglist.get(1).toString()), data);
}
} finally {
if (solrPort != null) {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java Sun Jan 5 21:04:58 2014
@@ -477,10 +477,7 @@ public final class ZkController {
host = "http://" + hostaddress;
} else {
Matcher m = URL_PREFIX.matcher(host);
- if (m.matches()) {
- String prefix = m.group(1);
- host = prefix + host;
- } else {
+ if (!m.matches()) {
host = "http://" + host;
}
}
@@ -1337,21 +1334,7 @@ public final class ZkController {
// this also gets us our assigned shard id if it was not specified
try {
CloudDescriptor cloudDesc = cd.getCloudDescriptor();
- if(cd.getCloudDescriptor().getCollectionName() !=null && cloudDesc.getCoreNodeName() != null ) {
- //we were already registered
- if(zkStateReader.getClusterState().hasCollection(cloudDesc.getCollectionName())){
- DocCollection coll = zkStateReader.getClusterState().getCollection(cloudDesc.getCollectionName());
- if(!"true".equals(coll.getStr("autoCreated"))){
- Slice slice = coll.getSlice(cloudDesc.getShardId());
- if(slice != null){
- if(slice.getReplica(cloudDesc.getCoreNodeName()) == null) {
- log.info("core_removed This core is removed from ZK");
- throw new SolrException(ErrorCode.NOT_FOUND,cloudDesc.getCoreNodeName() +" is removed");
- }
- }
- }
- }
- }
+
// make sure the node name is set on the descriptor
if (cloudDesc.getCoreNodeName() == null) {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Sun Jan 5 21:04:58 2014
@@ -399,7 +399,7 @@ public class DefaultSolrHighlighter exte
params.getBool( HighlightParams.USE_PHRASE_HIGHLIGHTER, true ),
// FVH cannot process hl.requireFieldMatch parameter per-field basis
params.getBool( HighlightParams.FIELD_MATCH, false ) );
- fvh.setPhraseLimit(params.getInt(HighlightParams.PHRASE_LIMIT, Integer.MAX_VALUE));
+ fvh.setPhraseLimit(params.getInt(HighlightParams.PHRASE_LIMIT, SolrHighlighter.DEFAULT_PHRASE_LIMIT));
FieldQuery fieldQuery = fvh.getFieldQuery( query, searcher.getIndexReader() );
// Highlight each document
@@ -602,6 +602,10 @@ public class DefaultSolrHighlighter exte
String alternateField = params.getFieldParam(fieldName, HighlightParams.ALTERNATE_FIELD);
if (alternateField != null && alternateField.length() > 0) {
StorableField[] docFields = doc.getFields(alternateField);
+ if (docFields.length == 0) {
+ // The alternate field did not exist, treat the original field as fallback instead
+ docFields = doc.getFields(fieldName);
+ }
List<String> listFields = new ArrayList<String>();
for (StorableField field : docFields) {
if (field.binaryValue() == null)
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java Sun Jan 5 21:04:58 2014
@@ -34,6 +34,8 @@ import org.apache.solr.util.SolrPluginUt
public abstract class SolrHighlighter
{
+
+ public static int DEFAULT_PHRASE_LIMIT = 5000;
public static Logger log = LoggerFactory.getLogger(SolrHighlighter.class);
@Deprecated
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java Sun Jan 5 21:04:58 2014
@@ -17,7 +17,12 @@
package org.apache.solr.search;
+import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
@@ -47,6 +52,7 @@ import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Iterator;
@@ -242,7 +248,7 @@ public class CollapsingQParserPlugin ext
SchemaField schemaField = schema.getField(this.field);
SortedDocValues docValues = null;
-
+ FunctionQuery funcQuery = null;
if(schemaField.hasDocValues()) {
docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
} else {
@@ -252,11 +258,39 @@ public class CollapsingQParserPlugin ext
FieldType fieldType = null;
if(this.max != null) {
- fieldType = searcher.getSchema().getField(this.max).getType();
+ if(this.max.indexOf("(") == -1) {
+ fieldType = searcher.getSchema().getField(this.max).getType();
+ } else {
+ LocalSolrQueryRequest request = null;
+ try {
+ SolrParams params = new ModifiableSolrParams();
+ request = new LocalSolrQueryRequest(searcher.getCore(), params);
+ FunctionQParser functionQParser = new FunctionQParser(this.max, null, null,request);
+ funcQuery = (FunctionQuery)functionQParser.parse();
+ } catch (Exception e) {
+ throw new IOException(e);
+ } finally {
+ request.close();
+ }
+ }
}
if(this.min != null) {
- fieldType = searcher.getSchema().getField(this.min).getType();
+ if(this.min.indexOf("(") == -1) {
+ fieldType = searcher.getSchema().getField(this.min).getType();
+ } else {
+ LocalSolrQueryRequest request = null;
+ try {
+ SolrParams params = new ModifiableSolrParams();
+ request = new LocalSolrQueryRequest(searcher.getCore(), params);
+ FunctionQParser functionQParser = new FunctionQParser(this.min, null, null,request);
+ funcQuery = (FunctionQuery)functionQParser.parse();
+ } catch (Exception e) {
+ throw new IOException(e);
+ } finally {
+ request.close();
+ }
+ }
}
int maxDoc = searcher.maxDoc();
@@ -274,7 +308,8 @@ public class CollapsingQParserPlugin ext
max != null,
this.needsScores,
fieldType,
- boostDocs);
+ boostDocs,
+ funcQuery);
} else {
return new CollapsingScoreCollector(maxDoc, leafCount, docValues, this.nullPolicy, boostDocs);
}
@@ -508,7 +543,8 @@ public class CollapsingQParserPlugin ext
boolean max,
boolean needsScores,
FieldType fieldType,
- IntOpenHashSet boostDocs) throws IOException{
+ IntOpenHashSet boostDocs,
+ FunctionQuery funcQuery) throws IOException{
this.maxDoc = maxDoc;
this.contexts = new AtomicReaderContext[segments];
@@ -517,14 +553,18 @@ public class CollapsingQParserPlugin ext
this.nullPolicy = nullPolicy;
this.needsScores = needsScores;
this.boostDocs = boostDocs;
- if(fieldType instanceof TrieIntField) {
- this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
- } else if(fieldType instanceof TrieLongField) {
- this.fieldValueCollapse = new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
- } else if(fieldType instanceof TrieFloatField) {
- this.fieldValueCollapse = new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ if(funcQuery != null) {
+ this.fieldValueCollapse = new ValueSourceCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs, funcQuery);
} else {
- throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
+ if(fieldType instanceof TrieIntField) {
+ this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ } else if(fieldType instanceof TrieLongField) {
+ this.fieldValueCollapse = new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ } else if(fieldType instanceof TrieFloatField) {
+ this.fieldValueCollapse = new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ } else {
+ throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
+ }
}
}
@@ -877,6 +917,97 @@ public class CollapsingQParserPlugin ext
}
}
+ private class ValueSourceCollapse extends FieldValueCollapse {
+
+ private FloatCompare comp;
+ private float nullVal;
+ private ValueSource valueSource;
+ private FunctionValues functionValues;
+ private float[] ordVals;
+ private Map rcontext = new HashMap();
+ private CollapseScore collapseScore = new CollapseScore();
+ private float score;
+ private boolean cscore;
+
+ public ValueSourceCollapse(int maxDoc,
+ String funcStr,
+ int nullPolicy,
+ int[] ords,
+ boolean max,
+ boolean needsScores,
+ IntOpenHashSet boostDocs,
+ FunctionQuery funcQuery) throws IOException {
+ super(maxDoc, null, nullPolicy, max, needsScores, boostDocs);
+ this.valueSource = funcQuery.getValueSource();
+ this.ords = ords;
+ this.ordVals = new float[ords.length];
+ Arrays.fill(ords, -1);
+
+ if(max) {
+ comp = new MaxFloatComp();
+ Arrays.fill(ordVals, -Float.MAX_VALUE );
+ } else {
+ this.nullVal = Float.MAX_VALUE;
+ comp = new MinFloatComp();
+ Arrays.fill(ordVals, Float.MAX_VALUE);
+ }
+
+ if(funcStr.indexOf("cscore()") != -1) {
+ this.cscore = true;
+ this.rcontext.put("CSCORE",this.collapseScore);
+ }
+
+ if(this.needsScores) {
+ this.scores = new float[ords.length];
+ if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
+ nullScores = new FloatArrayList();
+ }
+ }
+ }
+
+ public void setNextReader(AtomicReaderContext context) throws IOException {
+ functionValues = this.valueSource.getValues(rcontext, context);
+ }
+
+ public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
+ if(needsScores || cscore) {
+ this.score = scorer.score();
+ this.collapseScore.score = score;
+ }
+
+ float val = functionValues.floatVal(contextDoc);
+
+ if(ord > -1) {
+ if(comp.test(val, ordVals[ord])) {
+ ords[ord] = globalDoc;
+ ordVals[ord] = val;
+ if(needsScores) {
+ scores[ord] = score;
+ }
+ }
+ } else if (this.collapsedSet.fastGet(globalDoc)) {
+ //Elevated doc so do nothing
+ } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
+ if(comp.test(val, nullVal)) {
+ nullVal = val;
+ nullDoc = globalDoc;
+ if(needsScores) {
+ nullScore = score;
+ }
+ }
+ } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
+ this.collapsedSet.fastSet(globalDoc);
+ if(needsScores) {
+ nullScores.add(score);
+ }
+ }
+ }
+ }
+
+ public static final class CollapseScore {
+ public float score;
+ }
+
private interface IntCompare {
public boolean test(int i1, int i2);
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java Sun Jan 5 21:04:58 2014
@@ -41,6 +41,7 @@ import org.apache.solr.common.util.Named
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.*;
+import org.apache.solr.search.function.CollapseScoreFunction;
import org.apache.solr.search.function.distance.*;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
@@ -221,6 +222,12 @@ public abstract class ValueSourceParser
};
}
});
+ addParser("cscore", new ValueSourceParser() {
+ @Override
+ public ValueSource parse(FunctionQParser fp) throws SyntaxError {
+ return new CollapseScoreFunction();
+ }
+ });
addParser("sum", new ValueSourceParser() {
@Override
public ValueSource parse(FunctionQParser fp) throws SyntaxError {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/PeerSync.java Sun Jan 5 21:04:58 2014
@@ -291,7 +291,8 @@ public class PeerSync {
if (cantReachIsSuccess && sreq.purpose == 1 && srsp.getException() instanceof SolrServerException) {
Throwable solrException = ((SolrServerException) srsp.getException())
.getRootCause();
- if (solrException instanceof ConnectException || solrException instanceof ConnectTimeoutException
+ boolean connectTimeoutExceptionInChain = connectTimeoutExceptionInChain(srsp.getException());
+ if (connectTimeoutExceptionInChain || solrException instanceof ConnectException || solrException instanceof ConnectTimeoutException
|| solrException instanceof NoHttpResponseException || solrException instanceof SocketException) {
log.warn(msg() + " couldn't connect to " + srsp.getShardAddress() + ", counting as success");
@@ -309,6 +310,10 @@ public class PeerSync {
"Perhaps /get is not registered?");
return true;
}
+
+ // TODO: we should return the above information so that when we can request a recovery through zookeeper, we do
+ // that for these nodes
+
// TODO: at least log???
// srsp.getException().printStackTrace(System.out);
@@ -324,6 +329,23 @@ public class PeerSync {
}
}
+ // sometimes the root exception is a SocketTimeoutException, but ConnectTimeoutException
+ // is in the chain
+ private boolean connectTimeoutExceptionInChain(Throwable exception) {
+ Throwable t = exception;
+ while (true) {
+ if (t instanceof ConnectTimeoutException) {
+ return true;
+ }
+ Throwable cause = t.getCause();
+ if (cause != null) {
+ t = cause;
+ } else {
+ return false;
+ }
+ }
+ }
+
private boolean handleVersions(ShardResponse srsp) {
// we retrieved the last N updates from the replica
List<Long> otherVersions = (List<Long>)srsp.getSolrResponse().getResponse().get("versions");
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Sun Jan 5 21:04:58 2014
@@ -105,13 +105,8 @@ public class SolrIndexSplitter {
// - need to worry about if IW.addIndexes does a sync or not...
// - would be more efficient on the read side, but prob less efficient merging
- IndexReader[] subReaders = new IndexReader[leaves.size()];
for (int partitionNumber=0; partitionNumber<numPieces; partitionNumber++) {
- log.info("SolrIndexSplitter: partition #" + partitionNumber + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
-
- for (int segmentNumber = 0; segmentNumber<subReaders.length; segmentNumber++) {
- subReaders[segmentNumber] = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
- }
+ log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
boolean success = false;
@@ -130,8 +125,12 @@ public class SolrIndexSplitter {
}
try {
- // This merges the subreaders and will thus remove deletions (i.e. no optimize needed)
- iw.addIndexes(subReaders);
+ // This removes deletions but optimize might still be needed because sub-shards will have the same number of segments as the parent shard.
+ for (int segmentNumber = 0; segmentNumber<leaves.size(); segmentNumber++) {
+ log.info("SolrIndexSplitter: partition #" + partitionNumber + " partitionCount=" + numPieces + (ranges != null ? " range=" + ranges.get(partitionNumber) : "") + " segment #"+segmentNumber + " segmentCount=" + leaves.size());
+ IndexReader subReader = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
+ iw.addIndexes(subReader);
+ }
success = true;
} finally {
if (iwRef != null) {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Sun Jan 5 21:04:58 2014
@@ -614,6 +614,12 @@ public class DistributedUpdateProcessor
// if one node is a RetryNode, this was a forward request
if (errors.get(0).req.node instanceof RetryNode) {
rsp.setException(errors.get(0).e);
+ } else {
+ if (log.isWarnEnabled()) {
+ for (Error error : errors) {
+ log.warn("Error sending update", error.e);
+ }
+ }
}
// else
// for now we don't error - we assume if it was added locally, we
@@ -625,7 +631,6 @@ public class DistributedUpdateProcessor
// recover - the doc was already added locally, so it should have been
// legit
- // TODO: we should do this in the background it would seem
for (final SolrCmdDistributor.Error error : errors) {
if (error.req.node instanceof RetryNode) {
// we don't try to force a leader to recover
@@ -1271,22 +1276,10 @@ public class DistributedUpdateProcessor
return;
}
- if (zkController.isConnected()) {
+ if (!zkController.getZkClient().getConnectionManager().isLikelyExpired()) {
return;
}
- long timeoutAt = System.currentTimeMillis() + zkController.getClientTimeout();
- while (System.currentTimeMillis() < timeoutAt) {
- if (zkController.isConnected()) {
- return;
- }
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- break;
- }
- }
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Cannot talk to ZooKeeper - Updates are disabled.");
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Sun Jan 5 21:04:58 2014
@@ -282,14 +282,6 @@ public class BasicDistributedZk2Test ext
// kill a shard
CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
-
-
- // we are careful to make sure the downed node is no longer in the state,
- // because on some systems (especially freebsd w/ blackhole enabled), trying
- // to talk to a downed node causes grief
- Set<CloudJettyRunner> jetties = new HashSet<CloudJettyRunner>();
- jetties.addAll(shardToJetty.get(SHARD1));
- jetties.remove(deadShard);
// ensure shard is dead
try {
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Sun Jan 5 21:04:58 2014
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.util.LuceneTestCase.Slow;
@@ -100,6 +101,8 @@ public class BasicDistributedZkTest exte
private String oneInstanceCollection = "oneInstanceCollection";
private String oneInstanceCollection2 = "oneInstanceCollection2";
+ private AtomicInteger nodeCounter = new AtomicInteger();
+
ThreadPoolExecutor executor = new ThreadPoolExecutor(0,
Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
@@ -928,11 +931,16 @@ public class BasicDistributedZkTest exte
try {
server = new HttpSolrServer(baseUrl);
server.setConnectionTimeout(15000);
- server.setSoTimeout(60000);
Create createCmd = new Create();
createCmd.setRoles("none");
createCmd.setCoreName(collection + num);
createCmd.setCollection(collection);
+
+ if (random().nextBoolean()) {
+ // sometimes we use an explicit core node name
+ createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet());
+ }
+
if (shardId == null) {
createCmd.setNumShards(2);
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java Sun Jan 5 21:04:58 2014
@@ -34,7 +34,9 @@ import org.apache.solr.common.params.Map
import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
+@Ignore("Not currently valid see SOLR-5580")
public class DeleteInactiveReplicaTest extends DeleteReplicaTest{
private CloudSolrServer client;
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java Sun Jan 5 21:04:58 2014
@@ -218,6 +218,15 @@ public class OverseerCollectionProcessor
}).anyTimes();
}
+
+ solrZkClientMock.getZkClientTimeout();
+ expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() throws Throwable {
+ return 30000;
+ }
+ }).anyTimes();
+
clusterStateMock.hasCollection(anyObject(String.class));
expectLastCall().andAnswer(new IAnswer<Boolean>() {
@Override
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Sun Jan 5 21:04:58 2014
@@ -135,7 +135,7 @@ public class SyncSliceTest extends Abstr
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
// we only set the connect timeout, not so timeout
- baseServer.setConnectionTimeout(15000);
+ baseServer.setConnectionTimeout(30000);
baseServer.request(request);
waitForThingsToLevelOut(15);
@@ -165,7 +165,7 @@ public class SyncSliceTest extends Abstr
Thread.sleep(2000);
- waitForThingsToLevelOut(90);
+ waitForThingsToLevelOut(120);
Thread.sleep(1000);
@@ -228,7 +228,7 @@ public class SyncSliceTest extends Abstr
Thread.sleep(3000);
- waitForThingsToLevelOut(90);
+ waitForThingsToLevelOut(120);
Thread.sleep(2000);
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java Sun Jan 5 21:04:58 2014
@@ -33,6 +33,8 @@ import org.apache.solr.common.cloud.Solr
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.util.ExternalPaths;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -262,6 +264,47 @@ public class ZkCLITest extends SolrTestC
assertEquals(0, zkClient.getChildren("/", null, true).size());
}
+ @Test
+ public void testGet() throws Exception {
+ String getNode = "/getNode";
+ byte [] data = new String("getNode-data").getBytes("UTF-8");
+ this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+ "get", getNode};
+ ZkCLI.main(args);
+ }
+
+ @Test
+ public void testGetFile() throws Exception {
+ String getNode = "/getFileNode";
+ byte [] data = new String("getFileNode-data").getBytes("UTF-8");
+ this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
+
+ File file = new File(TEMP_DIR,
+ "solrtest-getfile-" + this.getClass().getName() + "-" + System.currentTimeMillis());
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+ "getfile", getNode, file.getAbsolutePath()};
+ ZkCLI.main(args);
+
+ byte [] readData = FileUtils.readFileToByteArray(file);
+ assertArrayEquals(data, readData);
+ }
+
+ @Test
+ public void testGetFileNotExists() throws Exception {
+ String getNode = "/getFileNotExistsNode";
+
+ File file = new File(TEMP_DIR,
+ "solrtest-getfilenotexists-" + this.getClass().getName() + "-" + System.currentTimeMillis());
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+ "getfile", getNode, file.getAbsolutePath()};
+ try {
+ ZkCLI.main(args);
+ fail("Expected NoNodeException");
+ } catch (KeeperException.NoNodeException ex) {
+ }
+ }
+
@Override
public void tearDown() throws Exception {
if (VERBOSE) {
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Sun Jan 5 21:04:58 2014
@@ -32,9 +32,11 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
+import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeoutException;
@Slow
public class ZkControllerTest extends SolrTestCaseJ4 {
@@ -255,6 +257,46 @@ public class ZkControllerTest extends So
}
+ @Test
+ public void testGetHostName() throws Exception {
+ String zkDir = dataDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+ CoreContainer cc = null;
+
+ ZkTestServer server = new ZkTestServer(zkDir);
+ try {
+ server.run();
+
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+ AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+
+ cc = getCoreContainer();
+ ZkController zkController = null;
+
+ try {
+ zkController = new ZkController(cc, server.getZkAddress(), TIMEOUT, 10000,
+ "http://127.0.0.1", "8983", "solr", 0, true, new CurrentCoreDescriptorProvider() {
+
+ @Override
+ public List<CoreDescriptor> getCurrentDescriptors() {
+ // do nothing
+ return null;
+ }
+ });
+ } catch (IllegalArgumentException e) {
+ fail("ZkController did not normalize host name correctly");
+ } finally {
+ if (zkController != null)
+ zkController.close();
+ }
+ } finally {
+ if (cc != null) {
+ cc.shutdown();
+ }
+ server.shutdown();
+ }
+ }
+
private CoreContainer getCoreContainer() {
CoreContainer cc = new CoreContainer(solrHomeDirectory.getAbsolutePath());
cc.load();
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java Sun Jan 5 21:04:58 2014
@@ -56,6 +56,8 @@ public class HdfsTestUtil {
System.setProperty("test.cache.data", dir.getAbsolutePath() + File.separator + "hdfs" + File.separator + "cache");
System.setProperty("solr.lock.type", "hdfs");
+ System.setProperty("solr.hdfs.home", "/solr_hdfs_home");
+
MiniDFSCluster dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null);
SolrTestCaseJ4.useFactory("org.apache.solr.core.HdfsDirectoryFactory");
@@ -68,6 +70,7 @@ public class HdfsTestUtil {
System.clearProperty("solr.lock.type");
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
+ System.clearProperty("solr.hdfs.home");
if (dfsCluster != null) {
dfsCluster.shutdown();
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java Sun Jan 5 21:04:58 2014
@@ -128,7 +128,6 @@ public class StressHdfsTest extends Basi
}
c.query(new SolrQuery("id:" + i));
- c.setSoTimeout(60000);
c.setConnectionTimeout(30000);
NamedList<Object> response = c.query(
new SolrQuery().setRequestHandler("/admin/system")).getResponse();
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java Sun Jan 5 21:04:58 2014
@@ -660,6 +660,16 @@ public class HighlighterTest extends Sol
"//lst[@name='highlighting']/lst[@name='1' and count(*)=1]",
"//lst[@name='highlighting']/lst[@name='1']/arr[@name='t_text']/str[.='a piece of text']"
);
+
+ // with a non-existing alternate field + max length
+ args.put("hl.alternateField", "NonExistingField");
+ args.put("hl.maxAlternateFieldLength", "15");
+ sumLRF = h.getRequestFactory("standard", 0, 200, args);
+ assertQ("Alternate summarization",
+ sumLRF.makeRequest("tv_text:keyword"),
+ "//lst[@name='highlighting']/lst[@name='1' and count(*)=1]",
+ "//lst[@name='highlighting']/lst[@name='1']/arr[@name='t_text']/str[.='a piece of text']"
+ );
}
@Test
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java Sun Jan 5 21:04:58 2014
@@ -343,6 +343,11 @@ public class QueryEqualityTest extends S
public void testFuncRord() throws Exception {
assertFuncEquals("rord(foo_s)","rord(foo_s )");
}
+
+ public void testFuncCscore() throws Exception {
+ assertFuncEquals("cscore()", "cscore( )");
+ }
+
public void testFuncTop() throws Exception {
assertFuncEquals("top(sum(3,foo_i))");
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java Sun Jan 5 21:04:58 2014
@@ -95,6 +95,40 @@ public class TestCollapseQParserPlugin e
"//result/doc[4]/float[@name='id'][.='6.0']"
);
+ // Test value source collapse criteria
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s nullPolicy=collapse min=field(test_ti)}");
+ params.add("sort", "test_ti desc");
+ assertQ(req(params), "*[count(//doc)=3]",
+ "//result/doc[1]/float[@name='id'][.='4.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']",
+ "//result/doc[3]/float[@name='id'][.='5.0']"
+ );
+
+ // Test value source collapse criteria with cscore function
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s nullPolicy=collapse min=cscore()}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ assertQ(req(params), "*[count(//doc)=3]",
+ "//result/doc[1]/float[@name='id'][.='4.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']",
+ "//result/doc[3]/float[@name='id'][.='5.0']"
+ );
+
+ // Test value source collapse criteria with compound cscore function
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s nullPolicy=collapse min=sum(cscore(),field(test_ti))}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ assertQ(req(params), "*[count(//doc)=3]",
+ "//result/doc[1]/float[@name='id'][.='4.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']",
+ "//result/doc[3]/float[@name='id'][.='5.0']"
+ );
//Test collapse by score with elevation
Modified: lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene5376/solr/example/solr/collection1/conf/schema.xml Sun Jan 5 21:04:58 2014
@@ -779,6 +779,18 @@
</analyzer>
</fieldType>
+ <!-- Kurdish -->
+ <fieldType name="text_ckb" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.SoraniNormalizationFilterFactory"/>
+ <!-- for any latin text -->
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ckb.txt"/>
+ <filter class="solr.SoraniStemFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
<!-- Czech -->
<fieldType name="text_cz" class="solr.TextField" positionIncrementGap="100">
<analyzer>
Modified: lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java?rev=1555629&r1=1555628&r2=1555629&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java (original)
+++ lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java Sun Jan 5 21:04:58 2014
@@ -384,7 +384,7 @@ public class ConcurrentUpdateSolrServer
synchronized (runners) {
runner = runners.peek();
}
- if (runner == null)
+ if (runner == null || scheduler.isTerminated())
break;
runner.runnerLock.lock();
runner.runnerLock.unlock();