You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/01/13 19:58:52 UTC
svn commit: r1557811 - in /lucene/dev/branches/lucene_solr_4_6: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/cloud/ solr/solrj/
solr/solrj/src/java/org/apache/solr/common/cloud/
Author: markrmiller
Date: Mon Jan 13 18:58:52 2014
New Revision: 1557811
URL: http://svn.apache.org/r1557811
Log:
SOLR-5615: Deadlock while trying to recover after a ZK session expiration.
Modified:
lucene/dev/branches/lucene_solr_4_6/ (props changed)
lucene/dev/branches/lucene_solr_4_6/solr/ (props changed)
lucene/dev/branches/lucene_solr_4_6/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_4_6/solr/core/ (props changed)
lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/lucene_solr_4_6/solr/solrj/ (props changed)
lucene/dev/branches/lucene_solr_4_6/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
Modified: lucene/dev/branches/lucene_solr_4_6/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_6/solr/CHANGES.txt?rev=1557811&r1=1557810&r2=1557811&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_6/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_6/solr/CHANGES.txt Mon Jan 13 18:58:52 2014
@@ -20,6 +20,214 @@ See the tutorial at http://lucene.apache
$Id$
+================== 4.7.0 ==================
+
+Versions of Major Components
+---------------------
+Apache Tika 1.4
+Carrot2 3.8.0
+Velocity 1.7 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.4.5
+
+Upgrading from Solr 4.6.0
+----------------------
+
+* CloudSolrServer and LBHttpSolrServer no longer declare MalformedURLException
+ as thrown from their constructors.
+
+Detailed Change List
+----------------------
+
+New Features
+----------------------
+
+* SOLR-5308: SOLR-5601: A new 'migrate' collection API to split all documents with a
+ route key into another collection (shalin)
+
+* SOLR-5441: Expose number of transaction log files and their size via JMX.
+ (RafaÅ KuÄ via shalin)
+
+* SOLR-5320: Added support for tri-level compositeId routing.
+ (Anshum Gupta via shalin)
+
+* SOLR-5287: You can edit files in the conf directory from the admin UI
+ (Erick Erickson, Stefan Matheis)
+
+* SOLR-5447, SOLR-5490: Add a QParserPlugin for Lucene's SimpleQueryParser.
+ (Jack Conradson via shalin)
+
+* SOLR-5446: Admin UI - Allow changing Schema and Config (steffkes)
+
+* SOLR-5456: Admin UI - Allow creating new Files (steffkes)
+
+* SOLR-5208: Support for the setting of core.properties key/values at create-time on
+ Collections API (Erick Erickson)
+
+* SOLR-5428: New 'stats.calcdistinct' parameter in StatsComponent returns
+ set of distinct values and their count. This can also be specified per field
+ e.g. 'f.field.stats.calcdistinct'. (Elran Dvir via shalin)
+
+* SOLR-5378, SOLR-5528: A new SuggestComponent that fully utilizes the Lucene suggester
+ module and adds pluggable dictionaries, payloads and better distributed support.
+ This is intended to eventually replace the Suggester support through the
+ SpellCheckComponent. (Areek Zillur, Varun Thacker via shalin)
+
+* SOLR-5492: Return the replica that actually served the query in shards.info
+ response. (shalin)
+
+* SOLR-5506: Support docValues in CollationField and ICUCollationField.
+ (Robert Muir)
+
+ * SOLR-5518: Added EditFileRequestHandler to deal with security issues around modifying
+ solr configuration files.
+
+* SOLR-5023: Add support for deleteInstanceDir to be passed from SolrJ for Core
+ Unload action. (Lyubov Romanchuk, shalin)
+
+* SOLR-1871: The 'map' function query accepts a ValueSource as target and
+ default value. (Chris Harris, shalin)
+
+* SOLR-5518: Make the EditFileRequestHandler it's own handler to allow
+ some security (Erick Erickson)
+
+* SOLR-5522: Remove EditFileRequestHandler from 4x (Erick Erickson)
+
+* SOLR-5556: Allow class of CollectionsHandler and InfoHandler to be specified
+ in solr.xml. (Gregory Chanan, Alan Woodward)
+
+* SOLR-5581: Give ZkCLI the ability to get files. (Gregory Chanan via Mark Miller)
+
+* SOLR-5536: Add ValueSource collapse criteria to CollapsingQParsingPlugin (Joel Bernstein)
+
+Bug Fixes
+----------------------
+
+* SOLR-5438: DebugComponent throws NPE when used with grouping.
+ (Tomás Fernández Löbbe via shalin)
+
+* SOLR-4612: Admin UI - Analysis Screen contains empty table-columns (steffkes)
+
+* SOLR-5451: SyncStrategy closes it's http connection manager before the
+ executor that uses it in it's close method. (Mark Miller)
+
+* SOLR-5460: SolrDispatchFilter#sendError can get a SolrCore that it does not
+ close. (Mark Miller)
+
+* SOLR-5461: Request proxying should only set con.setDoOutput(true) if the
+ request is a post. (Mark Miller)
+
+* SOLR-5481: SolrCmdDistributor should not let the http client do it's own
+ retries. (Mark Miller)
+
+* LUCENE-5347: Fixed Solr's Zookeeper Client to copy files to Zookeeper using
+ binary transfer. Previously data was read with default encoding and stored
+ in zookeeper as UTF-8. This bug was found after upgrading to forbidden-apis
+ 1.4. (Uwe Schindler)
+
+* SOLR-4376: DataImportHandler uses wrong date format for last_index_time if
+ a delta-import is run first before any full-imports.
+ (Sebastien Lorber, Arcadius Ahouansou via shalin)
+
+* SOLR-5494: CoreContainer#remove throws NPE rather than returning null when
+ a SolrCore does not exist in core discovery mode. (Mark Miller)
+
+* SOLR-5354: Distributed sort is broken with CUSTOM FieldType.
+ (Steve Rowe, hossman, Robert Muir, Jessica Cheng)
+
+* SOLR-5515: NPE when getting stats on date field with empty result on
+ SolrCloud. (Alexander Sagen, shalin)
+
+* SOLR-5204: StatsComponent and SpellCheckComponent do not support the
+ shards.tolerant=true parameter. (Anca Kopetz, shalin)
+
+* SOLR-5527: DIH logs spurious warning for special commands. (shalin)
+
+* SOLR-5524: Exception when using Query Function inside Scale Function.
+ (Trey Grainger, yonik)
+
+* SOLR-5540: HdfsLockFactory should explicitly create the lock parent directory
+ if necessary. (Mark Miller)
+
+* SOLR-5543: Core swaps resulted in duplicate core entries in solr.xml when
+ using solr.xml persistence. (Bill Bell, Alan Woodward)
+
+* SOLR-5562: ConcurrentUpdateSolrServer constructor ignores supplied httpclient.
+ (Kyle Halliday via Mark Miller)
+
+* SOLR-5567: ZkController getHostAddress duplicates url prefix.
+ (Kyle Halliday, Alexey Serba, shalin)
+
+* SOLR-5577: Likely ZooKeeper expiration should not slow down updates a given
+ amount, but instead cut off updates after a given time.
+ (Mark Miller, Christine Poerschke, Ramkumar Aiyengar)
+
+Optimizations
+----------------------
+
+* SOLR-5458: Admin UI - Remove separated Pages for Config & Schema (steffkes)
+
+* SOLR-5436: Eliminate the 1500ms wait in overseer loop as well as
+ polling the ZK distributed queue. (Noble Paul, Mark Miller)
+
+* SOLR-5189: Solr 4.x Web UI Log Viewer does not display 'date' column from
+ logs (steffkes)
+
+* SOLR-5512: Optimize DocValuesFacets. (Robert Muir)
+
+* SOLR-2960: fix DIH XPathEntityProcessor to add the correct number of "null"
+ placeholders for multi-valued fields (Michael Watts via James Dyer)
+
+* SOLR-5214: Reduce memory usage for shard splitting by merging segments one
+ at a time. (Christine Poerschke via shalin)
+
+Other Changes
+---------------------
+
+* SOLR-5399: Add distributed request tracking information to DebugComponent
+ (Tomás Fernández Löbbe via Ryan Ernst)
+
+* SOLR-5421: Remove double set of distrib.from param in processAdd method of
+ DistributedUpdateProcessor. (Anshum Gupta via shalin)
+
+* SOLR-5404: The example config references deprecated classes.
+ (Uwe Schindler, RafaÅ KuÄ via Mark Miller)
+
+* SOLR-5487: Replication factor error message doesn't match constraint.
+ (Patrick Hunt via shalin)
+
+* SOLR-5499: Log a warning if /get is not registered when using SolrCloud.
+ (Daniel Collins via shalin)
+
+* SOLR-5517: Return HTTP error on POST requests with no Content-Type.
+ (Ryan Ernst, Uwe Schindler)
+
+* SOLR-5502: Added a test for tri-level compositeId routing with documents
+ having a "/" in a document id. (Anshum Gupta via Mark Miller)
+
+* SOLR-5539: Admin UI - Remove ability to create/modify files (steffkes)
+
+* SOLR-5533: Improve out of the box support for running Solr on hdfs with
+ SolrCloud. (Mark Miller)
+
+* SOLR-5548: Give DistributedSearchTestCase / JettySolrRunner the ability to
+ specify extra filters. (Greg Chanan via Mark Miller)
+
+* SOLR-5555: LBHttpSolrServer and CloudSolrServer constructors don't need to
+ declare MalformedURLExceptions (Sushil Bajracharya, Alan Woodward)
+
+* SOLR-5565: Raise default ZooKeeper session timeout to 30 seconds from 15
+ seconds. (Mark Miller)
+
+* SOLR-5574: CoreContainer shutdown publishes all nodes as down and waits to
+ see that and then again publishes all nodes as down. (Mark Miller)
+
+* SOLR-5590: Upgrade HttpClient/HttpComponents to 4.3.x.
+ (Karl Wright via Shawn Heisey)
+
+* SOLR-2794: change the default of hl.phraseLimit to 5000.
+ (Michael Della Bitta via Robert Muir, Koji, zarni - pull request #11)
+
================== 4.6.1 ==================
Versions of Major Components
@@ -58,10 +266,6 @@ Bug Fixes
sets the alias name and the collections to alias to the same value.
(Aaron Schram, Mark Miller)
-* SOLR-5577: Likely ZooKeeper expiration should not slow down updates a given
- amount, but instead cut off updates after a given time.
- (Mark Miller, Christine Poerschke, Ramkumar Aiyengar)
-
* SOLR-5580: NPE when creating a core with both explicit shard and coreNodeName.
(YouPeng Yang, Mark Miller)
@@ -88,7 +292,7 @@ Bug Fixes
* SOLR-5587: ElectionContext implementations should use
ZkCmdExecutor#ensureExists to ensure their election paths are properly
created. (Mark Miller)
-
+
* SOLR-4709: The core reload after replication if config files have changed
can fail due to a race condition. (Mark Miller, Hossman))
@@ -97,25 +301,23 @@ Bug Fixes
* SOLR-5588: PeerSync doesn't count all connect failures as success.
(Mark Miller)
-
+
* SOLR-5564: hl.maxAlternateFieldLength should apply to original field when
fallback is attempted (janhoy)
* SOLR-5608: Don't allow a closed SolrCore to publish state to ZooKeeper.
(Mark Miller, Shawn Heisey)
-* SOLR-5543: Core swaps resulted in duplicate core entries in solr.xml when
- using solr.xml persistence. (Bill Bell, Alan Woodward)
-
-* SOLR-5618: Fix false cache hits in queryResultCache when hashCodes are equal
- and duplicate filter queries exist in one of the requests (hossman)
+* SOLR-5615: Deadlock while trying to recover after a ZK session expiration.
+ (Ramkumar Aiyengar, Mark Miller)
+
Optimizations
-----------------------
+----------------------
* SOLR-5576: Improve concurrency when registering and waiting for all
SolrCore's to register a DOWN state. (Christine Poerschke via Mark Miller)
-
+
================== 4.6.0 ==================
Versions of Major Components
@@ -190,7 +392,7 @@ New Features
* SOLR-5464: Add option to ConcurrentSolrServer to stream pure delete
requests. (Mark Miller)
-
+
Bug Fixes
----------------------
@@ -234,7 +436,7 @@ Bug Fixes
* SOLR-5465: SolrCmdDistributor retry logic has a concurrency race bug.
(Mark Miller)
-* SOLR-5452: Do not attempt to proxy internal update requests. (Mark Miller)
+* SOLR-5452: Do not attempt to proxy internal update requests. (Mark Miller)
Optimizations
----------------------
Modified: lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1557811&r1=1557810&r2=1557811&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Mon Jan 13 18:58:52 2014
@@ -443,4 +443,8 @@ final class OverseerElectionContext exte
overseer.start(id);
}
+ public void cancelElection() throws InterruptedException, KeeperException {
+ super.cancelElection();
+ overseer.close();
+ }
}
Modified: lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1557811&r1=1557810&r2=1557811&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Mon Jan 13 18:58:52 2014
@@ -63,12 +63,18 @@ public class LeaderElector {
protected SolrZkClient zkClient;
private ZkCmdExecutor zkCmdExecutor;
-
+
+ private volatile ElectionContext context;
+
public LeaderElector(SolrZkClient zkClient) {
this.zkClient = zkClient;
zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout());
}
+ public ElectionContext getContext() {
+ return context;
+ }
+
/**
* Check if the candidate with the given n_* sequence number is the leader.
* If it is, set the leaderId on the leader zk node. If it is not, start
Modified: lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1557811&r1=1557810&r2=1557811&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene_solr_4_6/solr/core/src/java/org/apache/solr/cloud/ZkController.java Mon Jan 13 18:58:52 2014
@@ -137,13 +137,13 @@ public final class ZkController {
}
private final Map<ContextKey, ElectionContext> electionContexts = Collections.synchronizedMap(new HashMap<ContextKey, ElectionContext>());
- private SolrZkClient zkClient;
- private ZkCmdExecutor cmdExecutor;
- private ZkStateReader zkStateReader;
+ private final SolrZkClient zkClient;
+ private final ZkCmdExecutor cmdExecutor;
+ private final ZkStateReader zkStateReader;
- private LeaderElector leaderElector;
+ private final LeaderElector leaderElector;
- private String zkServerAddress; // example: 127.0.0.1:54062/solr
+ private final String zkServerAddress; // example: 127.0.0.1:54062/solr
private final String localHostPort; // example: 54065
private final String localHostContext; // example: solr
@@ -224,6 +224,11 @@ public final class ZkController {
ZkController.this.overseer = new Overseer(shardHandler, adminPath, zkStateReader);
ElectionContext context = new OverseerElectionContext(zkClient, overseer, getNodeName());
+ ElectionContext prevContext = overseerElector.getContext();
+ if (prevContext != null) {
+ prevContext.cancelElection();
+ }
+
overseerElector.joinElection(context, true);
zkStateReader.createClusterStateWatchersAndUpdate();
@@ -937,6 +942,17 @@ public final class ZkController {
private void joinElection(CoreDescriptor cd, boolean afterExpiration) throws InterruptedException, KeeperException, IOException {
+ // look for old context - if we find it, cancel it
+ String collection = cd.getCloudDescriptor().getCollectionName();
+ final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
+
+ ContextKey contextKey = new ContextKey(collection, coreNodeName);
+
+ ElectionContext prevContext = electionContexts.get(contextKey);
+
+ if (prevContext != null) {
+ prevContext.cancelElection();
+ }
String shardId = cd.getCloudDescriptor().getShardId();
@@ -946,16 +962,15 @@ public final class ZkController {
props.put(ZkStateReader.CORE_NAME_PROP, cd.getName());
props.put(ZkStateReader.NODE_NAME_PROP, getNodeName());
- final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
+
ZkNodeProps ourProps = new ZkNodeProps(props);
- String collection = cd.getCloudDescriptor()
- .getCollectionName();
+
ElectionContext context = new ShardLeaderElectionContext(leaderElector, shardId,
collection, coreNodeName, ourProps, this, cc);
leaderElector.setup(context);
- electionContexts.put(new ContextKey(collection, coreNodeName), context);
+ electionContexts.put(contextKey, context);
leaderElector.joinElection(context, false);
}
Modified: lucene/dev/branches/lucene_solr_4_6/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_6/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java?rev=1557811&r1=1557810&r2=1557811&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_6/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java (original)
+++ lucene/dev/branches/lucene_solr_4_6/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java Mon Jan 13 18:58:52 2014
@@ -34,10 +34,8 @@ public class ConnectionManager implement
.getLogger(ConnectionManager.class);
private final String name;
- private CountDownLatch clientConnected;
- private KeeperState state = KeeperState.Disconnected;
+ private CountDownLatch clientConnected = new CountDownLatch(1);
private boolean connected = false;
- private boolean likelyExpired = true;
private final ZkClientConnectionStrategy connectionStrategy;
@@ -47,7 +45,9 @@ public class ConnectionManager implement
private final OnReconnect onReconnect;
+ private volatile KeeperState state = KeeperState.Disconnected;
private volatile boolean isClosed = false;
+ private volatile boolean likelyExpired = true;
private volatile Timer disconnectedTimer;
@@ -57,16 +57,16 @@ public class ConnectionManager implement
this.connectionStrategy = strat;
this.zkServerAddress = zkServerAddress;
this.onReconnect = onConnect;
- clientConnected = new CountDownLatch(1);
}
private synchronized void connected() {
- connected = true;
if (disconnectedTimer != null) {
disconnectedTimer.cancel();
disconnectedTimer = null;
}
+ connected = true;
likelyExpired = false;
+ notifyAll();
}
private synchronized void disconnected() {
@@ -80,18 +80,17 @@ public class ConnectionManager implement
@Override
public void run() {
- synchronized (ConnectionManager.this) {
- likelyExpired = true;
- }
+ likelyExpired = true;
}
}, (long) (client.getZkClientTimeout() * 0.90));
}
connected = false;
+ notifyAll();
}
@Override
- public synchronized void process(WatchedEvent event) {
+ public void process(WatchedEvent event) {
if (log.isInfoEnabled()) {
log.info("Watcher " + this + " name:" + name + " got event " + event
+ " path:" + event.getPath() + " type:" + event.getType());
@@ -101,8 +100,9 @@ public class ConnectionManager implement
log.info("Client->ZooKeeper status change trigger but we are already closed");
return;
}
-
+
state = event.getState();
+
if (state == KeeperState.SyncConnected) {
connected();
clientConnected.countDown();
@@ -115,10 +115,12 @@ public class ConnectionManager implement
connected = false;
likelyExpired = true;
+
log.info("Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...");
try {
- connectionStrategy.reconnect(zkServerAddress, client.getZkClientTimeout(), this,
+ connectionStrategy.reconnect(zkServerAddress,
+ client.getZkClientTimeout(), this,
new ZkClientConnectionStrategy.ZkUpdate() {
@Override
public void update(SolrZooKeeper keeper) {
@@ -142,12 +144,22 @@ public class ConnectionManager implement
throw new RuntimeException(t);
}
+ connected();
+
if (onReconnect != null) {
- onReconnect.command();
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ onReconnect.command();
+ } catch (Exception e) {
+ log.warn("Exception running onReconnect command", e);
+ }
+ }
+ };
+ thread.start();
}
- connected();
-
}
});
} catch (Exception e) {
@@ -161,7 +173,6 @@ public class ConnectionManager implement
} else {
disconnected();
}
- notifyAll();
}
public synchronized boolean isConnected() {
@@ -180,12 +191,8 @@ public class ConnectionManager implement
this.disconnectedTimer = null;
}
}
-
- public synchronized KeeperState state() {
- return state;
- }
- public synchronized boolean isLikelyExpired() {
+ public boolean isLikelyExpired() {
return likelyExpired;
}
@@ -236,4 +243,4 @@ public class ConnectionManager implement
"", e);
}
}
-}
+}
\ No newline at end of file