You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ry...@apache.org on 2012/03/06 19:13:50 UTC
svn commit: r1297628 [12/13] - in /lucene/dev/branches/solr_3159_jetty8: ./
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/maven/lucene/contrib/demo/
dev-tools/maven/lucene/contrib/highlighter/
dev-tools/maven/lucene/contrib/memory/ dev-tools/maven...
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java Tue Mar 6 18:13:38 2012
@@ -117,7 +117,7 @@ public class AddUpdateCommand extends Up
@Override
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
- if (indexedId != null) sb.append(",id=").append(indexedId);
+ sb.append(",id=").append(getPrintableId());
if (!overwrite) sb.append(",overwrite=").append(overwrite);
if (commitWithin != -1) sb.append(",commitWithin=").append(commitWithin);
sb.append('}');
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Tue Mar 6 18:13:38 2012
@@ -74,10 +74,20 @@ public final class DefaultSolrCoreState
} else if (indexWriter != null) {
indexWriter.close();
}
+ } catch (Throwable t) {
+ log.error("Error during shutdown of writer.", t);
+ }
+ try {
+ directoryFactory.close();
+ } catch (Throwable t) {
+ log.error("Error during shutdown of directory factory.", t);
+ }
+ try {
+ cancelRecovery();
} catch (Throwable t) {
- SolrException.log(log, t);
+ log.error("Error cancelling recovery", t);
}
- directoryFactory.close();
+
closed = true;
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Tue Mar 6 18:13:38 2012
@@ -42,12 +42,19 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.FunctionRangeQuery;
import org.apache.solr.search.QParser;
@@ -524,30 +531,73 @@ public class DirectUpdateHandler2 extend
// IndexWriterCloser interface method - called from solrCoreState.decref(this)
@Override
public void closeWriter(IndexWriter writer) throws IOException {
+ boolean clearRequestInfo = false;
commitLock.lock();
try {
+ SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ if (SolrRequestInfo.getRequestInfo() == null) {
+ clearRequestInfo = true;
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); // important for debugging
+ }
+
+
if (!commitOnClose) {
if (writer != null) {
writer.rollback();
}
// we shouldn't close the transaction logs either, but leaving them open
- // means we can't delete them on windows.
+ // means we can't delete them on windows (needed for tests)
if (ulog != null) ulog.close(false);
return;
}
- if (writer != null) {
- writer.close();
+ // do a commit before we quit?
+ boolean tryToCommit = writer != null && ulog != null && ulog.hasUncommittedChanges() && ulog.getState() == UpdateLog.State.ACTIVE;
+
+ try {
+ if (tryToCommit) {
+
+ CommitUpdateCommand cmd = new CommitUpdateCommand(req, false);
+ cmd.openSearcher = false;
+ cmd.waitSearcher = false;
+ cmd.softCommit = false;
+
+ // TODO: keep other commit callbacks from being called?
+ // this.commit(cmd); // too many test failures using this method... is it because of callbacks?
+
+ synchronized (this) {
+ ulog.preCommit(cmd);
+ }
+
+ // todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't be used)
+ final Map<String,String> commitData = new HashMap<String,String>();
+ commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));
+ writer.commit(commitData);
+
+ synchronized (this) {
+ ulog.postCommit(cmd);
+ }
+ }
+ } catch (Throwable th) {
+ log.error("Error in final commit", th);
+ }
+
+ // we went through the normal process to commit, so we don't have to artificially
+ // cap any ulog files.
+ try {
+ if (ulog != null) ulog.close(false);
+ } catch (Throwable th) {
+ log.error("Error closing log files", th);
}
- // if the writer hits an exception, it's OK (and perhaps desirable)
- // to not close the ulog.
+ if (writer != null) writer.close();
- if (ulog != null) ulog.close(true);
} finally {
commitLock.unlock();
+ if (clearRequestInfo) SolrRequestInfo.clearRequestInfo();
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/PeerSync.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/PeerSync.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/PeerSync.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/PeerSync.java Tue Mar 6 18:13:38 2012
@@ -184,6 +184,12 @@ public class PeerSync {
log.info(msg() + "START replicas=" + replicas + " nUpdates=" + nUpdates);
+ // TODO: does it ever make sense to allow sync when buffering or applying buffered? Someone might request that we do it...
+ if (!(ulog.getState() == UpdateLog.State.ACTIVE || ulog.getState()==UpdateLog.State.REPLAYING)) {
+ log.error(msg() + "ERROR, update log not in ACTIVE or REPLAY state. " + ulog);
+ // return false;
+ }
+
if (debug) {
if (startingVersions != null) {
log.debug(msg() + "startingVersions=" + startingVersions.size() + " " + startingVersions);
@@ -396,7 +402,7 @@ public class PeerSync {
private boolean requestUpdates(ShardResponse srsp, List<Long> toRequest) {
String replica = srsp.getShardRequest().shards[0];
- log.info(msg() + "Requesting updates from " + replica + " versions=" + toRequest);
+ log.info(msg() + "Requesting updates from " + replica + "n=" + toRequest.size() + " versions=" + toRequest);
// reuse our original request object
ShardRequest sreq = srsp.getShardRequest();
@@ -426,6 +432,7 @@ public class PeerSync {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(DistributedUpdateProcessor.SEEN_LEADER, true);
+ // params.set("peersync",true); // debugging
SolrQueryRequest req = new LocalSolrQueryRequest(uhandler.core, params);
SolrQueryResponse rsp = new SolrQueryResponse();
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Tue Mar 6 18:13:38 2012
@@ -92,8 +92,8 @@ public class SolrCmdDistributor {
public void finish() {
// piggyback on any outstanding adds or deletes if possible.
- flushAdds(1, null, null);
- flushDeletes(1, null, null);
+ flushAdds(1);
+ flushDeletes(1);
checkResponses(true);
}
@@ -108,11 +108,11 @@ public class SolrCmdDistributor {
}
}
- public void distribAdd(AddUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams commitParams) throws IOException {
+ public void distribAdd(AddUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {
checkResponses(false);
// make sure any pending deletes are flushed
- flushDeletes(1, null, null);
+ flushDeletes(1);
// TODO: this is brittle
// need to make a clone since these commands may be reused
@@ -124,7 +124,7 @@ public class SolrCmdDistributor {
clone.setVersion(cmd.getVersion());
AddRequest addRequest = new AddRequest();
addRequest.cmd = clone;
- addRequest.params = commitParams;
+ addRequest.params = params;
for (Node node : nodes) {
List<AddRequest> alist = adds.get(node);
@@ -135,7 +135,7 @@ public class SolrCmdDistributor {
alist.add(addRequest);
}
- flushAdds(maxBufferedAddsPerServer, null, null);
+ flushAdds(maxBufferedAddsPerServer);
}
public void distribCommit(CommitUpdateCommand cmd, List<Node> nodes,
@@ -168,7 +168,7 @@ public class SolrCmdDistributor {
private void doDelete(DeleteUpdateCommand cmd, List<Node> nodes,
ModifiableSolrParams params) throws IOException {
- flushAdds(1, null, null);
+ flushAdds(1);
DeleteUpdateCommand clonedCmd = clone(cmd);
DeleteRequest deleteRequest = new DeleteRequest();
@@ -184,7 +184,7 @@ public class SolrCmdDistributor {
dlist.add(deleteRequest);
}
- flushDeletes(maxBufferedDeletesPerServer, null, null);
+ flushDeletes(maxBufferedDeletesPerServer);
}
void addCommit(UpdateRequestExt ureq, CommitUpdateCommand cmd) {
@@ -193,7 +193,7 @@ public class SolrCmdDistributor {
: AbstractUpdateRequest.ACTION.COMMIT, false, cmd.waitSearcher);
}
- boolean flushAdds(int limit, CommitUpdateCommand ccmd, ModifiableSolrParams commitParams) {
+ boolean flushAdds(int limit) {
// check for pending deletes
Set<Node> removeNodes = new HashSet<Node>();
@@ -205,8 +205,6 @@ public class SolrCmdDistributor {
UpdateRequestExt ureq = new UpdateRequestExt();
- addCommit(ureq, ccmd);
-
ModifiableSolrParams combinedParams = new ModifiableSolrParams();
for (AddRequest aReq : alist) {
@@ -216,7 +214,6 @@ public class SolrCmdDistributor {
ureq.add(cmd.solrDoc, cmd.commitWithin, cmd.overwrite);
}
- if (commitParams != null) combinedParams.add(commitParams);
if (ureq.getParams() == null) ureq.setParams(new ModifiableSolrParams());
ureq.getParams().add(combinedParams);
@@ -232,7 +229,7 @@ public class SolrCmdDistributor {
return true;
}
- boolean flushDeletes(int limit, CommitUpdateCommand ccmd, ModifiableSolrParams commitParams) {
+ boolean flushDeletes(int limit) {
// check for pending deletes
Set<Node> removeNodes = new HashSet<Node>();
@@ -242,8 +239,6 @@ public class SolrCmdDistributor {
if (dlist == null || dlist.size() < limit) return false;
UpdateRequestExt ureq = new UpdateRequestExt();
- addCommit(ureq, ccmd);
-
ModifiableSolrParams combinedParams = new ModifiableSolrParams();
for (DeleteRequest dReq : dlist) {
@@ -255,7 +250,6 @@ public class SolrCmdDistributor {
ureq.deleteByQuery(cmd.query);
}
- if (commitParams != null) combinedParams.add(commitParams);
if (ureq.getParams() == null) ureq
.setParams(new ModifiableSolrParams());
ureq.getParams().add(combinedParams);
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/UpdateLog.java Tue Mar 6 18:13:38 2012
@@ -28,6 +28,7 @@ import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
@@ -68,6 +69,11 @@ public class UpdateLog implements Plugin
public int deletes;
public int deleteByQuery;
public int errors;
+
+ @Override
+ public String toString() {
+ return "RecoveryInfo{adds="+adds+" deletes="+deletes+ " deleteByQuery="+deleteByQuery+" errors="+errors + " positionOfStart="+positionOfStart+"}";
+ }
}
@@ -80,7 +86,7 @@ public class UpdateLog implements Plugin
private TransactionLog tlog;
private TransactionLog prevTlog;
private Deque<TransactionLog> logs = new LinkedList<TransactionLog>(); // list of recent logs, newest first
- private TransactionLog newestLogOnStartup;
+ private LinkedList<TransactionLog> newestLogsOnStartup = new LinkedList<TransactionLog>();
private int numOldRecords; // number of records in the recent logs
private Map<BytesRef,LogPtr> map = new HashMap<BytesRef, LogPtr>();
@@ -166,16 +172,23 @@ public class UpdateLog implements Plugin
File f = new File(tlogDir, oldLogName);
try {
oldLog = new TransactionLog( f, null, true );
- addOldLog(oldLog);
+ addOldLog(oldLog, false); // don't remove old logs on startup since more than one may be uncapped.
} catch (Exception e) {
SolrException.log(log, "Failure to open existing log file (non fatal) " + f, e);
f.delete();
}
}
- newestLogOnStartup = oldLog;
+ // Record first two logs (oldest first) at startup for potential tlog recovery.
+ // It's possible that at abnormal shutdown both "tlog" and "prevTlog" were uncapped.
+ for (TransactionLog ll : logs) {
+ newestLogsOnStartup.addFirst(ll);
+ if (newestLogsOnStartup.size() >= 2) break;
+ }
+
versionInfo = new VersionInfo(uhandler, 256);
+ // TODO: these startingVersions assume that we successfully recover from all non-complete tlogs.
UpdateLog.RecentUpdates startingRecentUpdates = getRecentUpdates();
try {
startingVersions = startingRecentUpdates.getVersions(numRecordsToKeep);
@@ -201,7 +214,7 @@ public class UpdateLog implements Plugin
/* Takes over ownership of the log, keeping it until no longer needed
and then decrementing it's reference and dropping it.
*/
- private void addOldLog(TransactionLog oldLog) {
+ private void addOldLog(TransactionLog oldLog, boolean removeOld) {
if (oldLog == null) return;
numOldRecords += oldLog.numRecords();
@@ -212,7 +225,7 @@ public class UpdateLog implements Plugin
currRecords += tlog.numRecords();
}
- while (logs.size() > 0) {
+ while (removeOld && logs.size() > 0) {
TransactionLog log = logs.peekLast();
int nrec = log.numRecords();
// remove oldest log if we don't need it to keep at least numRecordsToKeep, or if
@@ -362,6 +375,10 @@ public class UpdateLog implements Plugin
prevMap2 = null;
}
+ public boolean hasUncommittedChanges() {
+ return tlog != null;
+ }
+
public void preCommit(CommitUpdateCommand cmd) {
synchronized (this) {
if (debug) {
@@ -380,18 +397,16 @@ public class UpdateLog implements Plugin
// since we're changing the log, we must change the map.
newMap();
+ if (prevTlog != null) {
+ globalStrings = prevTlog.getGlobalStrings();
+ }
+
// since document additions can happen concurrently with commit, create
// a new transaction log first so that we know the old one is definitely
// in the index.
prevTlog = tlog;
tlog = null;
id++;
-
- if (prevTlog != null) {
- globalStrings = prevTlog.getGlobalStrings();
- }
-
- addOldLog(prevTlog);
}
}
@@ -404,6 +419,8 @@ public class UpdateLog implements Plugin
// if we made it through the commit, write a commit command to the log
// TODO: check that this works to cap a tlog we were using to buffer so we don't replay on startup.
prevTlog.writeCommit(cmd);
+
+ addOldLog(prevTlog, true);
// the old log list will decref when no longer needed
// prevTlog.decref();
prevTlog = null;
@@ -556,26 +573,32 @@ public class UpdateLog implements Plugin
}
}
+
public Future<RecoveryInfo> recoverFromLog() {
recoveryInfo = new RecoveryInfo();
- if (newestLogOnStartup == null) return null;
- if (!newestLogOnStartup.try_incref()) return null; // log file was already closed
+ List<TransactionLog> recoverLogs = new ArrayList<TransactionLog>(1);
+ for (TransactionLog ll : newestLogsOnStartup) {
+ if (!ll.try_incref()) continue;
- // now that we've incremented the reference, the log shouldn't go away.
- try {
- if (newestLogOnStartup.endsWithCommit()) {
- newestLogOnStartup.decref();
- return null;
+ try {
+ if (ll.endsWithCommit()) {
+ ll.decref();
+ continue;
+ }
+ } catch (IOException e) {
+ log.error("Error inspecting tlog " + ll);
+ ll.decref();
+ continue;
}
- } catch (IOException e) {
- log.error("Error inspecting tlog " + newestLogOnStartup);
- newestLogOnStartup.decref();
- return null;
+
+ recoverLogs.add(ll);
}
+ if (recoverLogs.isEmpty()) return null;
+
ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<RecoveryInfo>(recoveryExecutor);
- LogReplayer replayer = new LogReplayer(newestLogOnStartup, false);
+ LogReplayer replayer = new LogReplayer(recoverLogs, false);
versionInfo.blockUpdates();
try {
@@ -584,8 +607,9 @@ public class UpdateLog implements Plugin
versionInfo.unblockUpdates();
}
- return cs.submit(replayer, recoveryInfo);
+ // At this point, we are guaranteed that any new updates coming in will see the state as "replaying"
+ return cs.submit(replayer, recoveryInfo);
}
@@ -600,6 +624,22 @@ public class UpdateLog implements Plugin
}
}
+
+ private void doClose(TransactionLog theLog, boolean writeCommit) {
+ if (theLog != null) {
+ if (writeCommit) {
+ // record a commit
+ log.info("Recording current closed for " + uhandler.core + " log=" + theLog);
+ CommitUpdateCommand cmd = new CommitUpdateCommand(new LocalSolrQueryRequest(uhandler.core, new ModifiableSolrParams((SolrParams)null)), false);
+ theLog.writeCommit(cmd);
+ }
+
+ theLog.deleteOnClose = false;
+ theLog.decref();
+ theLog.forceClose();
+ }
+ }
+
public void close(boolean committed) {
synchronized (this) {
try {
@@ -610,24 +650,11 @@ public class UpdateLog implements Plugin
// Don't delete the old tlogs, we want to be able to replay from them and retrieve old versions
- if (prevTlog != null) {
- prevTlog.deleteOnClose = false;
- prevTlog.decref();
- prevTlog.forceClose();
- }
- if (tlog != null) {
- if (committed) {
- // record a commit
- CommitUpdateCommand cmd = new CommitUpdateCommand(new LocalSolrQueryRequest(uhandler.core, new ModifiableSolrParams((SolrParams)null)), false);
- tlog.writeCommit(cmd);
- }
-
- tlog.deleteOnClose = false;
- tlog.decref();
- tlog.forceClose();
- }
+ doClose(prevTlog, committed);
+ doClose(tlog, committed);
for (TransactionLog log : logs) {
+ if (log == prevTlog || log == tlog) continue;
log.deleteOnClose = false;
log.decref();
log.forceClose();
@@ -887,7 +914,7 @@ public class UpdateLog implements Plugin
throw new RuntimeException("executor is not running...");
}
ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<RecoveryInfo>(recoveryExecutor);
- LogReplayer replayer = new LogReplayer(tlog, true);
+ LogReplayer replayer = new LogReplayer(Arrays.asList(new TransactionLog[]{tlog}), true);
return cs.submit(replayer, recoveryInfo);
}
@@ -907,31 +934,62 @@ public class UpdateLog implements Plugin
private RecoveryInfo recoveryInfo;
- // TODO: do we let the log replayer run across core reloads?
class LogReplayer implements Runnable {
- TransactionLog translog;
+ private Logger loglog = log; // set to something different?
+
+ List<TransactionLog> translogs;
TransactionLog.LogReader tlogReader;
boolean activeLog;
boolean finishing = false; // state where we lock out other updates and finish those updates that snuck in before we locked
+ boolean debug = loglog.isDebugEnabled();
-
- public LogReplayer(TransactionLog translog, boolean activeLog) {
- this.translog = translog;
+ public LogReplayer(List<TransactionLog> translogs, boolean activeLog) {
+ this.translogs = translogs;
this.activeLog = activeLog;
}
+
+
+ private SolrQueryRequest req;
+ private SolrQueryResponse rsp;
+
+
@Override
public void run() {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(DistributedUpdateProcessor.SEEN_LEADER, true);
+ req = new LocalSolrQueryRequest(uhandler.core, params);
+ rsp = new SolrQueryResponse();
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); // setting request info will help logging
+
try {
+ for (TransactionLog translog : translogs) {
+ doReplay(translog);
+ }
+ } catch (Throwable e) {
+ recoveryInfo.errors++;
+ SolrException.log(log,e);
+ } finally {
+ // change the state while updates are still blocked to prevent races
+ state = State.ACTIVE;
+ if (finishing) {
+ versionInfo.unblockUpdates();
+ }
+ }
- uhandler.core.log.warn("Starting log replay " + translog + " active="+activeLog + "starting pos=" + recoveryInfo.positionOfStart);
+ loglog.warn("Log replay finished. recoveryInfo=" + recoveryInfo);
- tlogReader = translog.getReader(recoveryInfo.positionOfStart);
+ if (testing_logReplayFinishHook != null) testing_logReplayFinishHook.run();
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(DistributedUpdateProcessor.SEEN_LEADER, true);
- SolrQueryRequest req = new LocalSolrQueryRequest(uhandler.core, params);
- SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrRequestInfo.clearRequestInfo();
+ }
+
+
+ public void doReplay(TransactionLog translog) {
+ try {
+ loglog.warn("Starting log replay " + translog + " active="+activeLog + " starting pos=" + recoveryInfo.positionOfStart);
+
+ tlogReader = translog.getReader(recoveryInfo.positionOfStart);
// NOTE: we don't currently handle a core reload during recovery. This would cause the core
// to change underneath us.
@@ -1003,6 +1061,8 @@ public class UpdateLog implements Plugin
cmd.solrDoc = sdoc;
cmd.setVersion(version);
cmd.setFlags(UpdateCommand.REPLAY | UpdateCommand.IGNORE_AUTOCOMMIT);
+ if (debug) log.debug("add " + cmd);
+
proc.processAdd(cmd);
break;
}
@@ -1014,6 +1074,7 @@ public class UpdateLog implements Plugin
cmd.setIndexedId(new BytesRef(idBytes));
cmd.setVersion(version);
cmd.setFlags(UpdateCommand.REPLAY | UpdateCommand.IGNORE_AUTOCOMMIT);
+ if (debug) log.debug("delete " + cmd);
proc.processDelete(cmd);
break;
}
@@ -1026,6 +1087,7 @@ public class UpdateLog implements Plugin
cmd.query = query;
cmd.setVersion(version);
cmd.setFlags(UpdateCommand.REPLAY | UpdateCommand.IGNORE_AUTOCOMMIT);
+ if (debug) log.debug("deleteByQuery " + cmd);
proc.processDelete(cmd);
break;
}
@@ -1041,20 +1103,20 @@ public class UpdateLog implements Plugin
}
if (rsp.getException() != null) {
- log.error("Exception replaying log", rsp.getException());
+ loglog.error("REPLAY_ERR: Exception replaying log", rsp.getException());
throw rsp.getException();
}
} catch (IOException ex) {
recoveryInfo.errors++;
- log.warn("IOException reading log", ex);
+ loglog.warn("REYPLAY_ERR: IOException reading log", ex);
// could be caused by an incomplete flush if recovering from log
} catch (ClassCastException cl) {
recoveryInfo.errors++;
- log.warn("Unexpected log entry or corrupt log. Entry=" + o, cl);
+ loglog.warn("REPLAY_ERR: Unexpected log entry or corrupt log. Entry=" + o, cl);
// would be caused by a corrupt transaction log
} catch (Throwable ex) {
recoveryInfo.errors++;
- log.warn("Exception replaying log", ex);
+ loglog.warn("REPLAY_ERR: Exception replaying log", ex);
// something wrong with the request?
}
}
@@ -1065,12 +1127,13 @@ public class UpdateLog implements Plugin
cmd.waitSearcher = true;
cmd.setFlags(UpdateCommand.REPLAY);
try {
+ if (debug) log.debug("commit " + cmd);
uhandler.commit(cmd); // this should cause a commit to be added to the incomplete log and avoid it being replayed again after a restart.
} catch (IOException ex) {
recoveryInfo.errors++;
- log.error("Replay exception: final commit.", ex);
+ loglog.error("Replay exception: final commit.", ex);
}
-
+
if (!activeLog) {
// if we are replaying an old tlog file, we need to add a commit to the end
// so we don't replay it again if we restart right after.
@@ -1081,29 +1144,16 @@ public class UpdateLog implements Plugin
proc.finish();
} catch (IOException ex) {
recoveryInfo.errors++;
- log.error("Replay exception: finish()", ex);
+ loglog.error("Replay exception: finish()", ex);
}
- tlogReader.close();
- translog.decref();
-
- } catch (Throwable e) {
- recoveryInfo.errors++;
- SolrException.log(log,e);
} finally {
- // change the state while updates are still blocked to prevent races
- state = State.ACTIVE;
- if (finishing) {
- versionInfo.unblockUpdates();
- }
+ if (tlogReader != null) tlogReader.close();
+ translog.decref();
}
-
- log.warn("Ending log replay " + tlogReader);
-
- if (testing_logReplayFinishHook != null) testing_logReplayFinishHook.run();
}
}
-
+
public void cancelApplyBufferedUpdates() {
this.cancelApplyBufferUpdate = true;
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.java Tue Mar 6 18:13:38 2012
@@ -34,7 +34,7 @@ import org.apache.commons.lang.StringUti
/**
* Concatenates multiple values for fields matching the specified
* conditions using a configurable <code>delimiter</code> which defaults
- * to "<code> ,</code>".
+ * to "<code>, </code>".
* <p>
* By default, this processor concatenates the values for any field name
* which according to the schema is <code>multiValued="false"</code>
@@ -45,7 +45,7 @@ import org.apache.commons.lang.StringUti
* For example, in the configuration below, any "single valued" string and
* text field which is found to contain multiple values <i>except</i> for
* the <code>primary_author</code> field will be concatenated using the
- * string "<code> ;</code>" as a delimeter. For the
+ * string "<code>; </code>" as a delimeter. For the
* <code>primary_author</code> field, the multiple values will be left
* alone for <code>FirstFieldValueUpdateProcessorFactory</code> to deal with.
* </p>
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Tue Mar 6 18:13:38 2012
@@ -60,11 +60,14 @@ import org.apache.solr.update.UpdateHand
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.VersionBucket;
import org.apache.solr.update.VersionInfo;
-import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
// NOT mt-safe... create a new processor for each add thread
// TODO: we really should not wait for distrib after local? unless a certain replication factor is asked for
public class DistributedUpdateProcessor extends UpdateRequestProcessor {
+ public final static Logger log = LoggerFactory.getLogger(DistributedUpdateProcessor.class);
+
public static final String SEEN_LEADER = "leader";
public static final String COMMIT_END_POINT = "commit_end_point";
public static final String DELETE_BY_QUERY_LEVEL = "dbq_level";
@@ -203,7 +206,7 @@ public class DistributedUpdateProcessor
return cloudState.getShard(hash, collection);
}
- // used for deleteByQyery to get the list of nodes this leader should forward to
+ // used for deleteByQuery to get the list of nodes this leader should forward to
private List<Node> setupRequest() {
List<Node> nodes = null;
String shardId = cloudDesc.getShardId();
@@ -269,6 +272,7 @@ public class DistributedUpdateProcessor
if (isLeader) {
params.set(SEEN_LEADER, true);
}
+ params.remove("commit"); // this will be distributed from the local commit
cmdDistrib.distribAdd(cmd, nodes, params);
}
@@ -489,6 +493,7 @@ public class DistributedUpdateProcessor
if (isLeader) {
params.set(SEEN_LEADER, true);
}
+ params.remove("commit"); // we already will have forwarded this from our local commit
cmdDistrib.distribDelete(cmd, nodes, params);
}
@@ -565,6 +570,7 @@ public class DistributedUpdateProcessor
}
}
+ params.remove("commit"); // this will be distributed from the local commit
cmdDistrib.distribDelete(cmd, leaders, params);
if (!leaderForAnyShard) {
@@ -618,20 +624,6 @@ public class DistributedUpdateProcessor
doLocalDelete(cmd);
- // forward to all replicas
- if (replicas != null) {
- ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
- params.set(DELETE_BY_QUERY_LEVEL, 3);
- params.set(VERSION_FIELD, Long.toString(cmd.getVersion()));
- params.set(SEEN_LEADER, "true");
- cmdDistrib.distribDelete(cmd, replicas, params);
-
- // wait for DBQ responses before releasing the update block to eliminate the possibility
- // of an add being reordered.
- // TODO: this isn't strictly necessary - we could do the same thing we do for PeerSync
- // in DUH2 and add a clause that prevents deleting older docs.
- cmdDistrib.finish();
- }
} else {
cmd.setVersion(-versionOnUpdate);
@@ -655,6 +647,20 @@ public class DistributedUpdateProcessor
vinfo.unblockUpdates();
}
+
+
+ // TODO: need to handle reorders to replicas somehow
+ // forward to all replicas
+ if (leaderLogic && replicas != null) {
+ ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
+ params.set(DELETE_BY_QUERY_LEVEL, 3);
+ params.set(VERSION_FIELD, Long.toString(cmd.getVersion()));
+ params.set(SEEN_LEADER, "true");
+ cmdDistrib.distribDelete(cmd, replicas, params);
+ cmdDistrib.finish();
+ }
+
+
if (returnVersions && rsp != null) {
if (deleteByQueryResponse == null) {
deleteByQueryResponse = new NamedList<String>();
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessor.java Tue Mar 6 18:13:38 2012
@@ -40,6 +40,8 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Reusable base class for UpdateProcessors that will consider
@@ -57,7 +59,8 @@ import org.apache.solr.update.AddUpdateC
*/
public abstract class FieldMutatingUpdateProcessor
extends UpdateRequestProcessor {
-
+ public final static Logger log = LoggerFactory.getLogger(FieldMutatingUpdateProcessor.class);
+
private final FieldNameSelector selector;
public FieldMutatingUpdateProcessor(FieldNameSelector selector,
UpdateRequestProcessor next) {
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java Tue Mar 6 18:13:38 2012
@@ -47,7 +47,7 @@ import org.apache.solr.util.plugin.SolrC
* <p>
* This class provides all of the plumbing for configuring the
* FieldNameSelector using the following init params to specify selection
- * critera...
+ * criteria...
* </p>
* <ul>
* <li><code>fieldName</code> - selecting specific fields by field name lookup</li>
@@ -57,10 +57,10 @@ import org.apache.solr.util.plugin.SolrC
* </ul>
*
* <p>
- * Each critera can specified as either an <arr> of <str>, or
+ * Each criteria can specified as either an <arr> of <str>, or
* multiple <str> with the same name. When multiple criteria of a
* single type exist, fields must match <b>at least one</b> to be selected.
- * If more then one type of critera exist, fields must match
+ * If more then one type of criteria exist, fields must match
* <b>at least one of each</b> to be selected.
* </p>
* <p>
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java Tue Mar 6 18:13:38 2012
@@ -46,7 +46,6 @@ import org.slf4j.LoggerFactory;
public class LogUpdateProcessorFactory extends UpdateRequestProcessorFactory {
int maxNumToLog = 10;
-
@Override
public void init( final NamedList args ) {
if( args != null ) {
@@ -57,20 +56,13 @@ public class LogUpdateProcessorFactory e
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
- final Logger logger = LoggerFactory.getLogger(LogUpdateProcessor.class);
- boolean doLog = logger.isInfoEnabled();
- // LogUpdateProcessor.log.error("Will Log=" + doLog);
- if( doLog ) {
- // only create the log processor if we will use it
- final LogUpdateProcessor processor = new LogUpdateProcessor(req, rsp, this, next);
- assert processor.log == logger;
- return processor;
- }
- return null;
+ return LogUpdateProcessor.log.isInfoEnabled() ? new LogUpdateProcessor(req, rsp, this, next) : null;
}
}
class LogUpdateProcessor extends UpdateRequestProcessor {
+ public final static Logger log = LoggerFactory.getLogger(LogUpdateProcessor.class);
+
private final SolrQueryRequest req;
private final SolrQueryResponse rsp;
private final NamedList<Object> toLog;
@@ -99,7 +91,7 @@ class LogUpdateProcessor extends UpdateR
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
- if (logDebug) { log.debug(cmd.toString()); }
+ if (logDebug) { log.debug("PRE_UPDATE " + cmd.toString()); }
// call delegate first so we can log things like the version that get set later
if (next != null) next.processAdd(cmd);
@@ -122,7 +114,7 @@ class LogUpdateProcessor extends UpdateR
@Override
public void processDelete( DeleteUpdateCommand cmd ) throws IOException {
- if (logDebug) { log.debug(cmd.toString()); }
+ if (logDebug) { log.debug("PRE_UPDATE " + cmd.toString()); }
if (next != null) next.processDelete(cmd);
if (cmd.isDeleteById()) {
@@ -150,7 +142,7 @@ class LogUpdateProcessor extends UpdateR
@Override
public void processMergeIndexes(MergeIndexesCommand cmd) throws IOException {
- if (logDebug) { log.debug(cmd.toString()); }
+ if (logDebug) { log.debug("PRE_UPDATE " + cmd.toString()); }
if (next != null) next.processMergeIndexes(cmd);
toLog.add("mergeIndexes", cmd.toString());
@@ -158,7 +150,7 @@ class LogUpdateProcessor extends UpdateR
@Override
public void processCommit( CommitUpdateCommand cmd ) throws IOException {
- if (logDebug) { log.debug(cmd.toString()); }
+ if (logDebug) { log.debug("PRE_UPDATE " + cmd.toString()); }
if (next != null) next.processCommit(cmd);
@@ -171,7 +163,7 @@ class LogUpdateProcessor extends UpdateR
*/
@Override
public void processRollback( RollbackUpdateCommand cmd ) throws IOException {
- if (logDebug) { log.debug(cmd.toString()); }
+ if (logDebug) { log.debug("PRE_UPDATE " + cmd.toString()); }
if (next != null) next.processRollback(cmd);
toLog.add("rollback", "");
@@ -180,16 +172,28 @@ class LogUpdateProcessor extends UpdateR
@Override
public void finish() throws IOException {
+ if (logDebug) { log.debug("PRE_UPDATE finish()"); }
if (next != null) next.finish();
- if (logDebug) { log.debug("finish"); }
// LOG A SUMMARY WHEN ALL DONE (INFO LEVEL)
- // TODO: right now, update requests are logged twice...
- // this will slow down things compared to Solr 1.2
- // we should have extra log info on the SolrQueryResponse, to
- // be logged by SolrCore
-
+
+
+ NamedList<Object> stdLog = rsp.getToLog();
+
+ StringBuilder sb = new StringBuilder(req.getCore().getLogId());
+
+ for (int i=0; i<stdLog.size(); i++) {
+ String name = stdLog.getName(i);
+ Object val = stdLog.getVal(i);
+ if (name != null) {
+ sb.append(name).append('=');
+ }
+ sb.append(val).append(' ');
+ }
+
+ stdLog.clear(); // make it so SolrCore.exec won't log this again
+
// if id lists were truncated, show how many more there were
if (adds != null && numAdds > maxNumToLog) {
adds.add("... (" + numAdds + " adds)");
@@ -198,7 +202,9 @@ class LogUpdateProcessor extends UpdateR
deletes.add("... (" + numDeletes + " deletes)");
}
long elapsed = rsp.getEndTime() - req.getStartTime();
- log.info( ""+toLog + " 0 " + (elapsed) );
+
+ sb.append(toLog).append(" 0 ").append(elapsed);
+ log.info(sb.toString());
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java Tue Mar 6 18:13:38 2012
@@ -41,8 +41,6 @@ import org.apache.solr.update.RollbackUp
* @since solr 1.3
*/
public abstract class UpdateRequestProcessor {
- protected final Logger log = LoggerFactory.getLogger(getClass());
-
protected final UpdateRequestProcessor next;
public UpdateRequestProcessor( UpdateRequestProcessor next) {
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorFactory.java Tue Mar 6 18:13:38 2012
@@ -39,6 +39,5 @@ public abstract class UpdateRequestProce
// could process the Node
}
- abstract public UpdateRequestProcessor getInstance(
- SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next );
+ abstract public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next );
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java Tue Mar 6 18:13:38 2012
@@ -551,7 +551,7 @@ public class SolrPluginUtils {
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c == '\\' || c == '!' || c == '(' || c == ')' ||
- c == ':' || c == '^' || c == '[' || c == ']' ||
+ c == ':' || c == '^' || c == '[' || c == ']' || c == '/' ||
c == '{' || c == '}' || c == '~' || c == '*' || c == '?'
) {
sb.append('\\');
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java Tue Mar 6 18:13:38 2012
@@ -139,7 +139,7 @@ public abstract class AbstractPluginLoad
String defaultStr = DOMUtil.getAttr(node,"default", null );
T plugin = create(loader, name, className, node );
- log.info("created " + ((name != null) ? name : "") + ": " + plugin.getClass().getName());
+ log.debug("created " + ((name != null) ? name : "") + ": " + plugin.getClass().getName());
// Either initialize now or wait till everything has been registered
if( preRegister ) {
@@ -209,7 +209,7 @@ public abstract class AbstractPluginLoad
String name = DOMUtil.getAttr(node, "name", requireName ? type : null);
String className = DOMUtil.getAttr(node, "class", type);
plugin = create(loader, name, className, node);
- log.info("created " + name + ": " + plugin.getClass().getName());
+ log.debug("created " + name + ": " + plugin.getClass().getName());
// Either initialize now or wait till everything has been registered
if (preRegister) {
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test-files/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test-files/solr/conf/schema.xml?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test-files/solr/conf/schema.xml (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test-files/solr/conf/schema.xml Tue Mar 6 18:13:38 2012
@@ -538,6 +538,11 @@
<field name="nullfirst" type="string" indexed="true" stored="true" sortMissingFirst="true" multiValued="false"/>
+
+ <field name="cat" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="price" type="float" indexed="true" stored="true"/>
+ <field name="inStock" type="boolean" indexed="true" stored="true" />
+
<field name="subword" type="subword" indexed="true" stored="true"/>
<field name="subword_offsets" type="subword" indexed="true" stored="true" termOffsets="true"/>
<field name="numericsubword" type="numericsubword" indexed="true" stored="true"/>
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java Tue Mar 6 18:13:38 2012
@@ -18,6 +18,7 @@ package org.apache.solr;
*/
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
/**
@@ -169,6 +170,9 @@ public class TestDistributedGrouping ext
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " desc", "group.sort", "score desc"); // SOLR-2955
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", "score desc, _docid_ asc, id asc");
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10);
+
+ // Can't validate the response, but can check if no errors occur.
+ simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc", CommonParams.TIME_ALLOWED, 1);
}
private void simpleQuery(Object... queryParams) throws SolrServerException {
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedSearch.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestDistributedSearch.java Tue Mar 6 18:13:38 2012
@@ -17,8 +17,17 @@
package org.apache.solr;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.ChaosMonkey;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
@@ -292,7 +301,32 @@ public class TestDistributedSearch exten
assertNotNull("missing shard info", sinfo);
assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, cnt, sinfo.size());
-
+ // test shards.tolerant=true
+ for(int numDownServers = 0; numDownServers < jettys.size()-1; numDownServers++)
+ {
+ List<JettySolrRunner> upJettys = new ArrayList<JettySolrRunner>(jettys);
+ List<SolrServer> upClients = new ArrayList<SolrServer>(clients);
+ List<JettySolrRunner> downJettys = new ArrayList<JettySolrRunner>();
+ List<String> upShards = new ArrayList<String>(Arrays.asList(shardsArr));
+ for(int i=0; i<numDownServers; i++)
+ {
+ // shut down some of the jettys
+ int indexToRemove = r.nextInt(upJettys.size());
+ JettySolrRunner downJetty = upJettys.remove(indexToRemove);
+ upClients.remove(indexToRemove);
+ upShards.remove(indexToRemove);
+ ChaosMonkey.stop(downJetty);
+ downJettys.add(downJetty);
+ }
+
+ queryPartialResults(upShards, upClients, "q","*:*",ShardParams.SHARDS_INFO,"true",ShardParams.SHARDS_TOLERANT,"true");
+
+ // restart the jettys
+ for (JettySolrRunner downJetty : downJettys) {
+ downJetty.start();
+ }
+ }
+
// This index has the same number for every field
// TODO: This test currently fails because debug info is obtained only
@@ -301,5 +335,90 @@ public class TestDistributedSearch exten
// Thread.sleep(10000000000L);
}
+
+ protected void queryPartialResults(final List<String> upShards, List<SolrServer> upClients, Object... q) throws Exception {
+
+ final ModifiableSolrParams params = new ModifiableSolrParams();
+
+ for (int i = 0; i < q.length; i += 2) {
+ params.add(q[i].toString(), q[i + 1].toString());
+ }
+ // TODO: look into why passing true causes fails
+ params.set("distrib", "false");
+ final QueryResponse controlRsp = controlClient.query(params);
+ validateControlData(controlRsp);
+
+ params.remove("distrib");
+ setDistributedParams(params);
+
+ QueryResponse rsp = queryRandomUpServer(params,upClients);
+
+ comparePartialResponses(rsp, controlRsp, upShards);
+
+ if (stress > 0) {
+ log.info("starting stress...");
+ Thread[] threads = new Thread[nThreads];
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ for (int j = 0; j < stress; j++) {
+ int which = r.nextInt(clients.size());
+ SolrServer client = clients.get(which);
+ try {
+ QueryResponse rsp = client.query(new ModifiableSolrParams(params));
+ if (verifyStress) {
+ comparePartialResponses(rsp, controlRsp, upShards);
+ }
+ } catch (SolrServerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+ threads[i].start();
+ }
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+ }
+ }
+ protected QueryResponse queryRandomUpServer(ModifiableSolrParams params, List<SolrServer> upClients) throws SolrServerException {
+ // query a random "up" server
+ int which = r.nextInt(upClients.size());
+ SolrServer client = upClients.get(which);
+ QueryResponse rsp = client.query(params);
+ return rsp;
+ }
+
+ protected void comparePartialResponses(QueryResponse rsp, QueryResponse controlRsp, List<String> upShards)
+ {
+ NamedList<?> sinfo = (NamedList<?>) rsp.getResponse().get(ShardParams.SHARDS_INFO);
+
+ assertNotNull("missing shard info", sinfo);
+ assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, shardsArr.length, sinfo.size());
+ // identify each one
+ for (Map.Entry<String,?> entry : sinfo) {
+ String shard = entry.getKey();
+ NamedList<?> info = (NamedList<?>) entry.getValue();
+ boolean found = false;
+ for(int i=0; i<shardsArr.length; i++) {
+ String s = shardsArr[i];
+ if (shard.contains(s)) {
+ found = true;
+ // make sure that it responded if it's up
+ if (upShards.contains(s)) {
+ assertTrue("Expected to find numFound in the up shard info",info.get("numFound") != null);
+ }
+ else {
+ assertTrue("Expected to find error in the down shard info",info.get("error") != null);
+ }
+ }
+ }
+ assertTrue("Couldn't find shard " + shard + " represented in shards info", found);
+ }
+ }
+
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Tue Mar 6 18:13:38 2012
@@ -215,6 +215,19 @@ public class TestGroupingSearch extends
}
@Test
+ public void testGroupingWithTimeAllowed() throws Exception {
+ assertU(add(doc("id", "1")));
+ assertU(add(doc("id", "2")));
+ assertU(add(doc("id", "3")));
+ assertU(add(doc("id", "4")));
+ assertU(add(doc("id", "5")));
+ assertU(commit());
+
+ // Just checking if no errors occur
+ assertJQ(req("q", "*:*", "group", "true", "group.query", "id:1", "group.query", "id:2", "timeAllowed", "1"));
+ }
+
+ @Test
public void testGroupingSortByFunction() throws Exception {
assertU(add(doc("id", "1", "value1_i", "1", "value2_i", "1", "store", "45.18014,-93.87742")));
assertU(add(doc("id", "2", "value1_i", "1", "value2_i", "2", "store", "45.18014,-93.87743")));
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java Tue Mar 6 18:13:38 2012
@@ -50,7 +50,7 @@ public class TestKuromojiTokenizerFactor
factory.inform(new SolrResourceLoader(null, null));
TokenStream ts = factory.create(new StringReader("ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢"));
assertTokenStreamContents(ts,
- new String[] { "ã·ãã¢", "ã½ããã¦ã§ã¢", "ã¨ã³ã¸ãã¢" }
+ new String[] { "ã·ãã¢", "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢", "ã½ããã¦ã§ã¢", "ã¨ã³ã¸ãã¢" }
);
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java Tue Mar 6 18:13:38 2012
@@ -21,6 +21,7 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.solr.SolrTestCaseJ4;
@@ -211,12 +212,12 @@ public class TestWordDelimiterFilterFact
TokenStream ts = factoryDefault.create(
new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "5", "400", "00", "540000", "at", "25", "interest", "rate", "interestrate" });
ts = factoryDefault.create(
new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo", "bar", "foobar" });
@@ -229,13 +230,13 @@ public class TestWordDelimiterFilterFact
ts = factoryCustom.create(
new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "$5,400.00", "at", "25%", "interest", "rate", "interestrate" });
/* test custom behavior with a char > 0x7F, because we had to make a larger byte[] */
ts = factoryCustom.create(
new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo\u200Dbar" });
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java Tue Mar 6 18:13:38 2012
@@ -27,9 +27,9 @@ import org.apache.solr.common.cloud.Slic
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.zookeeper.KeeperException;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Before;
public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearchTestCase {
@@ -80,6 +80,14 @@ public abstract class AbstractDistribute
}
shards = sb.toString();
+
+ // now wait till we see the leader for each shard
+ for (int i = 1; i <= numShards; i++) {
+ ZkStateReader zkStateReader = ((SolrDispatchFilter) jettys.get(0)
+ .getDispatchFilter().getFilter()).getCores().getZkController()
+ .getZkStateReader();
+ zkStateReader.getLeaderProps("collection1", "shard" + (i + 2), 15000);
+ }
}
protected void waitForRecoveriesToFinish(String collection, ZkStateReader zkStateReader, boolean verbose)
@@ -180,8 +188,4 @@ public abstract class AbstractDistribute
zkClient.printLayoutToStdOut();
zkClient.close();
}
-
- @AfterClass
- public static void afterClass() throws InterruptedException {
- }
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java Tue Mar 6 18:13:38 2012
@@ -24,10 +24,8 @@ import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.core.SolrConfig;
import org.apache.zookeeper.CreateMode;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -61,7 +59,7 @@ public abstract class AbstractZkTestCase
System.setProperty("solrcloud.skip.autorecovery", "true");
System.setProperty("zkHost", zkServer.getZkAddress());
- System.setProperty("hostPort", "0000");
+ System.setProperty("jetty.port", "0000");
buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(),
"solrconfig.xml", "schema.xml");
@@ -120,6 +118,8 @@ public abstract class AbstractZkTestCase
System.clearProperty("solr.test.sys.prop1");
System.clearProperty("solr.test.sys.prop2");
System.clearProperty("solrcloud.skip.autorecovery");
+ System.clearProperty("jetty.port");
+
zkServer.shutdown();
// wait just a bit for any zk client threads to outlast timeout
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Tue Mar 6 18:13:38 2012
@@ -39,13 +39,17 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.update.SolrCmdDistributor.Request;
import org.apache.solr.util.DefaultSolrThreadFactory;
@@ -274,12 +278,46 @@ public class BasicDistributedZkTest exte
testANewCollectionInOneInstance();
testSearchByCollectionName();
testANewCollectionInOneInstanceWithManualShardAssignement();
+ testNumberOfCommitsWithCommitAfterAdd();
+
// Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
}
}
+ private void testNumberOfCommitsWithCommitAfterAdd()
+ throws MalformedURLException, SolrServerException, IOException {
+ long startCommits = getNumCommits((CommonsHttpSolrServer) clients.get(0));
+
+ ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update/csv");
+ up.addFile(getFile("books_numeric_ids.csv"));
+ up.setCommitWithin(900000);
+ up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+ NamedList<Object> result = clients.get(0).request(up);
+
+ long endCommits = getNumCommits((CommonsHttpSolrServer) clients.get(0));
+
+ assertEquals(startCommits + 1L, endCommits);
+ }
+
+ private Long getNumCommits(CommonsHttpSolrServer solrServer) throws MalformedURLException,
+ SolrServerException, IOException {
+ CommonsHttpSolrServer server = new CommonsHttpSolrServer(solrServer.getBaseURL());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("qt", "/admin/mbeans?key=updateHandler&stats=true");
+ // use generic request to avoid extra processing of queries
+ QueryRequest req = new QueryRequest(params);
+ NamedList<Object> resp = server.request(req);
+ NamedList mbeans = (NamedList) resp.get("solr-mbeans");
+ NamedList uhandlerCat = (NamedList) mbeans.get("UPDATEHANDLER");
+ NamedList uhandler = (NamedList) uhandlerCat.get("updateHandler");
+ NamedList stats = (NamedList) uhandler.get("stats");
+ Long commits = (Long) stats.get("commits");
+
+ return commits;
+ }
+
private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java Tue Mar 6 18:13:38 2012
@@ -46,9 +46,13 @@ public class BasicZkTest extends Abstrac
@Test
public void testBasic() throws Exception {
+
// test using ZooKeeper
assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware());
+ // for the really slow/busy computer, we wait to make sure we have a leader before starting
+ h.getCoreContainer().getZkController().getZkStateReader().getLeaderUrl("collection1", "shard1", 30000);
+
ZkController zkController = h.getCoreContainer().getZkController();
// test merge factor picked up
@@ -154,6 +158,7 @@ public class BasicZkTest extends Abstrac
}
+ zkController.getZkClient().printLayoutToStdOut();
}
public SolrQueryRequest request(String... q) {
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Tue Mar 6 18:13:38 2012
@@ -65,6 +65,7 @@ public class ChaosMonkeyNothingIsSafeTes
@Override
@After
public void tearDown() throws Exception {
+ System.clearProperty("numShards");
super.tearDown();
resetExceptionIgnores();
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Tue Mar 6 18:13:38 2012
@@ -61,6 +61,7 @@ public class ChaosMonkeySafeLeaderTest e
@Override
@After
public void tearDown() throws Exception {
+ System.clearProperty("numShards");
super.tearDown();
resetExceptionIgnores();
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Tue Mar 6 18:13:38 2012
@@ -60,7 +60,12 @@ import org.junit.Ignore;
*/
@Ignore
public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
-
+ @BeforeClass
+ public static void beforeFullSolrCloudTest() throws Exception {
+ // shorten the log output more for this test type
+ if (formatter != null) formatter.setShorterFormat();
+ }
+
private static final String SHARD2 = "shard2";
private boolean printLayoutOnTearDown = false;
@@ -138,7 +143,7 @@ public class FullSolrCloudTest extends A
@Override
public void setUp() throws Exception {
super.setUp();
- ignoreException(".*");
+ // ignoreException(".*");
System.setProperty("numShards", Integer.toString(sliceCount));
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java Tue Mar 6 18:13:38 2012
@@ -75,6 +75,8 @@ public class LeaderElectionIntegrationTe
super.setUp();
createTempDir();
ignoreException("No UpdateLog found - cannot sync");
+ ignoreException("No UpdateLog found - cannot recover");
+
System.setProperty("zkClientTimeout", "3000");
zkDir = dataDir.getAbsolutePath() + File.separator
@@ -269,6 +271,11 @@ public class LeaderElectionIntegrationTe
@AfterClass
public static void afterClass() throws InterruptedException {
System.clearProperty("solrcloud.skip.autorecovery");
+ System.clearProperty("zkClientTimeout");
+ System.clearProperty("zkHost");
+ System.clearProperty("shard");
+ System.clearProperty("solr.data.dir");
+ System.clearProperty("solr.solr.home");
resetExceptionIgnores();
// wait just a bit for any zk client threads to outlast timeout
Thread.sleep(2000);
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java Tue Mar 6 18:13:38 2012
@@ -110,7 +110,7 @@ public class LeaderElectionTest extends
try {
elector.setup(context);
- seq = elector.joinElection(context, null);
+ seq = elector.joinElection(context);
electionDone = true;
seqToThread.put(seq, this);
} catch (InterruptedException e) {
@@ -153,7 +153,7 @@ public class LeaderElectionTest extends
ElectionContext context = new ShardLeaderElectionContextBase(elector,
"shard2", "collection1", "dummynode1", props, zkStateReader);
elector.setup(context);
- elector.joinElection(context, null);
+ elector.joinElection(context);
assertEquals("http://127.0.0.1/solr/",
getLeaderUrl("collection1", "shard2"));
}
@@ -166,7 +166,7 @@ public class LeaderElectionTest extends
ElectionContext firstContext = new ShardLeaderElectionContextBase(first,
"slice1", "collection2", "dummynode1", props, zkStateReader);
first.setup(firstContext);
- first.joinElection(firstContext, null);
+ first.joinElection(firstContext);
Thread.sleep(1000);
assertEquals("original leader was not registered", "http://127.0.0.1/solr/1/", getLeaderUrl("collection2", "slice1"));
@@ -177,7 +177,7 @@ public class LeaderElectionTest extends
ElectionContext context = new ShardLeaderElectionContextBase(second,
"slice1", "collection2", "dummynode1", props, zkStateReader);
second.setup(context);
- second.joinElection(context, null);
+ second.joinElection(context);
Thread.sleep(1000);
assertEquals("original leader should have stayed leader", "http://127.0.0.1/solr/1/", getLeaderUrl("collection2", "slice1"));
firstContext.cancelElection();
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/NodeStateWatcherTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/NodeStateWatcherTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/NodeStateWatcherTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/NodeStateWatcherTest.java Tue Mar 6 18:13:38 2012
@@ -98,7 +98,7 @@ public class NodeStateWatcherTest extend
waitForCall(4, callCounter);
assertEquals(0, watcher.getCurrentState().size());
} finally {
-
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
if (zkClient != null) {
zkClient.close();
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Tue Mar 6 18:13:38 2012
@@ -142,7 +142,7 @@ public class OverseerTest extends SolrTe
ShardLeaderElectionContextBase ctx = new ShardLeaderElectionContextBase(
elector, shardId, collection, nodeName + "_" + coreName, props,
zkStateReader);
- elector.joinElection(ctx, null);
+ elector.joinElection(ctx);
break;
}
Thread.sleep(200);
@@ -218,7 +218,7 @@ public class OverseerTest extends SolrTe
collection1Desc.setCollectionName("collection1");
CoreDescriptor desc1 = new CoreDescriptor(null, "core" + (i + 1), "");
desc1.setCloudDescriptor(collection1Desc);
- zkController.preRegisterSetup(null, desc1, false);
+ zkController.preRegister(desc1);
ids[i] = zkController.register("core" + (i + 1), desc1);
}
@@ -237,6 +237,8 @@ public class OverseerTest extends SolrTe
assertNotNull(reader.getLeaderUrl("collection1", "shard3", 15000));
} finally {
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ System.clearProperty("bootstrap_confdir");
if (DEBUG) {
if (zkController != null) {
zkClient.printLayoutToStdOut();
@@ -250,8 +252,6 @@ public class OverseerTest extends SolrTe
}
server.shutdown();
}
-
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
}
@Test
@@ -318,7 +318,7 @@ public class OverseerTest extends SolrTe
final CoreDescriptor desc = new CoreDescriptor(null, coreName, "");
desc.setCloudDescriptor(collection1Desc);
try {
- controllers[slot % nodeCount].preRegisterSetup(null, desc, false);
+ controllers[slot % nodeCount].preRegister(desc);
ids[slot] = controllers[slot % nodeCount]
.register(coreName, desc);
} catch (Throwable e) {
@@ -394,6 +394,8 @@ public class OverseerTest extends SolrTe
}
} finally {
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ System.clearProperty("bootstrap_confdir");
if (DEBUG) {
if (controllers[0] != null) {
zkClient.printLayoutToStdOut();
@@ -414,8 +416,6 @@ public class OverseerTest extends SolrTe
nodeExecutors[i].shutdownNow();
}
}
-
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
}
//wait until collections are available
@@ -870,7 +870,7 @@ public class OverseerTest extends SolrTe
LeaderElector overseerElector = new LeaderElector(zkClient);
ElectionContext ec = new OverseerElectionContext(address.replaceAll("/", "_"), zkClient, reader);
overseerElector.setup(ec);
- overseerElector.joinElection(ec, null);
+ overseerElector.joinElection(ec);
return zkClient;
}
}
\ No newline at end of file
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Tue Mar 6 18:13:38 2012
@@ -190,7 +190,7 @@ public class ZkControllerTest extends So
collection1Desc.setCollectionName("collection1");
CoreDescriptor desc1 = new CoreDescriptor(null, "core" + (i + 1), "");
desc1.setCloudDescriptor(collection1Desc);
- zkController.preRegisterSetup(null, desc1, false);
+ zkController.preRegister(desc1);
ids[i] = zkController.register("core" + (i + 1), desc1);
}
@@ -220,9 +220,12 @@ public class ZkControllerTest extends So
assertNotNull("New leader was null.",
reader.getLeaderUrl("collection1", "shard1", 15000));
- Thread.sleep(1000);
+ Thread.sleep(2000);
assertEquals("shard was not unregistered", 1, zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size());
} finally {
+ System.clearProperty("solrcloud.skip.autorecovery");
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ System.clearProperty("bootstrap_confdir");
if (DEBUG) {
if (zkController != null) {
zkClient.printLayoutToStdOut();
@@ -235,8 +238,6 @@ public class ZkControllerTest extends So
zkController.close();
}
server.shutdown();
- System.clearProperty("solrcloud.skip.autorecovery");
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
}
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java Tue Mar 6 18:13:38 2012
@@ -20,6 +20,7 @@ package org.apache.solr.cloud;
import java.io.File;
import java.util.concurrent.atomic.AtomicInteger;
+import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -91,7 +92,7 @@ public class ZkSolrClientTest extends Ab
try {
zkClient.makePath("collections/collection2", false);
- TestCase.fail("Server should be down here");
+ Assert.fail("Server should be down here");
} catch (KeeperException.ConnectionLossException e) {
}
Modified: lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java?rev=1297628&r1=1297627&r2=1297628&view=diff
==============================================================================
--- lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (original)
+++ lucene/dev/branches/solr_3159_jetty8/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java Tue Mar 6 18:13:38 2012
@@ -71,7 +71,7 @@ public class TestArbitraryIndexDir exten
+ System.getProperty("file.separator") + "data");
dataDir.mkdirs();
- solrConfig = h.createConfig("solrconfig.xml");
+ solrConfig = TestHarness.createConfig("solrconfig.xml");
h = new TestHarness( dataDir.getAbsolutePath(),
solrConfig,
"schema12.xml");