You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rj...@apache.org on 2015/03/31 07:22:50 UTC
svn commit: r1670257 [25/39] - in /lucene/dev/branches/lucene6271: ./
dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneou...
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Tue Mar 31 05:22:40 2015
@@ -36,9 +36,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.zip.Adler32;
import java.util.zip.Checksum;
import java.util.zip.DeflaterOutputStream;
@@ -60,6 +64,7 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.FastOutputStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -75,6 +80,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.SolrIndexWriter;
+import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.NumberUtils;
import org.apache.solr.util.PropertiesInputStream;
import org.apache.solr.util.RefCounted;
@@ -90,8 +96,8 @@ import org.slf4j.LoggerFactory;
* file (command=filecontent&file=<FILE_NAME>) You can optionally specify an offset and length to get that
* chunk of the file. You can request a configuration file by using "cf" parameter instead of the "file" parameter.</li>
* <li>Get status/statistics (command=details)</li> </ol> <p>When running on the slave, it provides the following
- * commands <ol> <li>Perform a snap pull now (command=snappull)</li> <li>Get status/statistics (command=details)</li>
- * <li>Abort a snap pull (command=abort)</li> <li>Enable/Disable polling the master for new versions (command=enablepoll
+ * commands <ol> <li>Perform an index fetch now (command=snappull)</li> <li>Get status/statistics (command=details)</li>
+ * <li>Abort an index fetch (command=abort)</li> <li>Enable/Disable polling the master for new versions (command=enablepoll
* or command=disablepoll)</li> </ol>
*
*
@@ -134,9 +140,9 @@ public class ReplicationHandler extends
}
}
- private SnapPuller snapPuller;
+ private IndexFetcher pollingIndexFetcher;
- private ReentrantLock snapPullLock = new ReentrantLock();
+ private ReentrantLock indexFetchLock = new ReentrantLock();
private String includeConfFiles;
@@ -151,14 +157,18 @@ public class ReplicationHandler extends
private boolean replicateOnCommit = false;
private boolean replicateOnStart = false;
-
+
+ private ScheduledExecutorService executorService;
+
+ private volatile long executorStartTime;
+
private int numberBackupsToKeep = 0; //zero: do not delete old backups
private int numTimesReplicated = 0;
private final Map<String, FileInfo> confFileInfoCache = new HashMap<>();
- private Integer reserveCommitDuration = SnapPuller.readInterval("00:00:10");
+ private Integer reserveCommitDuration = readInterval("00:00:10");
volatile IndexCommit indexCommitPoint;
@@ -166,6 +176,19 @@ public class ReplicationHandler extends
private AtomicBoolean replicationEnabled = new AtomicBoolean(true);
+ private Integer pollInterval;
+
+ private String pollIntervalStr;
+
+ /**
+ * Disable the timer task for polling
+ */
+ private AtomicBoolean pollDisabled = new AtomicBoolean(false);
+
+ String getPollInterval() {
+ return pollIntervalStr;
+ }
+
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
rsp.setHttpCaching(false);
@@ -221,38 +244,38 @@ public class ReplicationHandler extends
return;
}
final SolrParams paramsCopy = new ModifiableSolrParams(solrParams);
- Thread puller = new Thread("explicit-fetchindex-cmd") {
+ Thread fetchThread = new Thread("explicit-fetchindex-cmd") {
@Override
public void run() {
doFetch(paramsCopy, false);
}
};
- puller.setDaemon(false);
- puller.start();
+ fetchThread.setDaemon(false);
+ fetchThread.start();
if (solrParams.getBool(WAIT, false)) {
- puller.join();
+ fetchThread.join();
}
rsp.add(STATUS, OK_STATUS);
} else if (command.equalsIgnoreCase(CMD_DISABLE_POLL)) {
- if (snapPuller != null){
- snapPuller.disablePoll();
+ if (pollingIndexFetcher != null){
+ disablePoll();
rsp.add(STATUS, OK_STATUS);
} else {
rsp.add(STATUS, ERR_STATUS);
rsp.add("message","No slave configured");
}
} else if (command.equalsIgnoreCase(CMD_ENABLE_POLL)) {
- if (snapPuller != null){
- snapPuller.enablePoll();
+ if (pollingIndexFetcher != null){
+ enablePoll();
rsp.add(STATUS, OK_STATUS);
}else {
rsp.add(STATUS,ERR_STATUS);
rsp.add("message","No slave configured");
}
} else if (command.equalsIgnoreCase(CMD_ABORT_FETCH)) {
- SnapPuller temp = tempSnapPuller;
- if (temp != null){
- temp.abortPull();
+ IndexFetcher fetcher = currentIndexFetcher;
+ if (fetcher != null){
+ fetcher.abortFetch();
rsp.add(STATUS, OK_STATUS);
} else {
rsp.add(STATUS,ERR_STATUS);
@@ -321,38 +344,35 @@ public class ReplicationHandler extends
return null;
}
- private volatile SnapPuller tempSnapPuller;
+ private volatile IndexFetcher currentIndexFetcher;
public boolean doFetch(SolrParams solrParams, boolean forceReplication) {
String masterUrl = solrParams == null ? null : solrParams.get(MASTER_URL);
- if (!snapPullLock.tryLock())
+ if (!indexFetchLock.tryLock())
return false;
try {
if (masterUrl != null) {
- if (tempSnapPuller != null && tempSnapPuller != snapPuller) {
- tempSnapPuller.destroy();
+ if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) {
+ currentIndexFetcher.destroy();
}
-
- NamedList<Object> nl = solrParams.toNamedList();
- nl.remove(SnapPuller.POLL_INTERVAL);
- tempSnapPuller = new SnapPuller(nl, this, core);
+ currentIndexFetcher = new IndexFetcher(solrParams.toNamedList(), this, core);
} else {
- tempSnapPuller = snapPuller;
+ currentIndexFetcher = pollingIndexFetcher;
}
- return tempSnapPuller.fetchLatestIndex(core, forceReplication);
+ return currentIndexFetcher.fetchLatestIndex(core, forceReplication);
} catch (Exception e) {
- SolrException.log(LOG, "SnapPull failed ", e);
+ SolrException.log(LOG, "Index fetch failed ", e);
} finally {
- if (snapPuller != null) {
- tempSnapPuller = snapPuller;
+ if (pollingIndexFetcher != null) {
+ currentIndexFetcher = pollingIndexFetcher;
}
- snapPullLock.unlock();
+ indexFetchLock.unlock();
}
return false;
}
boolean isReplicating() {
- return snapPullLock.isLocked();
+ return indexFetchLock.isLocked();
}
private void doSnapShoot(SolrParams params, SolrQueryResponse rsp,
@@ -390,10 +410,10 @@ public class ReplicationHandler extends
/**
* This method adds an Object of FileStream to the response . The FileStream implements a custom protocol which is
- * understood by SnapPuller.FileFetcher
+ * understood by IndexFetcher.FileFetcher
*
- * @see org.apache.solr.handler.SnapPuller.LocalFsFileFetcher
- * @see org.apache.solr.handler.SnapPuller.DirectoryFileFetcher
+ * @see IndexFetcher.LocalFsFileFetcher
+ * @see IndexFetcher.DirectoryFileFetcher
*/
private void getFileStream(SolrParams solrParams, SolrQueryResponse rsp) {
ModifiableSolrParams rawParams = new ModifiableSolrParams(solrParams);
@@ -538,18 +558,28 @@ public class ReplicationHandler extends
}
void disablePoll() {
- if (isSlave)
- snapPuller.disablePoll();
+ if (isSlave) {
+ pollDisabled.set(true);
+ LOG.info("inside disable poll, value of pollDisabled = " + pollDisabled);
+ }
}
void enablePoll() {
- if (isSlave)
- snapPuller.enablePoll();
+ if (isSlave) {
+ pollDisabled.set(false);
+ LOG.info("inside enable poll, value of pollDisabled = " + pollDisabled);
+ }
}
boolean isPollingDisabled() {
- if (snapPuller == null) return true;
- return snapPuller.isPollingDisabled();
+ return pollDisabled.get();
+ }
+
+ Long getNextScheduledExecTime() {
+ Long nextTime = null;
+ if (executorStartTime > 0)
+ nextTime = executorStartTime + pollInterval;
+ return nextTime;
}
int getTimesReplicatedSinceStartup() {
@@ -611,31 +641,31 @@ public class ReplicationHandler extends
list.add("isMaster", String.valueOf(isMaster));
list.add("isSlave", String.valueOf(isSlave));
- SnapPuller snapPuller = tempSnapPuller;
- if (snapPuller != null) {
- list.add(MASTER_URL, snapPuller.getMasterUrl());
- if (snapPuller.getPollInterval() != null) {
- list.add(SnapPuller.POLL_INTERVAL, snapPuller.getPollInterval());
+ IndexFetcher fetcher = currentIndexFetcher;
+ if (fetcher != null) {
+ list.add(MASTER_URL, fetcher.getMasterUrl());
+ if (getPollInterval() != null) {
+ list.add(POLL_INTERVAL, getPollInterval());
}
list.add("isPollingDisabled", String.valueOf(isPollingDisabled()));
list.add("isReplicating", String.valueOf(isReplicating()));
- long elapsed = getTimeElapsed(snapPuller);
- long val = SnapPuller.getTotalBytesDownloaded(snapPuller);
+ long elapsed = fetcher.getReplicationTimeElapsed();
+ long val = fetcher.getTotalBytesDownloaded();
if (elapsed > 0) {
list.add("timeElapsed", elapsed);
list.add("bytesDownloaded", val);
list.add("downloadSpeed", val / elapsed);
}
Properties props = loadReplicationProperties();
- addVal(list, SnapPuller.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class);
- addVal(list, SnapPuller.INDEX_REPLICATED_AT, props, Date.class);
- addVal(list, SnapPuller.CONF_FILES_REPLICATED_AT, props, Date.class);
- addVal(list, SnapPuller.REPLICATION_FAILED_AT, props, Date.class);
- addVal(list, SnapPuller.TIMES_FAILED, props, Integer.class);
- addVal(list, SnapPuller.TIMES_INDEX_REPLICATED, props, Integer.class);
- addVal(list, SnapPuller.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class);
- addVal(list, SnapPuller.TIMES_CONFIG_REPLICATED, props, Integer.class);
- addVal(list, SnapPuller.CONF_FILES_REPLICATED, props, String.class);
+ addVal(list, IndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class);
+ addVal(list, IndexFetcher.INDEX_REPLICATED_AT, props, Date.class);
+ addVal(list, IndexFetcher.CONF_FILES_REPLICATED_AT, props, Date.class);
+ addVal(list, IndexFetcher.REPLICATION_FAILED_AT, props, Date.class);
+ addVal(list, IndexFetcher.TIMES_FAILED, props, Integer.class);
+ addVal(list, IndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class);
+ addVal(list, IndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class);
+ addVal(list, IndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class);
+ addVal(list, IndexFetcher.CONF_FILES_REPLICATED, props, String.class);
}
if (isMaster) {
if (includeConfFiles != null) list.add("confFilesToReplicate", includeConfFiles);
@@ -677,12 +707,12 @@ public class ReplicationHandler extends
master.add("replicableGeneration", repCommitInfo.generation);
}
- SnapPuller snapPuller = tempSnapPuller;
- if (snapPuller != null) {
+ IndexFetcher fetcher = currentIndexFetcher;
+ if (fetcher != null) {
Properties props = loadReplicationProperties();
if (showSlaveDetails) {
try {
- NamedList nl = snapPuller.getDetails();
+ NamedList nl = fetcher.getDetails();
slave.add("masterDetails", nl.get(CMD_DETAILS));
} catch (Exception e) {
LOG.warn(
@@ -691,26 +721,26 @@ public class ReplicationHandler extends
slave.add(ERR_STATUS, "invalid_master");
}
}
- slave.add(MASTER_URL, snapPuller.getMasterUrl());
- if (snapPuller.getPollInterval() != null) {
- slave.add(SnapPuller.POLL_INTERVAL, snapPuller.getPollInterval());
+ slave.add(MASTER_URL, fetcher.getMasterUrl());
+ if (getPollInterval() != null) {
+ slave.add(POLL_INTERVAL, getPollInterval());
}
- if (snapPuller.getNextScheduledExecTime() != null && !isPollingDisabled()) {
- slave.add(NEXT_EXECUTION_AT, new Date(snapPuller.getNextScheduledExecTime()).toString());
+ if (getNextScheduledExecTime() != null && !isPollingDisabled()) {
+ slave.add(NEXT_EXECUTION_AT, new Date(getNextScheduledExecTime()).toString());
} else if (isPollingDisabled()) {
slave.add(NEXT_EXECUTION_AT, "Polling disabled");
}
- addVal(slave, SnapPuller.INDEX_REPLICATED_AT, props, Date.class);
- addVal(slave, SnapPuller.INDEX_REPLICATED_AT_LIST, props, List.class);
- addVal(slave, SnapPuller.REPLICATION_FAILED_AT_LIST, props, List.class);
- addVal(slave, SnapPuller.TIMES_INDEX_REPLICATED, props, Integer.class);
- addVal(slave, SnapPuller.CONF_FILES_REPLICATED, props, Integer.class);
- addVal(slave, SnapPuller.TIMES_CONFIG_REPLICATED, props, Integer.class);
- addVal(slave, SnapPuller.CONF_FILES_REPLICATED_AT, props, Integer.class);
- addVal(slave, SnapPuller.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class);
- addVal(slave, SnapPuller.TIMES_FAILED, props, Integer.class);
- addVal(slave, SnapPuller.REPLICATION_FAILED_AT, props, Date.class);
- addVal(slave, SnapPuller.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class);
+ addVal(slave, IndexFetcher.INDEX_REPLICATED_AT, props, Date.class);
+ addVal(slave, IndexFetcher.INDEX_REPLICATED_AT_LIST, props, List.class);
+ addVal(slave, IndexFetcher.REPLICATION_FAILED_AT_LIST, props, List.class);
+ addVal(slave, IndexFetcher.TIMES_INDEX_REPLICATED, props, Integer.class);
+ addVal(slave, IndexFetcher.CONF_FILES_REPLICATED, props, Integer.class);
+ addVal(slave, IndexFetcher.TIMES_CONFIG_REPLICATED, props, Integer.class);
+ addVal(slave, IndexFetcher.CONF_FILES_REPLICATED_AT, props, Integer.class);
+ addVal(slave, IndexFetcher.LAST_CYCLE_BYTES_DOWNLOADED, props, Long.class);
+ addVal(slave, IndexFetcher.TIMES_FAILED, props, Integer.class);
+ addVal(slave, IndexFetcher.REPLICATION_FAILED_AT, props, Date.class);
+ addVal(slave, IndexFetcher.PREVIOUS_CYCLE_TIME_TAKEN, props, Long.class);
slave.add("currentDate", new Date().toString());
slave.add("isPollingDisabled", String.valueOf(isPollingDisabled()));
@@ -720,13 +750,13 @@ public class ReplicationHandler extends
try {
long bytesToDownload = 0;
List<String> filesToDownload = new ArrayList<>();
- for (Map<String, Object> file : snapPuller.getFilesToDownload()) {
+ for (Map<String, Object> file : fetcher.getFilesToDownload()) {
filesToDownload.add((String) file.get(NAME));
bytesToDownload += (Long) file.get(SIZE);
}
//get list of conf files to download
- for (Map<String, Object> file : snapPuller.getConfFilesToDownload()) {
+ for (Map<String, Object> file : fetcher.getConfFilesToDownload()) {
filesToDownload.add((String) file.get(NAME));
bytesToDownload += (Long) file.get(SIZE);
}
@@ -737,18 +767,18 @@ public class ReplicationHandler extends
long bytesDownloaded = 0;
List<String> filesDownloaded = new ArrayList<>();
- for (Map<String, Object> file : snapPuller.getFilesDownloaded()) {
+ for (Map<String, Object> file : fetcher.getFilesDownloaded()) {
filesDownloaded.add((String) file.get(NAME));
bytesDownloaded += (Long) file.get(SIZE);
}
//get list of conf files downloaded
- for (Map<String, Object> file : snapPuller.getConfFilesDownloaded()) {
+ for (Map<String, Object> file : fetcher.getConfFilesDownloaded()) {
filesDownloaded.add((String) file.get(NAME));
bytesDownloaded += (Long) file.get(SIZE);
}
- Map<String, Object> currentFile = snapPuller.getCurrentFile();
+ Map<String, Object> currentFile = fetcher.getCurrentFile();
String currFile = null;
long currFileSize = 0, currFileSizeDownloaded = 0;
float percentDownloaded = 0;
@@ -767,10 +797,10 @@ public class ReplicationHandler extends
long estimatedTimeRemaining = 0;
- if (snapPuller.getReplicationStartTime() > 0) {
- slave.add("replicationStartTime", new Date(snapPuller.getReplicationStartTime()).toString());
+ if (fetcher.getReplicationStartTime() > 0) {
+ slave.add("replicationStartTime", new Date(fetcher.getReplicationStartTime()).toString());
}
- long elapsed = getTimeElapsed(snapPuller);
+ long elapsed = fetcher.getReplicationTimeElapsed();
slave.add("timeElapsed", String.valueOf(elapsed) + "s");
if (bytesDownloaded > 0)
@@ -840,13 +870,6 @@ public class ReplicationHandler extends
return replicateAfter;
}
- private long getTimeElapsed(SnapPuller snapPuller) {
- long timeElapsed = 0;
- if (snapPuller.getReplicationStartTime() > 0)
- timeElapsed = TimeUnit.SECONDS.convert(System.currentTimeMillis() - snapPuller.getReplicationStartTime(), TimeUnit.MILLISECONDS);
- return timeElapsed;
- }
-
Properties loadReplicationProperties() {
Directory dir = null;
try {
@@ -856,7 +879,7 @@ public class ReplicationHandler extends
IndexInput input;
try {
input = dir.openInput(
- SnapPuller.REPLICATION_PROPERTIES, IOContext.DEFAULT);
+ IndexFetcher.REPLICATION_PROPERTIES, IOContext.DEFAULT);
} catch (FileNotFoundException | NoSuchFileException e) {
return new Properties();
}
@@ -887,6 +910,37 @@ public class ReplicationHandler extends
// }
// }
+ private void setupPolling(String intervalStr) {
+ pollIntervalStr = intervalStr;
+ pollInterval = readInterval(pollIntervalStr);
+ if (pollInterval == null || pollInterval <= 0) {
+ LOG.info(" No value set for 'pollInterval'. Timer Task not started.");
+ return;
+ }
+
+ Runnable task = new Runnable() {
+ @Override
+ public void run() {
+ if (pollDisabled.get()) {
+ LOG.info("Poll disabled");
+ return;
+ }
+ try {
+ LOG.debug("Polling for index modifications");
+ executorStartTime = System.currentTimeMillis();
+ doFetch(null, false);
+ } catch (Exception e) {
+ LOG.error("Exception in fetching index", e);
+ }
+ }
+ };
+ executorService = Executors.newSingleThreadScheduledExecutor(
+ new DefaultSolrThreadFactory("indexFetcher"));
+ long initialDelay = pollInterval - (System.currentTimeMillis() % pollInterval);
+ executorService.scheduleAtFixedRate(task, initialDelay, pollInterval, TimeUnit.MILLISECONDS);
+ LOG.info("Poll Scheduled at an interval of " + pollInterval + "ms");
+ }
+
@Override
@SuppressWarnings("unchecked")
public void inform(SolrCore core) {
@@ -901,7 +955,8 @@ public class ReplicationHandler extends
NamedList slave = (NamedList) initArgs.get("slave");
boolean enableSlave = isEnabled( slave );
if (enableSlave) {
- tempSnapPuller = snapPuller = new SnapPuller(slave, this, core);
+ currentIndexFetcher = pollingIndexFetcher = new IndexFetcher(slave, this, core);
+ setupPolling((String) slave.get(POLL_INTERVAL));
isSlave = true;
}
NamedList master = (NamedList) initArgs.get("master");
@@ -1005,7 +1060,7 @@ public class ReplicationHandler extends
}
String reserve = (String) master.get(RESERVE);
if (reserve != null && !reserve.trim().equals("")) {
- reserveCommitDuration = SnapPuller.readInterval(reserve);
+ reserveCommitDuration = readInterval(reserve);
}
LOG.info("Commits will be reserved for " + reserveCommitDuration);
isMaster = true;
@@ -1029,11 +1084,20 @@ public class ReplicationHandler extends
core.addCloseHook(new CloseHook() {
@Override
public void preClose(SolrCore core) {
- if (snapPuller != null) {
- snapPuller.destroy();
+ try {
+ if (executorService != null) executorService.shutdown();
+ } finally {
+ try {
+ if (pollingIndexFetcher != null) {
+ pollingIndexFetcher.destroy();
+ }
+ } finally {
+ if (executorService != null) ExecutorUtil
+ .shutdownNowAndAwaitTermination(executorService);
+ }
}
- if (tempSnapPuller != null && tempSnapPuller != snapPuller) {
- tempSnapPuller.destroy();
+ if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) {
+ currentIndexFetcher.destroy();
}
}
@@ -1307,8 +1371,40 @@ public class ReplicationHandler extends
releaseCommitPointAndExtendReserve();
}
}
- }
-
+ }
+
+ static Integer readInterval(String interval) {
+ if (interval == null)
+ return null;
+ int result = 0;
+ if (interval != null) {
+ Matcher m = INTERVAL_PATTERN.matcher(interval.trim());
+ if (m.find()) {
+ String hr = m.group(1);
+ String min = m.group(2);
+ String sec = m.group(3);
+ result = 0;
+ try {
+ if (sec != null && sec.length() > 0)
+ result += Integer.parseInt(sec);
+ if (min != null && min.length() > 0)
+ result += (60 * Integer.parseInt(min));
+ if (hr != null && hr.length() > 0)
+ result += (60 * 60 * Integer.parseInt(hr));
+ result *= 1000;
+ } catch (NumberFormatException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ INTERVAL_ERR_MSG);
+ }
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ INTERVAL_ERR_MSG);
+ }
+
+ }
+ return result;
+ }
+
public static final String MASTER_URL = "masterUrl";
public static final String STATUS = "status";
@@ -1369,6 +1465,12 @@ public class ReplicationHandler extends
public static final String FILE_STREAM = "filestream";
+ public static final String POLL_INTERVAL = "pollInterval";
+
+ public static final String INTERVAL_ERR_MSG = "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'";
+
+ private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)");
+
public static final int PACKET_SZ = 1024 * 1024; // 1MB
public static final String RESERVE = "commitReserveDuration";
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java Tue Mar 31 05:22:40 2015
@@ -22,10 +22,10 @@ import org.apache.solr.common.params.Sol
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.PluginInfo;
-import org.apache.solr.core.RequestHandlers;
-import org.apache.solr.core.RequestParams;
+import org.apache.solr.core.PluginBag;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.handler.component.SearchHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
@@ -36,7 +36,6 @@ import org.apache.solr.util.stats.Timer;
import org.apache.solr.util.stats.TimerContext;
import java.net.URL;
-import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.apache.solr.core.RequestParams.USEPARAM;
@@ -138,7 +137,7 @@ public abstract class RequestHandlerBase
TimerContext timer = requestTimes.time();
try {
if(pluginInfo != null && pluginInfo.attributes.containsKey(USEPARAM)) req.getContext().put(USEPARAM,pluginInfo.attributes.get(USEPARAM));
- SolrPluginUtils.setDefaults(req,defaults,appends,invariants);
+ SolrPluginUtils.setDefaults(this, req, defaults, appends, invariants);
req.getContext().remove(USEPARAM);
rsp.setHttpCaching(httpCaching);
handleRequestBody( req, rsp );
@@ -215,7 +214,7 @@ public abstract class RequestHandlerBase
*
* This function is thread safe.
*/
- public static SolrRequestHandler getRequestHandler(String handlerName, Map<String, SolrRequestHandler> reqHandlers) {
+ public static SolrRequestHandler getRequestHandler(String handlerName, PluginBag<SolrRequestHandler> reqHandlers) {
if(handlerName == null) return null;
SolrRequestHandler handler = reqHandlers.get(handlerName);
int idx = 0;
@@ -226,9 +225,6 @@ public abstract class RequestHandlerBase
String firstPart = handlerName.substring(0, idx);
handler = reqHandlers.get(firstPart);
if (handler == null) continue;
- if(handler instanceof RequestHandlers.LazyRequestHandlerWrapper) {
- handler = ((RequestHandlers.LazyRequestHandlerWrapper)handler).getWrappedHandler();
- }
if (handler instanceof NestedRequestHandler) {
return ((NestedRequestHandler) handler).getSubHandler(handlerName.substring(idx));
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java Tue Mar 31 05:22:40 2015
@@ -39,6 +39,15 @@ import org.apache.solr.update.processor.
public class RequestHandlerUtils
{
/**
+ * A common way to mark the response format as experimental
+ */
+ public static void addExperimentalFormatWarning( SolrQueryResponse rsp )
+ {
+ rsp.add( "WARNING", "This response format is experimental. It is likely to change in the future." );
+ }
+
+
+ /**
* Check the request parameters and decide if it should commit or optimize.
* If it does, it will check other related parameters such as "waitFlush" and "waitSearcher"
*/
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java Tue Mar 31 05:22:40 2015
@@ -62,11 +62,9 @@ public class SchemaHandler extends Reque
}
break;
}
-
} else {
handleGET(req, rsp);
}
-
}
private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) {
@@ -165,6 +163,6 @@ public class SchemaHandler extends Reque
@Override
public String getDescription() {
- return "Edit schema.xml";
+ return "CRUD operations over the Solr schema";
}
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SnapShooter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SnapShooter.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SnapShooter.java Tue Mar 31 05:22:40 2015
@@ -144,7 +144,7 @@ public class SnapShooter {
details.add("snapshotName", snapshotName);
LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "<not named>" : snapshotName));
} catch (Exception e) {
- SnapPuller.delTree(snapShotDir);
+ IndexFetcher.delTree(snapShotDir);
LOG.error("Exception while creating snapshot", e);
details.add("snapShootException", e.getMessage());
} finally {
@@ -170,7 +170,7 @@ public class SnapShooter {
int i=1;
for (OldBackupDirectory dir : dirs) {
if (i++ > numberToKeep) {
- SnapPuller.delTree(dir.dir);
+ IndexFetcher.delTree(dir.dir);
}
}
}
@@ -181,7 +181,7 @@ public class SnapShooter {
NamedList<Object> details = new NamedList<>();
boolean isSuccess;
File f = new File(snapDir, "snapshot." + snapshotName);
- isSuccess = SnapPuller.delTree(f);
+ isSuccess = IndexFetcher.delTree(f);
if(isSuccess) {
details.add("status", "success");
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java Tue Mar 31 05:22:40 2015
@@ -19,7 +19,6 @@ package org.apache.solr.handler;
import java.io.IOException;
-import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -43,28 +42,25 @@ import org.apache.solr.common.util.Conte
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.ConfigOverlay;
import org.apache.solr.core.PluginInfo;
-import org.apache.solr.core.PluginsRegistry;
+import org.apache.solr.core.ImplicitPlugins;
import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
-import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.schema.SchemaManager;
-import org.apache.solr.search.QParserPlugin;
-import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.util.CommandOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static java.text.MessageFormat.format;
import static java.util.Collections.singletonList;
import static org.apache.solr.common.params.CoreAdminParams.NAME;
+import static org.apache.solr.common.util.StrUtils.formatString;
import static org.apache.solr.core.ConfigOverlay.NOT_EDITABLE;
+import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_CLASS;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME;
+import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME_IN_OVERLAY;
import static org.apache.solr.schema.FieldType.CLASS_NAME;
public class SolrConfigHandler extends RequestHandlerBase {
@@ -75,9 +71,8 @@ public class SolrConfigHandler extends R
static {
Map<String, SolrConfig.SolrPluginInfo> map = new HashMap<>();
for (SolrConfig.SolrPluginInfo plugin : SolrConfig.plugins) {
- if (plugin.options.contains(REQUIRE_NAME)) {
- map.put(plugin.tag.toLowerCase(Locale.ROOT), plugin);
-
+ if (plugin.options.contains(REQUIRE_NAME) || plugin.options.contains(REQUIRE_NAME_IN_OVERLAY)) {
+ map.put(plugin.getCleanTag().toLowerCase(Locale.ROOT), plugin);
}
}
namedPlugins = Collections.unmodifiableMap(map);
@@ -93,7 +88,11 @@ public class SolrConfigHandler extends R
if ("POST".equals(httpMethod)) {
if (configEditing_disabled)
throw new SolrException(SolrException.ErrorCode.FORBIDDEN, " solrconfig editing is not enabled");
- command.handlePOST();
+ try {
+ command.handlePOST();
+ } finally {
+ RequestHandlerUtils.addExperimentalFormatWarning(rsp);
+ }
} else {
command.handleGET();
}
@@ -152,7 +151,7 @@ public class SolrConfigHandler extends R
Map<String, Object> map = req.getCore().getSolrConfig().toMap();
Map reqHandlers = (Map) map.get(SolrRequestHandler.TYPE);
if (reqHandlers == null) map.put(SolrRequestHandler.TYPE, reqHandlers = new LinkedHashMap<>());
- List<PluginInfo> plugins = PluginsRegistry.getHandlers(req.getCore());
+ List<PluginInfo> plugins = ImplicitPlugins.getHandlers(req.getCore());
for (PluginInfo plugin : plugins) {
if (SolrRequestHandler.TYPE.equals(plugin.type)) {
if (!reqHandlers.containsKey(plugin.name)) {
@@ -257,7 +256,7 @@ public class SolrConfigHandler extends R
if (op.hasError()) break;
for (String s : name) {
if (params.getParams(s) == null) {
- op.addError(MessageFormat.format("can't delete . No such params ''{0}'' exist", s));
+ op.addError(formatString("can't delete . No such params ''{0}'' exist", s));
}
params = params.setParams(s, null);
}
@@ -307,19 +306,19 @@ public class SolrConfigHandler extends R
default: {
List<String> pcs = StrUtils.splitSmart(op.name.toLowerCase(Locale.ROOT), '-');
if (pcs.size() != 2) {
- op.addError(MessageFormat.format("Unknown operation ''{0}'' ", op.name));
+ op.addError(formatString("Unknown operation ''{0}'' ", op.name));
} else {
String prefix = pcs.get(0);
String name = pcs.get(1);
if (cmdPrefixes.contains(prefix) && namedPlugins.containsKey(name)) {
SolrConfig.SolrPluginInfo info = namedPlugins.get(name);
if ("delete".equals(prefix)) {
- overlay = deleteNamedComponent(op, overlay, info.tag);
+ overlay = deleteNamedComponent(op, overlay, info.getCleanTag());
} else {
- overlay = updateNamedPlugin(info, op, overlay, prefix.equals("create"));
+ overlay = updateNamedPlugin(info, op, overlay, prefix.equals("create") || prefix.equals("add"));
}
} else {
- op.addError(MessageFormat.format("Unknown operation ''{0}'' ", op.name));
+ op.addError(formatString("Unknown operation ''{0}'' ", op.name));
}
}
}
@@ -352,41 +351,42 @@ public class SolrConfigHandler extends R
if (overlay.getNamedPlugins(typ).containsKey(name)) {
return overlay.deleteNamedPlugin(name, typ);
} else {
- op.addError(MessageFormat.format("NO such {0} ''{1}'' ", typ, name));
+ op.addError(formatString("NO such {0} ''{1}'' ", typ, name));
return overlay;
}
}
private ConfigOverlay updateNamedPlugin(SolrConfig.SolrPluginInfo info, CommandOperation op, ConfigOverlay overlay, boolean isCeate) {
String name = op.getStr(NAME);
- String clz = op.getStr(CLASS_NAME);
+ String clz = info.options.contains(REQUIRE_CLASS) ? op.getStr(CLASS_NAME) : op.getStr(CLASS_NAME, null);
op.getMap(PluginInfo.DEFAULTS, null);
op.getMap(PluginInfo.INVARIANTS, null);
op.getMap(PluginInfo.APPENDS, null);
if (op.hasError()) return overlay;
if (!verifyClass(op, clz, info.clazz)) return overlay;
- if (overlay.getNamedPlugins(info.tag).containsKey(name)) {
+ if (overlay.getNamedPlugins(info.getCleanTag()).containsKey(name)) {
if (isCeate) {
- op.addError(MessageFormat.format(" ''{0}'' already exists . Do an ''{1}'' , if you want to change it ", name, "update-" + info.tag.toLowerCase(Locale.ROOT)));
+ op.addError(formatString(" ''{0}'' already exists . Do an ''{1}'' , if you want to change it ", name, "update-" + info.getTagCleanLower()));
return overlay;
} else {
- return overlay.addNamedPlugin(op.getDataMap(), info.tag);
+ return overlay.addNamedPlugin(op.getDataMap(), info.getCleanTag());
}
} else {
if (isCeate) {
- return overlay.addNamedPlugin(op.getDataMap(), info.tag);
+ return overlay.addNamedPlugin(op.getDataMap(), info.getCleanTag());
} else {
- op.addError(MessageFormat.format(" ''{0}'' does not exist . Do an ''{1}'' , if you want to create it ", name, "create-" + info.tag.toLowerCase(Locale.ROOT)));
+ op.addError(formatString(" ''{0}'' does not exist . Do an ''{1}'' , if you want to create it ", name, "create-" + info.getTagCleanLower()));
return overlay;
}
}
}
private boolean verifyClass(CommandOperation op, String clz, Class expected) {
- if (op.getStr("lib", null) == null) {
+ if (clz == null) return true;
+ if (!"true".equals(String.valueOf(op.getStr("runtimeLib", null)))) {
//this is not dynamically loaded so we can verify the class right away
try {
- SolrCore.createInstance(clz, expected, expected.getSimpleName(), req.getCore());
+ req.getCore().createInitInstance(new PluginInfo(SolrRequestHandler.TYPE, op.getDataMap()), expected, clz, "");
} catch (Exception e) {
op.addError(e.getMessage());
return false;
@@ -412,7 +412,7 @@ public class SolrConfigHandler extends R
if (op.hasError()) return overlay;
for (String o : name) {
if (!overlay.getUserProps().containsKey(o)) {
- op.addError(format("No such property ''{0}''", name));
+ op.addError(formatString("No such property ''{0}''", name));
} else {
overlay = overlay.unsetUserProperty(o);
}
@@ -427,7 +427,7 @@ public class SolrConfigHandler extends R
for (String o : name) {
if (!ConfigOverlay.isEditableProp(o, false, null)) {
- op.addError(format(NOT_EDITABLE, name));
+ op.addError(formatString(NOT_EDITABLE, name));
} else {
overlay = overlay.unsetProperty(o);
}
@@ -441,10 +441,42 @@ public class SolrConfigHandler extends R
for (Map.Entry<String, Object> e : m.entrySet()) {
String name = e.getKey();
Object val = e.getValue();
- if (!ConfigOverlay.isEditableProp(name, false, null)) {
- op.addError(format(NOT_EDITABLE, name));
+ Class typ = ConfigOverlay.checkEditable(name, false, null);
+ if (typ == null) {
+ op.addError(formatString(NOT_EDITABLE, name));
continue;
}
+
+ if (val != null) {
+ if (typ == String.class) val = val.toString();
+ String typeErr = "Property {0} must be of {1} type ";
+ if (typ == Boolean.class) {
+ try {
+ val = Boolean.parseBoolean(val.toString());
+ } catch (Exception exp) {
+ op.addError(formatString(typeErr, name, typ.getSimpleName()));
+ continue;
+ }
+ } else if (typ == Integer.class) {
+ try {
+ val = Integer.parseInt(val.toString());
+ } catch (Exception exp) {
+ op.addError(formatString(typeErr, typ.getSimpleName()));
+ continue;
+ }
+
+ } else if (typ == Float.class) {
+ try {
+ val = Float.parseFloat(val.toString());
+ } catch (Exception exp) {
+ op.addError(formatString(typeErr, typ.getSimpleName()));
+ continue;
+ }
+
+ }
+ }
+
+
overlay = overlay.setProperty(name, val);
}
return overlay;
@@ -463,7 +495,7 @@ public class SolrConfigHandler extends R
c == '.'
) continue;
else {
- return MessageFormat.format("''{0}'' name should only have chars [a-zA-Z_-.0-9] ", s);
+ return formatString("''{0}'' name should only have chars [a-zA-Z_-.0-9] ", s);
}
}
return null;
@@ -491,7 +523,7 @@ public class SolrConfigHandler extends R
static {
for (SolrConfig.SolrPluginInfo solrPluginInfo : SolrConfig.plugins)
- subPaths.add("/" + solrPluginInfo.tag.replaceAll("/", ""));
+ subPaths.add("/" + solrPluginInfo.getCleanTag());
}
@@ -522,6 +554,6 @@ public class SolrConfigHandler extends R
public static final String SET = "set";
public static final String UPDATE = "update";
public static final String CREATE = "create";
- private static Set<String> cmdPrefixes = ImmutableSet.of(CREATE, UPDATE, "delete");
+ private static Set<String> cmdPrefixes = ImmutableSet.of(CREATE, UPDATE, "delete", "add");
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java Tue Mar 31 05:22:40 2015
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
@@ -38,7 +39,7 @@ import org.slf4j.LoggerFactory;
* the plugins registered by this class are iplicitly registered by the system
*/
@Deprecated
-public class AdminHandlers implements SolrCoreAware, SolrRequestHandler
+public class AdminHandlers extends RequestHandlerBase implements SolrCoreAware
{
public static Logger log = LoggerFactory.getLogger(AdminHandlers.class);
NamedList initArgs = null;
@@ -61,17 +62,12 @@ public class AdminHandlers implements So
public void init(NamedList args) {
this.initArgs = args;
}
-
+
@Override
public void inform(SolrCore core)
{
String path = null;
- for( Map.Entry<String, SolrRequestHandler> entry : core.getRequestHandlers().entrySet() ) {
- if( entry.getValue() == this ) {
- path = entry.getKey();
- break;
- }
- }
+ path = getPluginInfo().name;
if( path == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"The AdminHandler is not registered with the current core." );
@@ -109,9 +105,9 @@ public class AdminHandlers implements So
log.warn("<requestHandler name=\"/admin/\" \n class=\"solr.admin.AdminHandlers\" /> is deprecated . It is not required anymore");
}
-
+
@Override
- public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"The AdminHandler should never be called directly" );
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Tue Mar 31 05:22:40 2015
@@ -80,9 +80,9 @@ import static org.apache.solr.cloud.Over
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_IF_DOWN;
import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
import static org.apache.solr.cloud.OverseerCollectionProcessor.REQUESTID;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARD_UNIQUE;
+import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
import static org.apache.solr.common.cloud.ZkStateReader.ACTIVE;
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
@@ -901,7 +901,7 @@ public class CollectionsHandler extends
log.info("Create shard: " + req.getParamString());
req.getParams().required().check(COLLECTION_PROP, SHARD_ID_PROP);
ClusterState clusterState = coreContainer.getZkController().getClusterState();
- if(!ImplicitDocRouter.NAME.equals( ((Map) clusterState.getCollection(req.getParams().get(COLLECTION_PROP)).get(ROUTER)).get("name") ) )
+ if(!ImplicitDocRouter.NAME.equals( ((Map) clusterState.getCollection(req.getParams().get(COLLECTION_PROP)).get(DOC_ROUTER)).get("name") ) )
throw new SolrException(ErrorCode.BAD_REQUEST, "shards can be added only to 'implicit' collections" );
Map<String, Object> map = makeMap(QUEUE_OPERATION, CREATESHARD.toLower());
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Tue Mar 31 05:22:40 2015
@@ -94,8 +94,7 @@ import static org.apache.solr.common.clo
public class CoreAdminHandler extends RequestHandlerBase {
protected static Logger log = LoggerFactory.getLogger(CoreAdminHandler.class);
protected final CoreContainer coreContainer;
- protected static HashMap<String, Map<String, TaskObject>> requestStatusMap =
- new HashMap<String,Map<String, TaskObject>>();
+ protected final Map<String, Map<String, TaskObject>> requestStatusMap;
protected final ExecutorService parallelExecutor = Executors.newFixedThreadPool(50,
new DefaultSolrThreadFactory("parallelCoreAdminExecutor"));
@@ -108,17 +107,16 @@ public class CoreAdminHandler extends Re
public static String RESPONSE_STATUS = "STATUS";
public static String RESPONSE_MESSAGE = "msg";
- static {
- requestStatusMap.put(RUNNING, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
- requestStatusMap.put(COMPLETED, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
- requestStatusMap.put(FAILED, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
- }
-
public CoreAdminHandler() {
super();
// Unlike most request handlers, CoreContainer initialization
// should happen in the constructor...
this.coreContainer = null;
+ HashMap<String, Map<String, TaskObject>> map = new HashMap<>(3, 1.0f);
+ map.put(RUNNING, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
+ map.put(COMPLETED, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
+ map.put(FAILED, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
+ requestStatusMap = Collections.unmodifiableMap(map);
}
@@ -129,6 +127,11 @@ public class CoreAdminHandler extends Re
*/
public CoreAdminHandler(final CoreContainer coreContainer) {
this.coreContainer = coreContainer;
+ HashMap<String, Map<String, TaskObject>> map = new HashMap<>(3, 1.0f);
+ map.put(RUNNING, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
+ map.put(COMPLETED, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
+ map.put(FAILED, Collections.synchronizedMap(new LinkedHashMap<String, TaskObject>()));
+ requestStatusMap = Collections.unmodifiableMap(map);
}
@@ -832,7 +835,7 @@ public class CoreAdminHandler extends Re
searchHolder.decref();
}
} catch (Exception e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, null, e);
+ log.debug("Error in solrcloud_debug block", e);
}
}
if (!success) {
@@ -1011,7 +1014,7 @@ public class CoreAdminHandler extends Re
searchHolder.decref();
}
} catch (Exception e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, null, e);
+ log.debug("Error in solrcloud_debug block", e);
}
}
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Tue Mar 31 05:22:40 2015
@@ -20,16 +20,37 @@ package org.apache.solr.handler.admin;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.*;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
@@ -48,18 +69,18 @@ import org.apache.solr.common.util.Simpl
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.CopyField;
import org.apache.solr.schema.FieldType;
-import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.CopyField;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.update.SolrIndexWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
-import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS;
import static org.apache.lucene.index.IndexOptions.DOCS;
+import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS;
+import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
/**
* This handler exposes the internal lucene index. It is inspired by and
@@ -185,6 +206,7 @@ public class LukeRequestHandler extends
flags.append( (f != null && f.fieldType().storeTermVectors()) ? FieldFlag.TERM_VECTOR_STORED.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().storeTermVectorOffsets()) ? FieldFlag.TERM_VECTOR_OFFSET.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().storeTermVectorPositions()) ? FieldFlag.TERM_VECTOR_POSITION.getAbbreviation() : '-' );
+ flags.append( (f != null && f.fieldType().storeTermVectorPayloads()) ? FieldFlag.TERM_VECTOR_PAYLOADS.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().omitNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-' );
flags.append( (f != null && DOCS == opts ) ?
@@ -223,6 +245,7 @@ public class LukeRequestHandler extends
flags.append( (f != null && f.storeTermVector() ) ? FieldFlag.TERM_VECTOR_STORED.getAbbreviation() : '-' );
flags.append( (f != null && f.storeTermOffsets() ) ? FieldFlag.TERM_VECTOR_OFFSET.getAbbreviation() : '-' );
flags.append( (f != null && f.storeTermPositions() ) ? FieldFlag.TERM_VECTOR_POSITION.getAbbreviation() : '-' );
+ flags.append( (f != null && f.storeTermPayloads() ) ? FieldFlag.TERM_VECTOR_PAYLOADS.getAbbreviation() : '-' );
flags.append( (f != null && f.omitNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-' );
flags.append( (f != null &&
f.omitTermFreqAndPositions() ) ? FieldFlag.OMIT_TF.getAbbreviation() : '-' );
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java Tue Mar 31 05:22:40 2015
@@ -105,7 +105,11 @@ public class DebugComponent extends Sear
else {
info.addAll( stdinfo );
}
-
+
+ if (rb.req.getJSON() != null) {
+ info.add("json", rb.req.getJSON());
+ }
+
if (rb.isDebugQuery() && rb.getQparser() != null) {
rb.getQparser().addDebugInfo(rb.getDebugInfo());
}
@@ -350,6 +354,9 @@ public class DebugComponent extends Sear
return dl;
}
+ // only add to list if JSON is different
+ if (source.equals(dest)) return source;
+
// merge unlike elements in a list
List<Object> t = new ArrayList<>();
t.add(dest);
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java Tue Mar 31 05:22:40 2015
@@ -126,75 +126,70 @@ public class FacetComponent extends Sear
if (!rb.doFacets) {
return ResponseBuilder.STAGE_DONE;
}
-
- if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
- // overlap facet refinement requests (those shards that we need a count
- // for particular facet values from), where possible, with
- // the requests to get fields (because we know that is the
- // only other required phase).
- // We do this in distributedProcess so we can look at all of the
- // requests in the outgoing queue at once.
-
- for (int shardNum = 0; shardNum < rb.shards.length; shardNum++) {
- List<String> distribFieldFacetRefinements = null;
-
- for (DistribFieldFacet dff : rb._facetInfo.facets.values()) {
- if (!dff.needRefinements) continue;
- List<String> refList = dff._toRefine[shardNum];
- if (refList == null || refList.size() == 0) continue;
-
- String key = dff.getKey(); // reuse the same key that was used for the
- // main facet
- String termsKey = key + "__terms";
- String termsVal = StrUtils.join(refList, ',');
-
- String facetCommand;
- // add terms into the original facet.field command
- // do it via parameter reference to avoid another layer of encoding.
-
- String termsKeyEncoded = QueryParsing.encodeLocalParamVal(termsKey);
- if (dff.localParams != null) {
- facetCommand = commandPrefix + termsKeyEncoded + " "
- + dff.facetStr.substring(2);
- } else {
- facetCommand = commandPrefix + termsKeyEncoded + '}' + dff.field;
- }
-
- if (distribFieldFacetRefinements == null) {
- distribFieldFacetRefinements = new ArrayList<>();
- }
- distribFieldFacetRefinements.add(facetCommand);
- distribFieldFacetRefinements.add(termsKey);
- distribFieldFacetRefinements.add(termsVal);
- }
-
- boolean pivotFacetRefinementRequestsExistForShard =
- doAnyPivotFacetRefinementRequestsExistForShard(rb._facetInfo, shardNum);
-
- if (distribFieldFacetRefinements == null
- && !pivotFacetRefinementRequestsExistForShard) {
- // nothing to refine, short circuit out
- continue;
- }
-
+ if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) {
+ return ResponseBuilder.STAGE_DONE;
+ }
+ // Overlap facet refinement requests (those shards that we need a count
+ // for particular facet values from), where possible, with
+ // the requests to get fields (because we know that is the
+ // only other required phase).
+ // We do this in distributedProcess so we can look at all of the
+ // requests in the outgoing queue at once.
+
+ for (int shardNum = 0; shardNum < rb.shards.length; shardNum++) {
+ List<String> distribFieldFacetRefinements = null;
+
+ // FieldFacetAdditions
+ for (DistribFieldFacet dff : rb._facetInfo.facets.values()) {
+ if (!dff.needRefinements) continue;
+ List<String> refList = dff._toRefine[shardNum];
+ if (refList == null || refList.size() == 0) continue;
+
+ String key = dff.getKey(); // reuse the same key that was used for the
+ // main facet
+ String termsKey = key + "__terms";
+ String termsVal = StrUtils.join(refList, ',');
+
+ String facetCommand;
+ // add terms into the original facet.field command
+ // do it via parameter reference to avoid another layer of encoding.
+
+ String termsKeyEncoded = QueryParsing.encodeLocalParamVal(termsKey);
+ if (dff.localParams != null) {
+ facetCommand = commandPrefix + termsKeyEncoded + " "
+ + dff.facetStr.substring(2);
+ } else {
+ facetCommand = commandPrefix + termsKeyEncoded + '}' + dff.field;
+ }
+
+ if (distribFieldFacetRefinements == null) {
+ distribFieldFacetRefinements = new ArrayList<>();
+ }
+
+ distribFieldFacetRefinements.add(facetCommand);
+ distribFieldFacetRefinements.add(termsKey);
+ distribFieldFacetRefinements.add(termsVal);
+ }
+
+ if (distribFieldFacetRefinements != null) {
String shard = rb.shards[shardNum];
ShardRequest shardsRefineRequest = null;
boolean newRequest = false;
-
+
// try to find a request that is already going out to that shard.
- // If nshards becomes to great, we way want to move to hashing for
+ // If nshards becomes too great, we may want to move to hashing for
// better scalability.
for (ShardRequest sreq : rb.outgoing) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0
- && sreq.shards != null
+ && sreq.shards != null
&& sreq.shards.length == 1
&& sreq.shards[0].equals(shard)) {
shardsRefineRequest = sreq;
break;
}
}
-
+
if (shardsRefineRequest == null) {
// we didn't find any other suitable requests going out to that shard,
// so create one ourselves.
@@ -206,47 +201,50 @@ public class FacetComponent extends Sear
shardsRefineRequest.params.remove(CommonParams.START);
shardsRefineRequest.params.set(CommonParams.ROWS, "0");
}
-
- // FieldFacetAdditions
- if (distribFieldFacetRefinements != null) {
- shardsRefineRequest.purpose |= ShardRequest.PURPOSE_REFINE_FACETS;
- shardsRefineRequest.params.set(FacetParams.FACET, "true");
- shardsRefineRequest.params.remove(FacetParams.FACET_FIELD);
- shardsRefineRequest.params.remove(FacetParams.FACET_QUERY);
- //TODO remove interval faceting, and ranges and heatmap too?
-
- for (int i = 0; i < distribFieldFacetRefinements.size();) {
- String facetCommand = distribFieldFacetRefinements.get(i++);
- String termsKey = distribFieldFacetRefinements.get(i++);
- String termsVal = distribFieldFacetRefinements.get(i++);
-
- shardsRefineRequest.params.add(FacetParams.FACET_FIELD,
- facetCommand);
- shardsRefineRequest.params.set(termsKey, termsVal);
- }
+
+ shardsRefineRequest.purpose |= ShardRequest.PURPOSE_REFINE_FACETS;
+ shardsRefineRequest.params.set(FacetParams.FACET, "true");
+ removeMainFacetTypeParams(shardsRefineRequest);
+
+ for (int i = 0; i < distribFieldFacetRefinements.size();) {
+ String facetCommand = distribFieldFacetRefinements.get(i++);
+ String termsKey = distribFieldFacetRefinements.get(i++);
+ String termsVal = distribFieldFacetRefinements.get(i++);
+
+ shardsRefineRequest.params.add(FacetParams.FACET_FIELD,
+ facetCommand);
+ shardsRefineRequest.params.set(termsKey, termsVal);
}
if (newRequest) {
rb.addRequest(this, shardsRefineRequest);
}
+ }
- // PivotFacetAdditions
- if (pivotFacetRefinementRequestsExistForShard) {
- if (newRequest) {
- shardsRefineRequest.params.remove(FacetParams.FACET_PIVOT);
- shardsRefineRequest.params.remove(FacetParams.FACET_PIVOT_MINCOUNT);
- }
-
- enqueuePivotFacetShardRequests(rb, shardNum);
- }
+
+ // PivotFacetAdditions
+ if (doAnyPivotFacetRefinementRequestsExistForShard(rb._facetInfo, shardNum)) {
+ enqueuePivotFacetShardRequests(rb, shardNum);
}
- }
-
+
+ } // for shardNum
+
return ResponseBuilder.STAGE_DONE;
}
-
+
+ public static String[] FACET_TYPE_PARAMS = {
+ FacetParams.FACET_FIELD, FacetParams.FACET_PIVOT, FacetParams.FACET_QUERY, FacetParams.FACET_DATE,
+ FacetParams.FACET_RANGE, FacetParams.FACET_INTERVAL, FacetParams.FACET_HEATMAP
+ };
+
+ private void removeMainFacetTypeParams(ShardRequest shardsRefineRequest) {
+ for (String param : FACET_TYPE_PARAMS) {
+ shardsRefineRequest.params.remove(param);
+ }
+ }
+
private void enqueuePivotFacetShardRequests(ResponseBuilder rb, int shardNum) {
-
+
FacetInfo fi = rb._facetInfo;
ShardRequest shardsRefineRequestPivot = new ShardRequest();
@@ -259,9 +257,8 @@ public class FacetComponent extends Sear
shardsRefineRequestPivot.purpose |= ShardRequest.PURPOSE_REFINE_PIVOT_FACETS;
shardsRefineRequestPivot.params.set(FacetParams.FACET, "true");
- shardsRefineRequestPivot.params.remove(FacetParams.FACET_PIVOT_MINCOUNT);
+ removeMainFacetTypeParams(shardsRefineRequestPivot);
shardsRefineRequestPivot.params.set(FacetParams.FACET_PIVOT_MINCOUNT, -1);
- shardsRefineRequestPivot.params.remove(FacetParams.FACET_PIVOT);
shardsRefineRequestPivot.params.remove(FacetParams.FACET_OFFSET);
for (int pivotIndex = 0; pivotIndex < fi.pivotFacets.size(); pivotIndex++) {
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Tue Mar 31 05:22:40 2015
@@ -16,6 +16,7 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import com.google.common.base.Strings;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
@@ -85,7 +86,9 @@ public class HttpShardHandler extends Sh
private static class SimpleSolrResponse extends SolrResponse {
+
long elapsedTime;
+
NamedList<Object> nl;
@Override
@@ -102,6 +105,11 @@ public class HttpShardHandler extends Sh
public void setResponse(NamedList<Object> rsp) {
nl = rsp;
}
+
+ @Override
+ public void setElapsedTime(long elapsedTime) {
+ this.elapsedTime = elapsedTime;
+ }
}
@@ -170,6 +178,8 @@ public class HttpShardHandler extends Sh
public void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) {
// do this outside of the callable for thread safety reasons
final List<String> urls = getURLs(sreq, shard);
+ if (Strings.isNullOrEmpty(shard))
+ System.out.println("Empty shard!");
Callable<ShardResponse> task = new Callable<ShardResponse>() {
@Override
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Tue Mar 31 05:22:40 2015
@@ -29,8 +29,8 @@ import org.apache.solr.common.util.Execu
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.URLUtil;
-import org.apache.solr.core.ConfigSolr;
import org.apache.solr.core.PluginInfo;
+import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,8 +69,8 @@ public class HttpShardHandlerFactory ext
protected HttpClient defaultClient;
private LBHttpSolrClient loadbalancer;
//default values:
- int soTimeout = ConfigSolr.DEFAULT_DISTRIBUPDATESOTIMEOUT;
- int connectionTimeout = ConfigSolr.DEFAULT_DISTRIBUPDATECONNTIMEOUT;
+ int soTimeout = UpdateShardHandlerConfig.DEFAULT_DISTRIBUPDATESOTIMEOUT;
+ int connectionTimeout = UpdateShardHandlerConfig.DEFAULT_DISTRIBUPDATECONNTIMEOUT;
int maxConnectionsPerHost = 20;
int maxConnections = 10000;
int corePoolSize = 0;
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java Tue Mar 31 05:22:40 2015
@@ -278,8 +278,7 @@ public class PivotFacetProcessor extends
for (StatsField statsField : statsFields) {
stv.put(statsField.getOutputKey(), statsField.computeLocalStatsValues(subset));
}
- // for pivots, we *always* include requested stats - even if 'empty'
- pivot.add("stats", StatsComponent.convertToResponse(true, stv));
+ pivot.add("stats", StatsComponent.convertToResponse(stv));
}
values.add( pivot );
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java Tue Mar 31 05:22:40 2015
@@ -183,8 +183,7 @@ public class PivotFacetValue {
}
if (null != statsValues) {
newList.add(PivotListEntry.STATS.getName(),
- // for pivots, we *always* include requested stats - even if 'empty'
- StatsComponent.convertToResponse(true, statsValues));
+ StatsComponent.convertToResponse(statsValues));
}
return newList;
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Tue Mar 31 05:22:40 2015
@@ -216,6 +216,16 @@ public class QueryComponent extends Sear
if (params.getBool(GroupParams.GROUP, false)) {
prepareGrouping(rb);
+ } else {
+ //Validate only in case of non-grouping search.
+ if(rb.getSortSpec().getCount() < 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'rows' parameter cannot be negative");
+ }
+ }
+
+ //Input validation.
+ if (rb.getQueryCommand().getOffset() < 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
}
}
@@ -306,10 +316,6 @@ public class QueryComponent extends Sear
statsCache.receiveGlobalStats(req);
}
- if (rb.getQueryCommand().getOffset() < 0) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
- }
-
// -1 as flag if not set.
long timeAllowed = params.getLong(CommonParams.TIME_ALLOWED, -1L);
if (null != rb.getCursorMark() && 0 < timeAllowed) {
@@ -882,30 +888,40 @@ public class QueryComponent extends Sear
sreq.params.set(ResponseBuilder.FIELD_SORT_VALUES,"true");
- // TODO: should this really sendGlobalDfs if just includeScore?
- if ( (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES)!=0 || rb.getSortSpec().includesScore()) {
- sreq.params.set(CommonParams.FL, rb.req.getSchema().getUniqueKeyField().getName() + ",score");
- StatsCache statsCache = rb.req.getCore().getStatsCache();
- statsCache.sendGlobalStats(rb, sreq);
- }
boolean shardQueryIncludeScore = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0 || rb.getSortSpec().includesScore();
- if (distribSinglePass) {
+ StringBuilder additionalFL = new StringBuilder();
+ boolean additionalAdded = false;
+ if (distribSinglePass) {
String[] fls = rb.req.getParams().getParams(CommonParams.FL);
if (fls != null && fls.length > 0 && (fls.length != 1 || !fls[0].isEmpty())) {
// If the outer request contains actual FL's use them...
sreq.params.set(CommonParams.FL, fls);
+ if (!fields.wantsField(keyFieldName)) {
+ additionalAdded = addFL(additionalFL, keyFieldName, additionalAdded);
+ }
} else {
// ... else we need to explicitly ask for all fields, because we are going to add
// additional fields below
sreq.params.set(CommonParams.FL, "*");
}
+ if (!fields.wantsScore() && shardQueryIncludeScore) {
+ additionalAdded = addFL(additionalFL, "score", additionalAdded);
+ }
+ } else {
+ // reset so that only unique key is requested in shard requests
+ sreq.params.set(CommonParams.FL, rb.req.getSchema().getUniqueKeyField().getName());
+ if (shardQueryIncludeScore) {
+ additionalAdded = addFL(additionalFL, "score", additionalAdded);
+ }
}
- StringBuilder additionalFL = new StringBuilder();
- boolean additionalAdded = false;
- if (!distribSinglePass || !fields.wantsField(keyFieldName))
- additionalAdded = addFL(additionalFL, keyFieldName, additionalAdded);
- if ((!distribSinglePass || !fields.wantsScore()) && shardQueryIncludeScore)
- additionalAdded = addFL(additionalFL, "score", additionalAdded);
+
+ // TODO: should this really sendGlobalDfs if just includeScore?
+
+ if (shardQueryIncludeScore) {
+ StatsCache statsCache = rb.req.getCore().getStatsCache();
+ statsCache.sendGlobalStats(rb, sreq);
+ }
+
if (additionalAdded) sreq.params.add(CommonParams.FL, additionalFL.toString());
rb.addRequest(this, sreq);
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java Tue Mar 31 05:22:40 2015
@@ -29,6 +29,7 @@ import org.apache.lucene.index.Indexable
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.solr.client.solrj.SolrResponse;
@@ -98,6 +99,25 @@ public class RealTimeGetComponent extend
val = params.get("getUpdates");
if (val != null) {
+ // solrcloud_debug
+ if (log.isDebugEnabled()) {
+ try {
+ RefCounted<SolrIndexSearcher> searchHolder = req.getCore()
+ .getNewestSearcher(false);
+ SolrIndexSearcher searcher = searchHolder.get();
+ try {
+ log.debug(req.getCore().getCoreDescriptor()
+ .getCoreContainer().getZkController().getNodeName()
+ + " min count to sync to (from most recent searcher view) "
+ + searcher.search(new MatchAllDocsQuery(), 1).totalHits);
+ } finally {
+ searchHolder.decref();
+ }
+ } catch (Exception e) {
+ log.debug("Error in solrcloud_debug block", e);
+ }
+ }
+
processGetUpdates(rb);
return;
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchComponent.java Tue Mar 31 05:22:40 2015
@@ -19,9 +19,13 @@ package org.apache.solr.handler.componen
import java.io.IOException;
import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.search.facet.FacetModule;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
/**
@@ -122,4 +126,22 @@ public abstract class SearchComponent im
public NamedList getStatistics() {
return null;
}
+
+ public static final Map<String, Class<? extends SearchComponent>> standard_components;
+ ;
+
+ static {
+ HashMap<String, Class<? extends SearchComponent>> map = new HashMap<>();
+ map.put(HighlightComponent.COMPONENT_NAME, HighlightComponent.class);
+ map.put(QueryComponent.COMPONENT_NAME, QueryComponent.class);
+ map.put(FacetComponent.COMPONENT_NAME, FacetComponent.class);
+ map.put(FacetModule.COMPONENT_NAME, FacetModule.class);
+ map.put(MoreLikeThisComponent.COMPONENT_NAME, MoreLikeThisComponent.class);
+ map.put(StatsComponent.COMPONENT_NAME, StatsComponent.class);
+ map.put(DebugComponent.COMPONENT_NAME, DebugComponent.class);
+ map.put(RealTimeGetComponent.COMPONENT_NAME, RealTimeGetComponent.class);
+ map.put(ExpandComponent.COMPONENT_NAME, ExpandComponent.class);
+ standard_components = Collections.unmodifiableMap(map);
+ }
+
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java Tue Mar 31 05:22:40 2015
@@ -35,6 +35,7 @@ import org.apache.solr.handler.RequestHa
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrQueryTimeoutImpl;
+import org.apache.solr.search.facet.FacetModule;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.SolrPluginUtils;
import org.apache.solr.util.plugin.PluginInfoInitialized;
@@ -75,6 +76,7 @@ public class SearchHandler extends Reque
ArrayList<String> names = new ArrayList<>(6);
names.add( QueryComponent.COMPONENT_NAME );
names.add( FacetComponent.COMPONENT_NAME );
+ names.add( FacetModule.COMPONENT_NAME );
names.add( MoreLikeThisComponent.COMPONENT_NAME );
names.add( HighlightComponent.COMPONENT_NAME );
names.add( StatsComponent.COMPONENT_NAME );
@@ -168,17 +170,10 @@ public class SearchHandler extends Reque
public List<SearchComponent> getComponents() {
return components;
}
-
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception
{
- // int sleep = req.getParams().getInt("sleep",0);
- // if (sleep > 0) {log.error("SLEEPING for " + sleep); Thread.sleep(sleep);}
- if (req.getContentStreams() != null && req.getContentStreams().iterator().hasNext()) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Search requests cannot accept content streams");
- }
-
ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
if (rb.requestInfo != null) {
rb.requestInfo.setResponseBuilder(rb);
@@ -190,8 +185,7 @@ public class SearchHandler extends Reque
SolrPluginUtils.getDebugInterests(req.getParams().getParams(CommonParams.DEBUG), rb);
}
- final RTimer timer = rb.isDebug() ? new RTimer() : null;
-
+ final RTimer timer = rb.isDebug() ? req.getRequestTimer() : null;
ShardHandler shardHandler1 = shardHandlerFactory.getShardHandler();
shardHandler1.checkDistributed(rb);
@@ -237,7 +231,6 @@ public class SearchHandler extends Reque
rb.getTimer().stop();
}
subt.stop();
- timer.stop();
// add the timing info
if (rb.isDebugTimings()) {
@@ -395,7 +388,7 @@ public class SearchHandler extends Reque
nl.add("maxScore", rb.getResults().docList.maxScore());
}
nl.add("shardAddress", rb.shortCircuitedURL);
- nl.add("time", rsp.getEndTime()-req.getStartTime()); // elapsed time of this request so far
+ nl.add("time", req.getRequestTimer().getTime()); // elapsed time of this request so far
int pos = rb.shortCircuitedURL.indexOf("://");
String shardInfoName = pos != -1 ? rb.shortCircuitedURL.substring(pos+3) : rb.shortCircuitedURL;
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Tue Mar 31 05:22:40 2015
@@ -52,8 +52,6 @@ public class StatsComponent extends Sear
@Override
public void process(ResponseBuilder rb) throws IOException {
if (!rb.doStats) return;
-
- boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false);
Map<String, StatsValues> statsValues = new LinkedHashMap<>();
for (StatsField statsField : rb._statsInfo.getStatsFields()) {
@@ -61,7 +59,7 @@ public class StatsComponent extends Sear
statsValues.put(statsField.getOutputKey(), statsField.computeLocalStatsValues(docs));
}
- rb.rsp.add( "stats", convertToResponse(isShard, statsValues) );
+ rb.rsp.add( "stats", convertToResponse(statsValues) );
}
@Override
@@ -122,7 +120,7 @@ public class StatsComponent extends Sear
// so that "result" is already stored in the response (for aesthetics)
Map<String, StatsValues> allStatsValues = rb._statsInfo.getAggregateStatsValues();
- rb.rsp.add("stats", convertToResponse(false, allStatsValues));
+ rb.rsp.add("stats", convertToResponse(allStatsValues));
rb._statsInfo = null; // free some objects
}
@@ -142,7 +140,7 @@ public class StatsComponent extends Sear
* including the esoteric "stats_fields" wrapper.
*/
public static NamedList<NamedList<NamedList<?>>> convertToResponse
- (boolean force, Map<String,StatsValues> statsValues) {
+ (Map<String,StatsValues> statsValues) {
NamedList<NamedList<NamedList<?>>> stats = new SimpleOrderedMap<>();
NamedList<NamedList<?>> stats_fields = new SimpleOrderedMap<>();
@@ -151,11 +149,7 @@ public class StatsComponent extends Sear
for (Map.Entry<String,StatsValues> entry : statsValues.entrySet()) {
String key = entry.getKey();
NamedList stv = entry.getValue().getStatsValues();
- if (force || ((Long) stv.get("count") != 0)) {
- stats_fields.add(key, stv);
- } else {
- stats_fields.add(key, null);
- }
+ stats_fields.add(key, stv);
}
return stats;
}