You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/20 17:27:23 UTC
svn commit: r1458924 [11/13] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/maven/
dev-tools/maven/solr/solrj/src/java/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/ lucene/analysis/c...
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -20,10 +20,14 @@ package org.apache.solr.core;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext.Context;
@@ -42,13 +46,42 @@ import org.slf4j.LoggerFactory;
* per path. Most DirectoryFactory implementations will want to extend this
* class and simply implement {@link DirectoryFactory#create(String, DirContext)}.
*
+ * This is an expert class and these API's are subject to change.
+ *
*/
public abstract class CachingDirectoryFactory extends DirectoryFactory {
protected class CacheValue {
- public Directory directory;
+ final public String path;
+ final public Directory directory;
+
+ // use the setter!
+ private boolean deleteOnClose = false;
+
+ public CacheValue(String path, Directory directory) {
+ this.path = path;
+ this.directory = directory;
+ this.closeEntries.add(this);
+ }
public int refCnt = 1;
- public String path;
+ // if we are latestForPath, I'm currently using my path
+ // otherwise a new Directory instance is using my path
+ // and I must be manipulated by Directory
+ public boolean latestForPath = false;
+ // has close(Directory) been called on this?
+ public boolean closeDirectoryCalled = false;
public boolean doneWithDir = false;
+ private boolean deleteAfterCoreClose = false;
+ public Set<CacheValue> removeEntries = new HashSet<CacheValue>();
+ public Set<CacheValue> closeEntries = new HashSet<CacheValue>();
+
+ public void setDeleteOnClose(boolean deleteOnClose, boolean deleteAfterCoreClose) {
+ if (deleteOnClose) {
+ removeEntries.add(this);
+ }
+ this.deleteOnClose = deleteOnClose;
+ this.deleteAfterCoreClose = deleteAfterCoreClose;
+ }
+
@Override
public String toString() {
return "CachedDir<<" + directory.toString() + ";refCount=" + refCnt + ";path=" + path + ";done=" + doneWithDir + ">>";
@@ -60,9 +93,11 @@ public abstract class CachingDirectoryFa
protected Map<String,CacheValue> byPathCache = new HashMap<String,CacheValue>();
- protected Map<Directory,CacheValue> byDirectoryCache = new HashMap<Directory,CacheValue>();
+ protected Map<Directory,CacheValue> byDirectoryCache = new IdentityHashMap<Directory,CacheValue>();
protected Map<Directory,List<CloseListener>> closeListeners = new HashMap<Directory,List<CloseListener>>();
+
+ protected Set<CacheValue> removeEntries = new HashSet<CacheValue>();
private Double maxWriteMBPerSecFlush;
@@ -123,7 +158,9 @@ public abstract class CachingDirectoryFa
public void close() throws IOException {
synchronized (this) {
this.closed = true;
- for (CacheValue val : byDirectoryCache.values()) {
+ Collection<CacheValue> values = new ArrayList<CacheValue>();
+ values.addAll(byDirectoryCache.values());
+ for (CacheValue val : values) {
try {
// if there are still refs out, we have to wait for them
int cnt = 0;
@@ -135,7 +172,15 @@ public abstract class CachingDirectoryFa
break;
}
}
-
+ assert val.refCnt == 0 : val.refCnt;
+ } catch (Throwable t) {
+ SolrException.log(log, "Error closing directory", t);
+ }
+ }
+
+ values = byDirectoryCache.values();
+ for (CacheValue val : values) {
+ try {
assert val.refCnt == 0 : val.refCnt;
log.info("Closing directory when closing factory: " + val.path);
closeDirectory(val);
@@ -143,8 +188,14 @@ public abstract class CachingDirectoryFa
SolrException.log(log, "Error closing directory", t);
}
}
+
byDirectoryCache.clear();
byPathCache.clear();
+
+ for (CacheValue val : removeEntries) {
+ log.info("Removing directory: " + val.path);
+ removeDirectory(val);
+ }
}
}
@@ -166,7 +217,13 @@ public abstract class CachingDirectoryFa
closeDirectory(cacheValue);
byDirectoryCache.remove(directory);
- byPathCache.remove(cacheValue.path);
+
+ // if it's been closed, it's path is now
+ // owned by another Directory instance
+ if (!cacheValue.latestForPath) {
+ byPathCache.remove(cacheValue.path);
+ cacheValue.latestForPath = true;
+ }
}
}
}
@@ -182,13 +239,56 @@ public abstract class CachingDirectoryFa
}
}
}
- try {
- log.info("Closing directory: " + cacheValue.path);
- cacheValue.directory.close();
- } catch (Throwable t) {
- SolrException.log(log, "Error closing directory", t);
+
+ cacheValue.closeDirectoryCalled = true;
+
+ if (cacheValue.deleteOnClose) {
+
+ // see if we are a subpath
+ Collection<CacheValue> values = byPathCache.values();
+
+ Collection<CacheValue> cacheValues = new ArrayList<CacheValue>();
+ cacheValues.addAll(values);
+ cacheValues.remove(cacheValue);
+ for (CacheValue otherCacheValue : cacheValues) {
+ // if we are a parent path and all our sub children are not already closed,
+ // get a sub path to close us later
+ if (otherCacheValue.path.startsWith(cacheValue.path) && !otherCacheValue.closeDirectoryCalled) {
+ // we let the sub dir remove and close us
+ if (!otherCacheValue.deleteAfterCoreClose && cacheValue.deleteAfterCoreClose) {
+ otherCacheValue.deleteAfterCoreClose = true;
+ }
+ otherCacheValue.removeEntries.addAll(cacheValue.removeEntries);
+ otherCacheValue.closeEntries.addAll(cacheValue.closeEntries);
+ cacheValue.closeEntries.clear();
+ break;
+ }
+ }
+ }
+
+ for (CacheValue val : cacheValue.removeEntries) {
+ if (!val.deleteAfterCoreClose) {
+ try {
+ log.info("Removing directory: " + val.path);
+ removeDirectory(val);
+ } catch (Throwable t) {
+ SolrException.log(log, "Error removing directory", t);
+ }
+ } else {
+ removeEntries.add(val);
+ }
}
+ for (CacheValue val : cacheValue.closeEntries) {
+ try {
+ log.info("Closing directory: " + val.path);
+ val.directory.close();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error closing directory", t);
+ }
+
+ }
+
if (listeners != null) {
for (CloseListener listener : listeners) {
try {
@@ -199,12 +299,12 @@ public abstract class CachingDirectoryFa
}
}
}
-
+
@Override
protected abstract Directory create(String path, DirContext dirContext) throws IOException;
@Override
- public boolean exists(String path) {
+ public boolean exists(String path) throws IOException {
// back compat behavior
File dirFile = new File(path);
return dirFile.canRead() && dirFile.list().length > 0;
@@ -231,7 +331,7 @@ public abstract class CachingDirectoryFa
@Override
public final Directory get(String path, DirContext dirContext, String rawLockType, boolean forceNew)
throws IOException {
- String fullPath = new File(path).getAbsolutePath();
+ String fullPath = normalize(path);
synchronized (this) {
if (closed) {
throw new RuntimeException("Already closed");
@@ -247,18 +347,14 @@ public abstract class CachingDirectoryFa
// we make a quick close attempt,
// otherwise this should be closed
// when whatever is using it, releases it
-
if (cacheValue.refCnt == 0) {
- try {
- // the following will decref, so
- // first incref
- cacheValue.refCnt++;
- close(cacheValue.directory);
- } catch (IOException e) {
- SolrException.log(log, "Error closing directory", e);
- }
+ closeDirectory(cacheValue);
}
+ // close the entry, it will be owned by the new dir
+ // we count on it being released by directory
+ cacheValue.latestForPath = true;
+
}
}
@@ -267,11 +363,9 @@ public abstract class CachingDirectoryFa
directory = rateLimit(directory);
- CacheValue newCacheValue = new CacheValue();
- newCacheValue.directory = directory;
- newCacheValue.path = fullPath;
+ CacheValue newCacheValue = new CacheValue(fullPath, directory);
- injectLockFactory(directory, path, rawLockType);
+ injectLockFactory(directory, fullPath, rawLockType);
byDirectoryCache.put(directory, newCacheValue);
byPathCache.put(fullPath, newCacheValue);
@@ -345,6 +439,38 @@ public abstract class CachingDirectoryFa
close(directory);
}
+ @Override
+ public void remove(String path) throws IOException {
+ remove(path, false);
+ }
+
+ @Override
+ public void remove(Directory dir) throws IOException {
+ remove(dir, false);
+ }
+
+ @Override
+ public void remove(String path, boolean deleteAfterCoreClose) throws IOException {
+ synchronized (this) {
+ CacheValue val = byPathCache.get(normalize(path));
+ if (val == null) {
+ throw new IllegalArgumentException("Unknown directory " + path);
+ }
+ val.setDeleteOnClose(true, deleteAfterCoreClose);
+ }
+ }
+
+ @Override
+ public void remove(Directory dir, boolean deleteAfterCoreClose) throws IOException {
+ synchronized (this) {
+ CacheValue val = byDirectoryCache.get(dir);
+ if (val == null) {
+ throw new IllegalArgumentException("Unknown directory " + dir);
+ }
+ val.setDeleteOnClose(true, deleteAfterCoreClose);
+ }
+ }
+
private static Directory injectLockFactory(Directory dir, String lockPath,
String rawLockType) throws IOException {
if (null == rawLockType) {
@@ -372,4 +498,21 @@ public abstract class CachingDirectoryFa
}
return dir;
}
+
+ protected void removeDirectory(CacheValue cacheValue) throws IOException {
+ empty(cacheValue.directory);
+ }
+
+ @Override
+ public String normalize(String path) throws IOException {
+ path = stripTrailingSlash(path);
+ return path;
+ }
+
+ private String stripTrailingSlash(String path) {
+ if (path.endsWith("/")) {
+ path = path.substring(0, path.length() - 1);
+ }
+ return path;
+ }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlBackCompat.java Wed Mar 20 16:27:13 2013
@@ -25,6 +25,7 @@ import org.apache.solr.schema.IndexSchem
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.PropertiesUtil;
import org.apache.solr.util.SystemIdResolver;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.zookeeper.KeeperException;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
@@ -132,9 +133,16 @@ public class ConfigSolrXmlBackCompat ext
m.put("class", HttpShardHandlerFactory.class.getName());
info = new PluginInfo("shardHandlerFactory", m, null, Collections.<PluginInfo>emptyList());
}
- HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
- if (info != null) {
- fac.init(info);
+
+ ShardHandlerFactory fac;
+ try {
+ fac = getResourceLoader().findClass(info.className, ShardHandlerFactory.class).newInstance();
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error instantiating shardHandlerFactory class " + info.className);
+ }
+ if (fac instanceof PluginInfoInitialized) {
+ ((PluginInfoInitialized) fac).init(info);
}
return fac;
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreContainer.java Wed Mar 20 16:27:13 2013
@@ -377,7 +377,7 @@ public class CoreContainer
cfg = new ConfigSolrXmlBackCompat(loader, null, is, null, false);
this.cfg = new ConfigSolrXmlBackCompat(loader, (ConfigSolrXmlBackCompat)cfg);
} else {
- cfg = new SolrProperties(this, is, fileName);
+ cfg = new SolrProperties(this, loader, is, fileName);
this.cfg = new SolrProperties(this, loader, (SolrProperties)cfg);
}
} catch (Exception e) {
@@ -497,7 +497,6 @@ public class CoreContainer
TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
new DefaultSolrThreadFactory("coreLoadExecutor"));
try {
-
CompletionService<SolrCore> completionService = new ExecutorCompletionService<SolrCore>(
coreLoadExecutor);
Set<Future<SolrCore>> pending = new HashSet<Future<SolrCore>>();
@@ -588,7 +587,6 @@ public class CoreContainer
return c;
}
};
-
pending.add(completionService.submit(task));
} else {
@@ -607,7 +605,7 @@ public class CoreContainer
Future<SolrCore> future = completionService.take();
if (future == null) return;
pending.remove(future);
-
+
try {
SolrCore c = future.get();
// track original names
@@ -680,7 +678,7 @@ public class CoreContainer
try {
- // First allow the closer thread to drain all the pending closes it can.
+ // First wake up the closer thread, it'll terminate almost immediately since it checks isShutDown.
synchronized (coreMaps.getLocker()) {
coreMaps.getLocker().notifyAll(); // wake up anyone waiting
}
@@ -1032,53 +1030,55 @@ public class CoreContainer
*/
public void reload(String name) {
try {
+ name = checkDefault(name);
- name= checkDefault(name);
SolrCore core = coreMaps.getCore(name);
if (core == null)
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "No such core: " + name );
- CoreDescriptor cd = core.getCoreDescriptor();
-
- File instanceDir = new File(cd.getInstanceDir());
+ try {
+ coreMaps.waitAddPendingCoreOps(name);
+ CoreDescriptor cd = core.getCoreDescriptor();
- log.info("Reloading SolrCore '{}' using instanceDir: {}",
- cd.getName(), instanceDir.getAbsolutePath());
-
- SolrResourceLoader solrLoader;
- if(zkController == null) {
- solrLoader = new SolrResourceLoader(instanceDir.getAbsolutePath(), libLoader, SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd));
- } else {
- try {
- String collection = cd.getCloudDescriptor().getCollectionName();
- zkController.createCollectionZkNode(cd.getCloudDescriptor());
+ File instanceDir = new File(cd.getInstanceDir());
+
+ log.info("Reloading SolrCore '{}' using instanceDir: {}",
+ cd.getName(), instanceDir.getAbsolutePath());
+ SolrResourceLoader solrLoader;
+ if(zkController == null) {
+ solrLoader = new SolrResourceLoader(instanceDir.getAbsolutePath(), libLoader, SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd));
+ } else {
+ try {
+ String collection = cd.getCloudDescriptor().getCollectionName();
+ zkController.createCollectionZkNode(cd.getCloudDescriptor());
- String zkConfigName = zkController.readConfigName(collection);
- if (zkConfigName == null) {
- log.error("Could not find config name for collection:" + collection);
+ String zkConfigName = zkController.readConfigName(collection);
+ if (zkConfigName == null) {
+ log.error("Could not find config name for collection:" + collection);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "Could not find config name for collection:" + collection);
+ }
+ solrLoader = new ZkSolrResourceLoader(instanceDir.getAbsolutePath(), zkConfigName, libLoader,
+ SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd), zkController);
+ } catch (KeeperException e) {
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
- "Could not find config name for collection:" + collection);
+ "", e);
}
- solrLoader = new ZkSolrResourceLoader(instanceDir.getAbsolutePath(), zkConfigName, libLoader,
- SolrProperties.getCoreProperties(instanceDir.getAbsolutePath(), cd), zkController);
- } catch (KeeperException e) {
- log.error("", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
- "", e);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- log.error("", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
- "", e);
}
+ SolrCore newCore = core.reload(solrLoader, core);
+ // keep core to orig name link
+ coreMaps.removeCoreToOrigName(newCore, core);
+ registerCore(false, name, newCore, false);
+ } finally {
+ coreMaps.removeFromPendingOps(name);
}
- SolrCore newCore = core.reload(solrLoader, core);
- // keep core to orig name link
- coreMaps.removeCoreToOrigName(newCore, core);
-
- registerCore(false, name, newCore, false);
-
// :TODO: Java7...
// http://docs.oracle.com/javase/7/docs/technotes/guides/language/catch-multiple.html
} catch (Exception ex) {
@@ -1137,7 +1137,10 @@ public class CoreContainer
// Do this in two phases since we don't want to lock access to the cores over a load.
SolrCore core = coreMaps.getCoreFromAnyList(name);
- if (core != null) return core;
+ if (core != null) {
+ core.open();
+ return core;
+ }
// OK, it's not presently in any list, is it in the list of dynamic cores but not loaded yet? If so, load it.
CoreDescriptor desc = coreMaps.getDynamicDescriptor(name);
@@ -1145,22 +1148,25 @@ public class CoreContainer
return null;
}
- core = coreMaps.waitPendingCoreOps(name); // This will put an entry in pending core ops if the core isn't loaded
+ // This will put an entry in pending core ops if the core isn't loaded
+ core = coreMaps.waitAddPendingCoreOps(name);
if (isShutDown) return null; // We're quitting, so stop. This needs to be after the wait above since we may come off
// the wait as a consequence of shutting down.
-
- if (core == null) {
- try {
+ try {
+ if (core == null) {
core = create(desc); // This should throw an error if it fails.
core.open();
registerCore(desc.isTransient(), name, core, false);
- } catch (Exception ex) {
- throw recordAndThrow(name, "Unable to create core: " + name, ex);
- } finally {
- coreMaps.releasePending(name);
+ } else {
+ core.open();
}
+ } catch(Exception ex){
+ throw recordAndThrow(name, "Unable to create core: " + name, ex);
+ } finally {
+ coreMaps.removeFromPendingOps(name);
}
+
return core;
}
@@ -1371,6 +1377,7 @@ public class CoreContainer
// dynamicDescriptors
//
+
class CoreMaps {
private static Object locker = new Object(); // for locking around manipulating any of the core maps.
@@ -1387,11 +1394,13 @@ class CoreMaps {
private final CoreContainer container;
- // It's a little clumsy to have two, but closing requires a SolrCore, whereas pending loads don't have a core.
- private static final Set<String> pendingDynamicLoads = new TreeSet<String>();
-
- // Holds cores from the time they're removed from the transient cache until after they're closed.
- private static final List<SolrCore> pendingDynamicCloses = new ArrayList<SolrCore>();
+ // This map will hold objects that are being currently operated on. The core (value) may be null in the case of
+ // initial load. The rule is, never to any operation on a core that is currently being operated upon.
+ private static final Set<String> pendingCoreOps = new HashSet<String>();
+
+ // Due to the fact that closes happen potentially whenever anything is _added_ to the transient core list, we need
+ // to essentially queue them up to be handled via pendingCoreOps.
+ private static final List<SolrCore> pendingCloses = new ArrayList<SolrCore>();
CoreMaps(CoreContainer container) {
this.container = container;
@@ -1408,11 +1417,8 @@ class CoreMaps {
protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
if (size() > transientCacheSize) {
synchronized (locker) {
- SolrCore closeMe = eldest.getValue();
- synchronized (locker) {
- pendingDynamicCloses.add(closeMe);
- locker.notifyAll(); // Wakes up closer thread too
- }
+ pendingCloses.add(eldest.getValue()); // Essentially just queue this core up for closing.
+ locker.notifyAll(); // Wakes up closer thread too
}
return true;
}
@@ -1433,11 +1439,11 @@ class CoreMaps {
protected void clearMaps(ConfigSolr cfg) {
List<String> coreNames;
List<String> transientNames;
- List<SolrCore> pendingClosers;
+ List<SolrCore> pendingToClose;
synchronized (locker) {
coreNames = new ArrayList(cores.keySet());
transientNames = new ArrayList(transientCores.keySet());
- pendingClosers = new ArrayList(pendingDynamicCloses);
+ pendingToClose = new ArrayList(pendingCloses);
}
for (String coreName : coreNames) {
SolrCore core = cores.get(coreName);
@@ -1466,8 +1472,12 @@ class CoreMaps {
transientCores.clear();
// We might have some cores that we were _thinking_ about shutting down, so take care of those too.
- for (SolrCore core : pendingClosers) {
- core.close();
+ for (SolrCore core : pendingToClose) {
+ try {
+ core.close();
+ } catch (Throwable t) {
+ SolrException.log(CoreContainer.log, "Error shutting down core", t);
+ }
}
}
@@ -1610,10 +1620,9 @@ class CoreMaps {
protected SolrCore getCoreFromAnyList(String name) {
SolrCore core;
- synchronized (locker) { // This one's OK, the core.open is just an increment
+ synchronized (locker) {
core = cores.get(name);
if (core != null) {
- core.open(); // increment the ref count while still synchronized
return core;
}
@@ -1621,15 +1630,8 @@ class CoreMaps {
return null; // Nobody even tried to define any transient cores, so we're done.
}
// Now look for already loaded transient cores.
- core = transientCores.get(name);
- if (core != null) {
- core.open(); // Just increments ref count, so it's ok that we're in a synch block
- return core;
- }
+ return transientCores.get(name);
}
-
- return null;
-
}
protected CoreDescriptor getDynamicDescriptor(String name) {
@@ -1719,29 +1721,22 @@ class CoreMaps {
}
}
}
- // We get here when we're being loaded, and the presumption is that we're not in the list yet.
- protected SolrCore waitPendingCoreOps(String name) {
-
- // Keep multiple threads from opening or closing a core at one time.
- SolrCore ret = null;
+ // Wait here until any pending operations (load, unload or reload) are completed on this core.
+ protected SolrCore waitAddPendingCoreOps(String name) {
+ // Keep multiple threads from operating on a core at one time.
synchronized (locker) {
boolean pending;
- do { // We're either loading or unloading this core,
- pending = pendingDynamicLoads.contains(name); // wait for the core to be loaded
- if (! pending) {
- // Check pending closes. This is a linear search is inefficient, but maps don't work without a lot of complexity,
- // we'll live with it unless it proves to be a bottleneck. In the "usual" case, this list shouldn't be
- // very long. In the stress test associated with SOLR-4196, this hovered around 0-3, occasionally spiking
- // very briefly to around 30.
- for (SolrCore core : pendingDynamicCloses) {
+ do { // Are we currently doing anything to this core? Loading, unloading, reloading?
+ pending = pendingCoreOps.contains(name); // wait for the core to be done being operated upon
+ if (! pending) { // Linear list, but shouldn't be too long
+ for (SolrCore core : pendingCloses) {
if (core.getName().equals(name)) {
pending = true;
break;
}
}
}
-
if (container.isShutDown()) return null; // Just stop already.
if (pending) {
@@ -1752,26 +1747,29 @@ class CoreMaps {
}
}
} while (pending);
-
- if (!container.isShutDown()) {
- ret = getCoreFromAnyList(name); // we might have been _unloading_ the core, so check.
- if (ret == null) {
- pendingDynamicLoads.add(name); // the caller is going to load us. If we happen to be shutting down, we don't care.
+ // We _really_ need to do this within the synchronized block!
+ if (! container.isShutDown()) {
+ if (! pendingCoreOps.add(name)) {
+ CoreContainer.log.warn("Replaced an entry in pendingCoreOps {}, we should not be doing this", name);
}
+ return getCoreFromAnyList(name); // we might have been _unloading_ the core, so return the core if it was loaded.
}
}
-
- return ret;
+ return null;
}
- // The core is loaded, remove it from the pendin gloads
- protected void releasePending(String name) {
+ // We should always be removing the first thing in the list with our name! The idea here is to NOT do anything n
+ // any core while some other operation is working on that core.
+ protected void removeFromPendingOps(String name) {
synchronized (locker) {
- pendingDynamicLoads.remove(name);
+ if (! pendingCoreOps.remove(name)) {
+ CoreContainer.log.warn("Tried to remove core {} from pendingCoreOps and it wasn't there. ", name);
+ }
locker.notifyAll();
}
}
+
protected void persistCores(ConfigSolr cfg, Map<String, SolrCore> whichCores, SolrResourceLoader loader) {
for (SolrCore solrCore : whichCores.values()) {
addPersistOneCore(cfg, solrCore, loader);
@@ -1880,13 +1878,14 @@ class CoreMaps {
// Be a little careful. We don't want to either open or close a core unless it's _not_ being opened or closed by
// another thread. So within this lock we'll walk along the list of pending closes until we find something NOT in
- // the list of threads currently being opened. The "usual" case will probably return the very first one anyway..
+ // the list of threads currently being loaded or reloaded. The "usual" case will probably return the very first
+ // one anyway..
protected SolrCore getCoreToClose() {
synchronized (locker) {
- if (pendingDynamicCloses.size() == 0) return null; // nothing to do.
- // Yes, a linear search but this is a pretty short list in the normal case and usually we'll take the first one.
- for (SolrCore core : pendingDynamicCloses) {
- if (! pendingDynamicLoads.contains(core.getName())) { // Don't try close a core if it's being opened.
+ for (SolrCore core : pendingCloses) {
+ if (! pendingCoreOps.contains(core.getName())) {
+ pendingCoreOps.add(core.getName());
+ pendingCloses.remove(core);
return core;
}
}
@@ -1894,12 +1893,7 @@ class CoreMaps {
return null;
}
- protected void removeClosedFromCloser(SolrCore core) {
- synchronized (locker) {
- pendingDynamicCloses.remove(core);
- locker.notifyAll();
- }
- }
+
}
class CloserThread extends Thread {
@@ -1936,7 +1930,7 @@ class CloserThread extends Thread {
coreMaps.addPersistOneCore(cfg, removeMe, container.loader);
removeMe.close();
} finally {
- coreMaps.removeClosedFromCloser(removeMe);
+ coreMaps.removeFromPendingOps(removeMe.getName());
}
}
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java Wed Mar 20 16:27:13 2013
@@ -64,6 +64,8 @@ public class CoreDescriptor {
// them individually.
private Properties coreProperties = new Properties();
+ private boolean loadedImplicit = false;
+
private final CoreContainer coreContainer;
private CloudDescriptor cloudDesc;
@@ -133,11 +135,11 @@ public class CoreDescriptor {
public Properties initImplicitProperties() {
Properties implicitProperties = new Properties(coreContainer.getContainerProperties());
- implicitProperties.setProperty(CORE_NAME, getName());
- implicitProperties.setProperty(CORE_INSTDIR, getInstanceDir());
- implicitProperties.setProperty(CORE_DATADIR, getDataDir());
- implicitProperties.setProperty(CORE_CONFIG, getConfigName());
- implicitProperties.setProperty(CORE_SCHEMA, getSchemaName());
+ implicitProperties.setProperty("solr.core.name", getName());
+ implicitProperties.setProperty("solr.core.instanceDir", getInstanceDir());
+ implicitProperties.setProperty("solr.core.dataDir", getDataDir());
+ implicitProperties.setProperty("solr.core.configName", getConfigName());
+ implicitProperties.setProperty("solr.core.schemaName", getSchemaName());
return implicitProperties;
}
@@ -166,19 +168,8 @@ public class CoreDescriptor {
public String getDataDir() {
String dataDir = coreProperties.getProperty(CORE_DATADIR);
- if (dataDir == null) {
- dataDir = getDefaultDataDir();
- }
- if (new File(dataDir).isAbsolute()) {
- return dataDir;
- } else {
- if (new File(getInstanceDir()).isAbsolute()) {
- return SolrResourceLoader.normalizeDir(SolrResourceLoader.normalizeDir(getInstanceDir()) + dataDir);
- } else {
- return SolrResourceLoader.normalizeDir(coreContainer.getSolrHome() +
- SolrResourceLoader.normalizeDir(getRawInstanceDir()) + dataDir);
- }
- }
+ if (dataDir == null) dataDir = getDefaultDataDir();
+ return dataDir;
}
public void setDataDir(String s) {
@@ -262,13 +253,14 @@ public class CoreDescriptor {
* Under any circumstance, the properties passed in will override any already present.Merge
*/
public void setCoreProperties(Properties coreProperties) {
- if (this.coreProperties == null) {
+ if (! loadedImplicit) {
+ loadedImplicit = true;
Properties p = initImplicitProperties();
- this.coreProperties = new Properties(p);
- }
- // The caller presumably wants whatever properties passed in to override the current core props, so just add them.
- if(coreProperties != null) {
- this.coreProperties.putAll(coreProperties);
+ this.coreProperties.putAll(p);
+ // The caller presumably wants whatever properties passed in to override the current core props, so just add them.
+ if (coreProperties != null) {
+ this.coreProperties.putAll(coreProperties);
+ }
}
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -18,6 +18,7 @@ package org.apache.solr.core;
*/
import java.io.Closeable;
+import java.io.File;
import java.io.IOException;
import org.apache.lucene.store.Directory;
@@ -77,9 +78,10 @@ public abstract class DirectoryFactory i
/**
* Returns true if a Directory exists for a given path.
+ * @throws IOException If there is a low-level I/O error.
*
*/
- public abstract boolean exists(String path);
+ public abstract boolean exists(String path) throws IOException;
/**
* Removes the Directory's persistent storage.
@@ -91,6 +93,26 @@ public abstract class DirectoryFactory i
public abstract void remove(Directory dir) throws IOException;
/**
+ * Removes the Directory's persistent storage.
+ * For example: A file system impl may remove the
+ * on disk directory.
+ * @throws IOException If there is a low-level I/O error.
+ *
+ */
+ public abstract void remove(Directory dir, boolean afterCoreClose) throws IOException;
+
+ /**
+ * This remove is special in that it may be called even after
+ * the factory has been closed. Remove only makes sense for
+ * persistent directory factories.
+ *
+ * @param path to remove
+ * @param afterCoreClose whether to wait until after the core is closed.
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public abstract void remove(String path, boolean afterCoreClose) throws IOException;
+
+ /**
* This remove is special in that it may be called even after
* the factory has been closed. Remove only makes sense for
* persistent directory factories.
@@ -172,6 +194,15 @@ public abstract class DirectoryFactory i
return path;
}
+ /**
+ * @param path the path to check
+ * @return true if absolute, as in not relative
+ */
+ public boolean isAbsolute(String path) {
+ // back compat
+ return new File(path).isAbsolute();
+ }
+
public static long sizeOfDirectory(Directory directory) throws IOException {
final String[] files = directory.listAll();
long size = 0;
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -19,7 +19,9 @@ package org.apache.solr.core;
import java.io.File;
import java.io.IOException;
+import org.apache.commons.io.FileUtils;
import org.apache.lucene.store.Directory;
+import org.apache.solr.core.CachingDirectoryFactory.CacheValue;
/**
* Directory provider for implementations that do not persist over reboots.
@@ -28,8 +30,8 @@ import org.apache.lucene.store.Directory
public abstract class EphemeralDirectoryFactory extends CachingDirectoryFactory {
@Override
- public boolean exists(String path) {
- String fullPath = new File(path).getAbsolutePath();
+ public boolean exists(String path) throws IOException {
+ String fullPath = normalize(path);
synchronized (this) {
CacheValue cacheValue = byPathCache.get(fullPath);
Directory directory = null;
@@ -49,17 +51,7 @@ public abstract class EphemeralDirectory
}
@Override
- public void remove(Directory dir) throws IOException {
- // ram dir does not persist its dir anywhere
- }
-
- @Override
- public void remove(String path) throws IOException {
- // ram dir does not persist its dir anywhere
- }
-
- @Override
- public String normalize(String path) throws IOException {
- return path;
+ public boolean isAbsolute(String path) {
+ return true;
}
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -66,4 +66,9 @@ public class MMapDirectoryFactory extend
}
return mapDirectory;
}
+
+ @Override
+ public boolean isAbsolute(String path) {
+ return new File(path).isAbsolute();
+ }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -16,13 +16,12 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.NIOFSDirectory;
-import org.apache.solr.core.DirectoryFactory.DirContext;
-
import java.io.File;
import java.io.IOException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.NIOFSDirectory;
+
/**
* Factory to instantiate {@link org.apache.lucene.store.NIOFSDirectory}
@@ -35,4 +34,9 @@ public class NIOFSDirectoryFactory exten
return new NIOFSDirectory(new File(path));
}
+ @Override
+ public boolean isAbsolute(String path) {
+ return new File(path).isAbsolute();
+ }
+
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -25,7 +25,6 @@ import org.apache.lucene.store.FSDirecto
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.DirectoryFactory.DirContext;
/**
* Factory to instantiate {@link org.apache.lucene.store.NRTCachingDirectory}
@@ -54,5 +53,10 @@ public class NRTCachingDirectoryFactory
protected Directory create(String path, DirContext dirContext) throws IOException {
return new NRTCachingDirectory(FSDirectory.open(new File(path)), maxMergeSizeMB, maxCachedMB);
}
+
+ @Override
+ public boolean isAbsolute(String path) {
+ return new File(path).isAbsolute();
+ }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -16,13 +16,12 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.SimpleFSDirectory;
-import org.apache.solr.core.DirectoryFactory.DirContext;
-
import java.io.File;
import java.io.IOException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.SimpleFSDirectory;
+
/**
* Factory to instantiate {@link org.apache.lucene.store.SimpleFSDirectory}
@@ -35,4 +34,8 @@ public class SimpleFSDirectoryFactory ex
return new SimpleFSDirectory(new File(path));
}
+ @Override
+ public boolean isAbsolute(String path) {
+ return new File(path).isAbsolute();
+ }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrCore.java Wed Mar 20 16:27:13 2013
@@ -406,7 +406,7 @@ public final class SolrCore implements S
}
SolrCore core = new SolrCore(getName(), getDataDir(), config,
- schema, coreDescriptor, updateHandler, prev);
+ schema, coreDescriptor, updateHandler, this.solrDelPolicy, prev);
core.solrDelPolicy = this.solrDelPolicy;
core.getUpdateHandler().getSolrCoreState().newIndexWriter(core, false, false);
@@ -459,7 +459,7 @@ public final class SolrCore implements S
boolean indexExists = getDirectoryFactory().exists(indexDir);
boolean firstTime;
synchronized (SolrCore.class) {
- firstTime = dirs.add(new File(indexDir).getCanonicalPath());
+ firstTime = dirs.add(getDirectoryFactory().normalize(indexDir));
}
boolean removeLocks = solrConfig.unlockOnStartup;
@@ -616,7 +616,7 @@ public final class SolrCore implements S
* @since solr 1.3
*/
public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd) {
- this(name, dataDir, config, schema, cd, null, null);
+ this(name, dataDir, config, schema, cd, null, null, null);
}
@@ -652,16 +652,28 @@ public final class SolrCore implements S
*
*@since solr 1.3
*/
- public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd, UpdateHandler updateHandler, SolrCore prev) {
+ public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd, UpdateHandler updateHandler, IndexDeletionPolicyWrapper delPolicy, SolrCore prev) {
coreDescriptor = cd;
this.setName( name );
resourceLoader = config.getResourceLoader();
- if (dataDir == null){
- if(cd.usingDefaultDataDir()) {
- dataDir = config.getDataDir();
- }
- if(dataDir == null) {
+ this.solrConfig = config;
+
+ if (updateHandler == null) {
+ initDirectoryFactory();
+ }
+
+ if (dataDir == null) {
+ if (cd.usingDefaultDataDir()) dataDir = config.getDataDir();
+ if (dataDir == null) {
dataDir = cd.getDataDir();
+ try {
+ if (!directoryFactory.isAbsolute(dataDir)) {
+ dataDir = directoryFactory.normalize(SolrResourceLoader
+ .normalizeDir(cd.getInstanceDir()) + dataDir);
+ }
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e);
+ }
}
}
@@ -703,7 +715,6 @@ public final class SolrCore implements S
this.schema = schema;
this.dataDir = dataDir;
- this.solrConfig = config;
this.startTime = System.currentTimeMillis();
this.maxWarmingSearchers = config.maxWarmingSearchers;
@@ -715,8 +726,10 @@ public final class SolrCore implements S
initListeners();
- if (updateHandler == null) {
+ if (delPolicy == null) {
initDeletionPolicy();
+ } else {
+ this.solrDelPolicy = delPolicy;
}
this.codec = initCodec(solrConfig, schema);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/SolrProperties.java Wed Mar 20 16:27:13 2013
@@ -27,6 +27,7 @@ import org.apache.solr.handler.component
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.PropertiesUtil;
import org.apache.solr.util.SystemIdResolver;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,13 +105,15 @@ public class SolrProperties implements C
* Create a SolrProperties object from an opened input stream, useful for creating defaults
*
* @param container - the container for this Solr instance. There should be one and only one...
+ * @param loader - Solr resource loader
* @param is - Input stream for loading properties.
* @param fileName - the name for this properties object.
* @throws IOException - It's possible to walk a very deep tree, if that process goes awry, or if reading any
* of the files found doesn't work, you'll get an IO exception
*/
- public SolrProperties(CoreContainer container, InputStream is, String fileName) throws IOException {
+ public SolrProperties(CoreContainer container, SolrResourceLoader loader, InputStream is, String fileName) throws IOException {
origsolrprops.load(is);
+ this.loader = loader;
this.container = container;
init(fileName);
}
@@ -241,29 +244,38 @@ public class SolrProperties implements C
boolean haveHandler = false;
for (String s : solrProperties.stringPropertyNames()) {
String val = solrProperties.getProperty(s);
- if (s.indexOf(SHARD_HANDLER_FACTORY) != -1) {
+ int index = s.indexOf(SHARD_HANDLER_FACTORY);
+ if (index != -1) {
haveHandler = true;
if (SHARD_HANDLER_NAME.equals(s) || SHARD_HANDLER_CLASS.equals(s)) {
- attrs.put(s, val);
+ // remove shardHandlerFactory. prefix
+ attrs.put(s.substring(SHARD_HANDLER_FACTORY.length()+1), val);
} else {
- args.add(s, val);
+ // remove shardHandlerFactory. prefix
+ args.add(s.substring(SHARD_HANDLER_FACTORY.length()+1), val);
}
}
}
if (haveHandler) {
- // public PluginInfo(String type, Map<String, String> attrs ,NamedList initArgs, List<PluginInfo> children) {
-
info = new PluginInfo(SHARD_HANDLER_FACTORY, attrs, args, null);
} else {
Map m = new HashMap();
m.put("class", HttpShardHandlerFactory.class.getName());
info = new PluginInfo("shardHandlerFactory", m, null, Collections.<PluginInfo>emptyList());
}
- HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
- if (info != null) {
- fac.init(info);
+
+ assert loader != null;
+ ShardHandlerFactory fac;
+ try {
+ fac = loader.findClass(info.className, ShardHandlerFactory.class).newInstance();
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ if (fac instanceof PluginInfoInitialized) {
+ ((PluginInfoInitialized) fac).init(info);
}
+
return fac;
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java Wed Mar 20 16:27:13 2013
@@ -25,6 +25,7 @@ import org.apache.lucene.store.FSDirecto
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.store.RateLimitedDirectoryWrapper;
+import org.apache.solr.core.CachingDirectoryFactory.CacheValue;
/**
* Directory provider which mimics original Solr
@@ -43,7 +44,9 @@ public class StandardDirectoryFactory ex
@Override
public String normalize(String path) throws IOException {
- return new File(path).getCanonicalPath();
+ String cpath = new File(path).getCanonicalPath();
+
+ return super.normalize(cpath);
}
public boolean isPersistent() {
@@ -51,20 +54,14 @@ public class StandardDirectoryFactory ex
}
@Override
- public void remove(Directory dir) throws IOException {
- CacheValue val = byDirectoryCache.get(dir);
- if (val == null) {
- throw new IllegalArgumentException("Unknown directory " + dir);
- }
- File dirFile = new File(val.path);
- FileUtils.deleteDirectory(dirFile);
+ public boolean isAbsolute(String path) {
+ // back compat
+ return new File(path).isAbsolute();
}
-
@Override
- public void remove(String path) throws IOException {
- String fullPath = new File(path).getAbsolutePath();
- File dirFile = new File(fullPath);
+ protected void removeDirectory(CacheValue cacheValue) throws IOException {
+ File dirFile = new File(cacheValue.path);
FileUtils.deleteDirectory(dirFile);
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Wed Mar 20 16:27:13 2013
@@ -439,8 +439,12 @@ public class SnapPuller {
@Override
public void preClose() {
LOG.info("removing old index files " + freezeIndexDir);
- if (core.getDirectoryFactory().exists(freezeIndexDirPath)) {
- DirectoryFactory.empty(freezeIndexDir);
+ try {
+ if (core.getDirectoryFactory().exists(freezeIndexDirPath)) {
+ DirectoryFactory.empty(freezeIndexDir);
+ }
+ } catch (IOException e) {
+ SolrException.log(LOG, null, e);
}
}
@@ -477,13 +481,6 @@ public class SnapPuller {
throw new InterruptedException("Index fetch interrupted");
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Index fetch failed : ", e);
- } finally {
- if (deleteTmpIdxDir) {
- LOG.info("removing temporary index download directory files " + tmpIndexDir);
- if (tmpIndex != null && core.getDirectoryFactory().exists(tmpIndex)) {
- DirectoryFactory.empty(tmpIndexDir);
- }
- }
}
} finally {
try {
@@ -500,9 +497,6 @@ public class SnapPuller {
stop = false;
fsyncException = null;
} finally {
- if (tmpIndexDir != null) {
- core.getDirectoryFactory().release(tmpIndexDir);
- }
if (deleteTmpIdxDir && tmpIndexDir != null) {
try {
core.getDirectoryFactory().remove(tmpIndexDir);
@@ -510,6 +504,11 @@ public class SnapPuller {
SolrException.log(LOG, "Error removing directory " + tmpIndexDir, e);
}
}
+
+ if (tmpIndexDir != null) {
+ core.getDirectoryFactory().release(tmpIndexDir);
+ }
+
if (indexDir != null) {
core.getDirectoryFactory().release(indexDir);
}
@@ -674,13 +673,14 @@ public class SnapPuller {
}
-
/**
* All the files are copied to a temp dir first
*/
private String createTempindexDir(SolrCore core, String tmpIdxDirName) {
- File tmpIdxDir = new File(core.getDataDir(), tmpIdxDirName);
- return tmpIdxDir.toString();
+ // TODO: there should probably be a DirectoryFactory#concatPath(parent, name)
+ // or something
+ String tmpIdxDir = core.getDataDir() + tmpIdxDirName;
+ return tmpIdxDir;
}
private void reloadCore() {
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Wed Mar 20 16:27:13 2013
@@ -40,6 +40,7 @@ import org.apache.solr.common.params.Cor
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
@@ -102,35 +103,38 @@ public class CollectionsHandler extends
if (a != null) {
action = CollectionAction.get(a);
}
- if (action != null) {
- switch (action) {
- case CREATE: {
- this.handleCreateAction(req, rsp);
- break;
- }
- case DELETE: {
- this.handleDeleteAction(req, rsp);
- break;
- }
- case RELOAD: {
- this.handleReloadAction(req, rsp);
- break;
- }
- case SYNCSHARD: {
- this.handleSyncShardAction(req, rsp);
- break;
- }
- case CREATEALIAS: {
- this.handleCreateAliasAction(req, rsp);
- break;
- }
- case DELETEALIAS: {
- this.handleDeleteAliasAction(req, rsp);
- break;
- }
- default: {
- throw new RuntimeException("Unknown action: " + action);
- }
+ if (action == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown action: " + a);
+ }
+
+ switch (action) {
+ case CREATE: {
+ this.handleCreateAction(req, rsp);
+ break;
+ }
+ case DELETE: {
+ this.handleDeleteAction(req, rsp);
+ break;
+ }
+ case RELOAD: {
+ this.handleReloadAction(req, rsp);
+ break;
+ }
+ case SYNCSHARD: {
+ this.handleSyncShardAction(req, rsp);
+ break;
+ }
+ case CREATEALIAS: {
+ this.handleCreateAliasAction(req, rsp);
+ break;
+ }
+ case DELETEALIAS: {
+ this.handleDeleteAliasAction(req, rsp);
+ break;
+ }
+ default: {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown action: "
+ + action);
}
}
@@ -148,6 +152,11 @@ public class CollectionsHandler extends
if (event.getBytes() != null) {
SolrResponse response = SolrResponse.deserialize(event.getBytes());
rsp.getValues().addAll(response.getResponse());
+ SimpleOrderedMap exp = (SimpleOrderedMap) response.getResponse().get("exception");
+ if (exp != null) {
+ Integer code = (Integer) exp.get("rspCode");
+ rsp.setException(new SolrException(code != null && code != -1 ? ErrorCode.getErrorCode(code) : ErrorCode.SERVER_ERROR, (String)exp.get("msg")));
+ }
} else {
if (System.currentTimeMillis() - time >= DEFAULT_ZK_TIMEOUT) {
throw new SolrException(ErrorCode.SERVER_ERROR, operation
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Wed Mar 20 16:27:13 2013
@@ -29,8 +29,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import javax.xml.parsers.ParserConfigurationException;
-
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.DirectoryReader;
@@ -73,7 +71,6 @@ import org.apache.solr.util.RefCounted;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
/**
*
@@ -408,8 +405,8 @@ public class CoreAdminHandler extends Re
//for now, do not allow creating new core with same name when in cloud mode
//XXX perhaps it should just be unregistered from cloud before readding it?,
//XXX perhaps we should also check that cores are of same type before adding new core to collection?
- if (coreContainer.getZkController() != null) {
- if (coreContainer.getCore(name) != null) {
+ if (coreContainer.isZooKeeperAware()) {
+ if (coreContainer.getCoreNames().contains(name)) {
log.info("Re-creating a core with existing name is not allowed in cloud mode");
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Core with name '" + name + "' already exists.");
@@ -568,43 +565,24 @@ public class CoreAdminHandler extends Re
+ e.getMessage(), e);
}
}
- }
- if (params.getBool(CoreAdminParams.DELETE_INDEX, false)) {
- core.addCloseHook(new CloseHook() {
- private String indexDir;
-
- @Override
- public void preClose(SolrCore core) {
- indexDir = core.getIndexDir();
- }
-
- @Override
- public void postClose(SolrCore core) {
- try {
- core.getDirectoryFactory().remove(indexDir);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+
+ if (params.getBool(CoreAdminParams.DELETE_INDEX, false)) {
+ try {
+ core.getDirectoryFactory().remove(core.getIndexDir());
+ } catch (Exception e) {
+ SolrException.log(log, "Failed to flag index dir for removal for core:"
+ + core.getName() + " dir:" + core.getIndexDir());
}
- });
+ }
}
-
+
if (params.getBool(CoreAdminParams.DELETE_DATA_DIR, false)) {
- core.addCloseHook(new CloseHook() {
- @Override
- public void preClose(SolrCore core) {}
-
- @Override
- public void postClose(SolrCore core) {
- File dataDir = new File(core.getDataDir());
- try {
- FileUtils.deleteDirectory(dataDir);
- } catch (IOException e) {
- SolrException.log(log, "Failed to delete data dir for core:"
- + core.getName() + " dir:" + dataDir.getAbsolutePath());
- }
- }
- });
+ try {
+ core.getDirectoryFactory().remove(core.getDataDir(), true);
+ } catch (Exception e) {
+ SolrException.log(log, "Failed to flag data dir for removal for core:"
+ + core.getName() + " dir:" + core.getDataDir());
+ }
}
if (params.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, false)) {
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Wed Mar 20 16:27:13 2013
@@ -18,7 +18,6 @@ package org.apache.solr.handler.componen
import java.net.ConnectException;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -27,7 +26,6 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import org.apache.http.client.HttpClient;
@@ -61,21 +59,20 @@ public class HttpShardHandler extends Sh
private HttpShardHandlerFactory httpShardHandlerFactory;
private CompletionService<ShardResponse> completionService;
- private Set<Future<ShardResponse>> pending;
+ private Set<Future<ShardResponse>> pending;
private Map<String,List<String>> shardToURLs;
private HttpClient httpClient;
-
public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, HttpClient httpClient) {
this.httpClient = httpClient;
this.httpShardHandlerFactory = httpShardHandlerFactory;
- completionService = new ExecutorCompletionService<ShardResponse>(httpShardHandlerFactory.commExecutor);
+ completionService = httpShardHandlerFactory.newCompletionService();
pending = new HashSet<Future<ShardResponse>>();
// maps "localhost:8983|localhost:7574" to a shuffled List("http://localhost:8983","http://localhost:7574")
- // This is primarily to keep track of what order we should use to query the replicas of a shard
- // so that we use the same replica for all phases of a distributed request.
+ // This is primarily to keep track of what order we should use to query the replicas of a shard
+ // so that we use the same replica for all phases of a distributed request.
shardToURLs = new HashMap<String,List<String>>();
}
@@ -106,21 +103,8 @@ public class HttpShardHandler extends Sh
// Don't modify the returned URL list.
private List<String> getURLs(String shard) {
List<String> urls = shardToURLs.get(shard);
- if (urls==null) {
- urls = StrUtils.splitSmart(shard, "|", true);
-
- // convert shard to URL
- for (int i=0; i<urls.size(); i++) {
- urls.set(i, httpShardHandlerFactory.scheme + urls.get(i));
- }
-
- //
- // Shuffle the list instead of use round-robin by default.
- // This prevents accidental synchronization where multiple shards could get in sync
- // and query the same replica at the same time.
- //
- if (urls.size() > 1)
- Collections.shuffle(urls, httpShardHandlerFactory.r);
+ if (urls == null) {
+ urls = httpShardHandlerFactory.makeURLList(shard);
shardToURLs.put(shard, urls);
}
return urls;
@@ -137,6 +121,9 @@ public class HttpShardHandler extends Sh
public ShardResponse call() throws Exception {
ShardResponse srsp = new ShardResponse();
+ if (sreq.nodeName != null) {
+ srsp.setNodeName(sreq.nodeName);
+ }
srsp.setShardRequest(sreq);
srsp.setShard(shard);
SimpleSolrResponse ssr = new SimpleSolrResponse();
@@ -168,7 +155,7 @@ public class HttpShardHandler extends Sh
SolrServer server = new HttpSolrServer(url, httpClient);
ssr.nl = server.request(req);
} else {
- LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.loadbalancer.request(new LBHttpSolrServer.Req(req, urls));
+ LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls);
ssr.nl = rsp.getResponse();
srsp.setShardAddress(rsp.getServer());
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Wed Mar 20 16:27:13 2013
@@ -16,24 +16,36 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import java.io.IOException;
import java.net.MalformedURLException;
+import java.util.Collections;
+import java.util.List;
import java.util.Random;
-import java.util.concurrent.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.LBHttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class HttpShardHandlerFactory extends ShardHandlerFactory implements PluginInfoInitialized {
+
+public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized {
protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
// We want an executor that doesn't take up any resources if
@@ -42,7 +54,7 @@ public class HttpShardHandlerFactory ext
//
// Consider CallerRuns policy and a lower max threads to throttle
// requests at some point (or should we simply return failure?)
- ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
+ private ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
0,
Integer.MAX_VALUE,
5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
@@ -51,7 +63,7 @@ public class HttpShardHandlerFactory ext
);
private HttpClient defaultClient;
- LBHttpSolrServer loadbalancer;
+ private LBHttpSolrServer loadbalancer;
//default values:
int soTimeout = 0;
int connectionTimeout = 0;
@@ -62,9 +74,9 @@ public class HttpShardHandlerFactory ext
int queueSize = -1;
boolean accessPolicy = false;
- public String scheme = "http://"; //current default values
+ private String scheme = "http://"; //current default values
- final Random r = new Random();
+ private final Random r = new Random();
// URL scheme to be used in distributed search.
static final String INIT_URL_SCHEME = "urlScheme";
@@ -112,6 +124,12 @@ public class HttpShardHandlerFactory ext
this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime);
this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, queueSize);
this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy);
+
+ // magic sysprop to make tests reproducible: set by SolrTestCaseJ4.
+ String v = System.getProperty("tests.shardhandler.randomSeed");
+ if (v != null) {
+ r.setSeed(Long.parseLong(v));
+ }
BlockingQueue<Runnable> blockingQueue = (this.queueSize == -1) ?
new SynchronousQueue<Runnable>(this.accessPolicy) :
@@ -177,4 +195,48 @@ public class HttpShardHandlerFactory ext
}
}
+
+ /**
+ * Makes a request to one or more of the given urls, using the configured load balancer.
+ *
+ * @param req The solr search request that should be sent through the load balancer
+ * @param urls The list of solr server urls to load balance across
+ * @return The response from the request
+ */
+ public LBHttpSolrServer.Rsp makeLoadBalancedRequest(final QueryRequest req, List<String> urls)
+ throws SolrServerException, IOException {
+ return loadbalancer.request(new LBHttpSolrServer.Req(req, urls));
+ }
+
+ /**
+ * Creates a randomized list of urls for the given shard.
+ *
+ * @param shard the urls for the shard (minus "http://"), separated by '|'
+ * @return A list of valid urls (including protocol) that are replicas for the shard
+ */
+ public List<String> makeURLList(String shard) {
+ List<String> urls = StrUtils.splitSmart(shard, "|", true);
+
+ // convert shard to URL
+ for (int i=0; i<urls.size(); i++) {
+ urls.set(i, scheme + urls.get(i));
+ }
+
+ //
+ // Shuffle the list instead of use round-robin by default.
+ // This prevents accidental synchronization where multiple shards could get in sync
+ // and query the same replica at the same time.
+ //
+ if (urls.size() > 1)
+ Collections.shuffle(urls, r);
+
+ return urls;
+ }
+
+ /**
+ * Creates a new completion service for use by a single set of distributed requests.
+ */
+ public CompletionService newCompletionService() {
+ return new ExecutorCompletionService<ShardResponse>(commExecutor);
+ }
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java Wed Mar 20 16:27:13 2013
@@ -53,6 +53,9 @@ public class ShardRequest {
/** actual shards to send the request to, filled out by framework */
public String[] actualShards;
+ /** may be null */
+ public String nodeName;
+
// TODO: one could store a list of numbers to correlate where returned docs
// go in the top-level response rather than looking up by id...
// this would work well if we ever transitioned to using internal ids and
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java Wed Mar 20 16:27:13 2013
@@ -22,6 +22,7 @@ import org.apache.solr.common.SolrExcept
public final class ShardResponse {
private ShardRequest req;
private String shard;
+ private String nodeName;
private String shardAddress; // the specific shard that this response was received from
private int rspCode;
private Throwable exception;
@@ -56,6 +57,11 @@ public final class ShardResponse {
return shard;
}
+ public String getNodeName()
+ {
+ return nodeName;
+ }
+
public void setShardRequest(ShardRequest rsp)
{
this.req = rsp;
@@ -80,9 +86,15 @@ public final class ShardResponse {
{
this.rspCode = rspCode;
}
+
+ void setNodeName(String nodeName)
+ {
+ this.nodeName = nodeName;
+ }
/** What was the shard address that returned this response. Example: "http://localhost:8983/solr" */
public String getShardAddress() { return this.shardAddress; }
void setShardAddress(String addr) { this.shardAddress = addr; }
+
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java Wed Mar 20 16:27:13 2013
@@ -22,8 +22,8 @@ import java.io.StringReader;
import java.util.*;
import org.apache.commons.io.IOUtils;
-import org.apache.noggit.JSONParser;
-import org.apache.noggit.ObjectBuilder;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java Wed Mar 20 16:27:13 2013
@@ -438,6 +438,7 @@ public class XMLLoader extends ContentSt
}
boost = 1.0f;
update = null;
+ isNull = false;
String attrVal = "";
for (int i = 0; i < parser.getAttributeCount(); i++) {
attrName = parser.getAttributeLocalName(i);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java Wed Mar 20 16:27:13 2013
@@ -26,8 +26,7 @@ import java.util.Set;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.postingshighlight.Passage;
import org.apache.lucene.search.postingshighlight.PassageFormatter;
import org.apache.lucene.search.postingshighlight.PassageScorer;
import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
@@ -55,7 +54,11 @@ import org.apache.solr.util.plugin.Plugi
* preTag="&lt;em&gt;"
* postTag="&lt;/em&gt;"
* ellipsis="... "
- * maxLength=10000/>
+ * k1="1.2"
+ * b="0.75"
+ * pivot="87"
+ * maxLength=10000
+ * summarizeEmpty=true/>
* </searchComponent>
* </pre>
* <p>
@@ -78,7 +81,23 @@ public class PostingsSolrHighlighter ext
public void init(PluginInfo info) {
Map<String,String> attributes = info.attributes;
BreakIterator breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT);
- PassageScorer scorer = new PassageScorer();
+
+ // scorer parameters: k1/b/pivot
+ String k1 = attributes.get("k1");
+ if (k1 == null) {
+ k1 = "1.2";
+ }
+
+ String b = attributes.get("b");
+ if (b == null) {
+ b = "0.75";
+ }
+
+ String pivot = attributes.get("pivot");
+ if (pivot == null) {
+ pivot = "87";
+ }
+ PassageScorer scorer = new PassageScorer(Float.parseFloat(k1), Float.parseFloat(b), Float.parseFloat(pivot));
// formatter parameters: preTag/postTag/ellipsis
String preTag = attributes.get("preTag");
@@ -94,13 +113,30 @@ public class PostingsSolrHighlighter ext
ellipsis = "... ";
}
PassageFormatter formatter = new PassageFormatter(preTag, postTag, ellipsis);
-
+
+ String summarizeEmpty = attributes.get("summarizeEmpty");
+ final boolean summarizeEmptyBoolean;
+ if (summarizeEmpty == null) {
+ summarizeEmptyBoolean = true;
+ } else {
+ summarizeEmptyBoolean = Boolean.parseBoolean(summarizeEmpty);
+ }
+
// maximum content size to process
int maxLength = PostingsHighlighter.DEFAULT_MAX_LENGTH;
if (attributes.containsKey("maxLength")) {
maxLength = Integer.parseInt(attributes.get("maxLength"));
}
- highlighter = new PostingsHighlighter(maxLength, breakIterator, scorer, formatter);
+ highlighter = new PostingsHighlighter(maxLength, breakIterator, scorer, formatter) {
+ @Override
+ protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
+ if (summarizeEmptyBoolean) {
+ return super.getEmptyHighlight(fieldName, bi, maxPassages);
+ } else {
+ return new Passage[0];
+ }
+ }
+ };
}
@Override
@@ -110,16 +146,16 @@ public class PostingsSolrHighlighter ext
// if highlighting isnt enabled, then why call doHighlighting?
if (isHighlightingEnabled(params)) {
SolrIndexSearcher searcher = req.getSearcher();
- TopDocs topDocs = toTopDocs(docs);
+ int[] docIDs = toDocIDs(docs);
// fetch the unique keys
- String[] keys = getUniqueKeys(searcher, topDocs);
+ String[] keys = getUniqueKeys(searcher, docIDs);
// query-time parameters
String[] fieldNames = getHighlightFields(query, req, defaultFields);
int numSnippets = params.getInt(HighlightParams.SNIPPETS, 1);
- Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, topDocs, numSnippets);
+ Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, docIDs, numSnippets);
return encodeSnippets(keys, fieldNames, snippets);
} else {
return null;
@@ -152,38 +188,38 @@ public class PostingsSolrHighlighter ext
return list;
}
- /** Converts solr's DocList to a lucene TopDocs */
- protected TopDocs toTopDocs(DocList docs) {
- ScoreDoc[] scoreDocs = new ScoreDoc[docs.size()];
+ /** Converts solr's DocList to the int[] docIDs */
+ protected int[] toDocIDs(DocList docs) {
+ int[] docIDs = new int[docs.size()];
DocIterator iterator = docs.iterator();
- for (int i = 0; i < scoreDocs.length; i++) {
+ for (int i = 0; i < docIDs.length; i++) {
if (!iterator.hasNext()) {
throw new AssertionError();
}
- scoreDocs[i] = new ScoreDoc(iterator.nextDoc(), Float.NaN);
+ docIDs[i] = iterator.nextDoc();
}
if (iterator.hasNext()) {
throw new AssertionError();
}
- return new TopDocs(docs.matches(), scoreDocs, Float.NaN);
+ return docIDs;
}
/** Retrieves the unique keys for the topdocs to key the results */
- protected String[] getUniqueKeys(SolrIndexSearcher searcher, TopDocs topDocs) throws IOException {
+ protected String[] getUniqueKeys(SolrIndexSearcher searcher, int[] docIDs) throws IOException {
IndexSchema schema = searcher.getSchema();
SchemaField keyField = schema.getUniqueKeyField();
if (keyField != null) {
Set<String> selector = Collections.singleton(keyField.getName());
- String uniqueKeys[] = new String[topDocs.scoreDocs.length];
- for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- int docid = topDocs.scoreDocs[i].doc;
+ String uniqueKeys[] = new String[docIDs.length];
+ for (int i = 0; i < docIDs.length; i++) {
+ int docid = docIDs[i];
StoredDocument doc = searcher.doc(docid, selector);
String id = schema.printableUniqueKey(doc);
uniqueKeys[i] = id;
}
return uniqueKeys;
} else {
- return new String[topDocs.scoreDocs.length];
+ return new String[docIDs.length];
}
}
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java Wed Mar 20 16:27:13 2013
@@ -922,7 +922,8 @@ public abstract class SolrQueryParserBas
}
protected String analyzeIfMultitermTermText(String field, String part, FieldType fieldType) {
- if (part == null) return part;
+
+ if (part == null || ! (fieldType instanceof TextField) || ((TextField)fieldType).getMultiTermAnalyzer() == null) return part;
SchemaField sf = schema.getFieldOrNull((field));
if (sf == null || ! (fieldType instanceof TextField)) return part;
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java Wed Mar 20 16:27:13 2013
@@ -23,7 +23,6 @@ import java.util.concurrent.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedDocValuesTermsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
@@ -155,7 +154,7 @@ class PerSegmentSingleValuedFaceting {
seg.pos = seg.startTermIndex;
}
if (seg.pos < seg.endTermIndex) {
- seg.tenum = new SortedDocValuesTermsEnum(seg.si);
+ seg.tenum = seg.si.termsEnum();
seg.tenum.seekExact(seg.pos);
seg.tempBR = seg.tenum.term();
queue.add(seg);