You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [28/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/...
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java Fri Jan 18 18:30:54 2013
@@ -22,6 +22,7 @@ import org.apache.lucene.store.LockFacto
import org.apache.lucene.store.MMapDirectory;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,6 +47,7 @@ public class MMapDirectoryFactory extend
@Override
public void init(NamedList args) {
+ super.init(args);
SolrParams params = SolrParams.toSolrParams( args );
maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_BUFF);
if (maxChunk <= 0){
@@ -55,7 +57,7 @@ public class MMapDirectoryFactory extend
}
@Override
- protected Directory create(String path) throws IOException {
+ protected Directory create(String path, DirContext dirContext) throws IOException {
MMapDirectory mapDirectory = new MMapDirectory(new File(path), null, maxChunk);
try {
mapDirectory.setUseUnmap(unmapHack);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NIOFSDirectoryFactory.java Fri Jan 18 18:30:54 2013
@@ -18,6 +18,7 @@ package org.apache.solr.core;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import java.io.File;
import java.io.IOException;
@@ -30,7 +31,7 @@ import java.io.IOException;
public class NIOFSDirectoryFactory extends StandardDirectoryFactory {
@Override
- protected Directory create(String path) throws IOException {
+ protected Directory create(String path, DirContext dirContext) throws IOException {
return new NIOFSDirectory(new File(path));
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java Fri Jan 18 18:30:54 2013
@@ -23,15 +23,34 @@ import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.NRTCachingDirectory;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory.DirContext;
/**
* Factory to instantiate {@link org.apache.lucene.store.NRTCachingDirectory}
*/
public class NRTCachingDirectoryFactory extends StandardDirectoryFactory {
+ private double maxMergeSizeMB;
+ private double maxCachedMB;
@Override
- protected Directory create(String path) throws IOException {
- return new NRTCachingDirectory(FSDirectory.open(new File(path)), 4, 48);
+ public void init(NamedList args) {
+ super.init(args);
+ SolrParams params = SolrParams.toSolrParams(args);
+ maxMergeSizeMB = params.getDouble("maxMergeSizeMB", 4);
+ if (maxMergeSizeMB <= 0){
+ throw new IllegalArgumentException("maxMergeSizeMB must be greater than 0");
+ }
+ maxCachedMB = params.getDouble("maxCachedMB", 48);
+ if (maxCachedMB <= 0){
+ throw new IllegalArgumentException("maxCachedMB must be greater than 0");
+ }
+ }
+
+ @Override
+ protected Directory create(String path, DirContext dirContext) throws IOException {
+ return new NRTCachingDirectory(FSDirectory.open(new File(path)), maxMergeSizeMB, maxCachedMB);
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java Fri Jan 18 18:30:54 2013
@@ -74,7 +74,7 @@ public class PluginInfo {
if (type != null) sb.append("type = " + type + ",");
if (name != null) sb.append("name = " + name + ",");
if (className != null) sb.append("class = " + className + ",");
- if (initArgs.size() > 0) sb.append("args = " + initArgs);
+ if (initArgs != null && initArgs.size() > 0) sb.append("args = " + initArgs);
sb.append("}");
return sb.toString();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RAMDirectoryFactory.java Fri Jan 18 18:30:54 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.store.RAMDirect
public class RAMDirectoryFactory extends EphemeralDirectoryFactory {
@Override
- protected Directory create(String path) throws IOException {
+ protected Directory create(String path, DirContext dirContext) throws IOException {
return new RAMDirectory();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java Fri Jan 18 18:30:54 2013
@@ -225,6 +225,7 @@ public final class RequestHandlers {
/**
* In normal use, this function will not be called
*/
+ @Override
public void init(NamedList args) {
// do nothing
}
@@ -232,6 +233,7 @@ public final class RequestHandlers {
/**
* Wait for the first request before initializing the wrapped handler
*/
+ @Override
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
SolrRequestHandler handler = _handler;
if (handler == null) {
@@ -266,10 +268,12 @@ public final class RequestHandlers {
//////////////////////// SolrInfoMBeans methods //////////////////////
+ @Override
public String getName() {
return "Lazy["+_className+"]";
}
+ @Override
public String getDescription()
{
if( _handler == null ) {
@@ -278,6 +282,7 @@ public final class RequestHandlers {
return _handler.getDescription();
}
+ @Override
public String getVersion() {
if( _handler != null ) {
return _handler.getVersion();
@@ -285,6 +290,7 @@ public final class RequestHandlers {
return null;
}
+ @Override
public String getSource() {
String rev = "$URL$";
if( _handler != null ) {
@@ -293,6 +299,7 @@ public final class RequestHandlers {
return rev;
}
+ @Override
public URL[] getDocs() {
if( _handler == null ) {
return null;
@@ -300,11 +307,13 @@ public final class RequestHandlers {
return _handler.getDocs();
}
+ @Override
public Category getCategory()
{
return Category.QUERYHANDLER;
}
+ @Override
public NamedList getStatistics() {
if( _handler != null ) {
return _handler.getStatistics();
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SimpleFSDirectoryFactory.java Fri Jan 18 18:30:54 2013
@@ -18,6 +18,7 @@ package org.apache.solr.core;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import java.io.File;
import java.io.IOException;
@@ -30,7 +31,7 @@ import java.io.IOException;
public class SimpleFSDirectoryFactory extends StandardDirectoryFactory {
@Override
- protected Directory create(String path) throws IOException {
+ protected Directory create(String path, DirContext dirContext) throws IOException {
return new SimpleFSDirectory(new File(path));
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrCore.java Fri Jan 18 18:30:54 2013
@@ -17,6 +17,40 @@
package org.apache.solr.core;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.xml.parsers.ParserConfigurationException;
+
import org.apache.commons.io.IOUtils;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.DirectoryReader;
@@ -32,8 +66,11 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.CommonParams.EchoParamStyle;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.apache.solr.handler.SnapPuller;
import org.apache.solr.handler.admin.ShowFileRequestHandler;
import org.apache.solr.handler.component.DebugComponent;
import org.apache.solr.handler.component.FacetComponent;
@@ -67,16 +104,17 @@ import org.apache.solr.search.ValueSourc
import org.apache.solr.update.DefaultSolrCoreState;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.SolrCoreState;
+import org.apache.solr.update.SolrCoreState.IndexWriterCloser;
import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.update.UpdateHandler;
import org.apache.solr.update.VersionInfo;
-import org.apache.solr.update.SolrCoreState.IndexWriterCloser;
import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
import org.apache.solr.update.processor.LogUpdateProcessorFactory;
import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.PropertiesInputStream;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
@@ -85,41 +123,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
-import javax.xml.parsers.ParserConfigurationException;
-
-import java.io.EOFException;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Writer;
-import java.lang.reflect.Constructor;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.StringTokenizer;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-
/**
*
@@ -235,30 +238,11 @@ public final class SolrCore implements S
Properties p = new Properties();
Directory dir = null;
try {
- dir = getDirectoryFactory().get(getDataDir(), null);
- if (dir.fileExists("index.properties")){
- final IndexInput input = dir.openInput("index.properties", IOContext.DEFAULT);
+ dir = getDirectoryFactory().get(getDataDir(), DirContext.META_DATA, getSolrConfig().indexConfig.lockType);
+ if (dir.fileExists(SnapPuller.INDEX_PROPERTIES)){
+ final IndexInput input = dir.openInput(SnapPuller.INDEX_PROPERTIES, IOContext.DEFAULT);
- final InputStream is = new InputStream() {
-
- @Override
- public int read() throws IOException {
- byte next;
- try {
- next = input.readByte();
- } catch (EOFException e) {
- return -1;
- }
- return next;
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- input.close();
- }
- };
-
+ final InputStream is = new PropertiesInputStream(input);
try {
p.load(is);
@@ -268,7 +252,7 @@ public final class SolrCore implements S
}
} catch (Exception e) {
- log.error("Unable to load index.properties", e);
+ log.error("Unable to load " + SnapPuller.INDEX_PROPERTIES, e);
} finally {
IOUtils.closeQuietly(is);
}
@@ -276,11 +260,12 @@ public final class SolrCore implements S
} catch (IOException e) {
SolrException.log(log, "", e);
} finally {
-
- try {
- getDirectoryFactory().release(dir);
- } catch (IOException e) {
- SolrException.log(log, "", e);
+ if (dir != null) {
+ try {
+ getDirectoryFactory().release(dir);
+ } catch (IOException e) {
+ SolrException.log(log, "", e);
+ }
}
}
if (!result.equals(lastNewIndexDir)) {
@@ -300,6 +285,7 @@ public final class SolrCore implements S
return indexReaderFactory;
}
+ @Override
public String getName() {
return name;
}
@@ -469,7 +455,7 @@ public final class SolrCore implements S
if (indexExists && firstTime && !reload) {
- Directory dir = directoryFactory.get(indexDir,
+ Directory dir = directoryFactory.get(indexDir, DirContext.DEFAULT,
getSolrConfig().indexConfig.lockType);
try {
if (IndexWriter.isLocked(dir)) {
@@ -581,6 +567,9 @@ public final class SolrCore implements S
} else if (o instanceof NamedListInitializedPlugin) {
((NamedListInitializedPlugin) o).init(info.initArgs);
}
+ if(o instanceof SearchComponent) {
+ ((SearchComponent) o).setName(info.name);
+ }
return o;
}
@@ -748,6 +737,7 @@ public final class SolrCore implements S
// until after inform() has been called for all components.
// searchExecutor must be single-threaded for this to work
searcherExecutor.submit(new Callable() {
+ @Override
public Object call() throws Exception {
latch.await();
return null;
@@ -973,7 +963,7 @@ public final class SolrCore implements S
try {
infoRegistry.clear();
- } catch (Exception e) {
+ } catch (Throwable e) {
SolrException.log(log, e);
}
@@ -996,22 +986,11 @@ public final class SolrCore implements S
}
try {
- searcherExecutor.shutdown();
- if (!searcherExecutor.awaitTermination(60, TimeUnit.SECONDS)) {
- log.error("Timeout waiting for searchExecutor to terminate");
- }
- } catch (InterruptedException e) {
- searcherExecutor.shutdownNow();
- try {
- if (!searcherExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
- log.error("Timeout waiting for searchExecutor to terminate");
- }
- } catch (InterruptedException e2) {
- SolrException.log(log, e2);
- }
- } catch (Exception e) {
+ ExecutorUtil.shutdownAndAwaitTermination(searcherExecutor);
+ } catch (Throwable e) {
SolrException.log(log, e);
}
+
try {
// Since we waited for the searcherExecutor to shut down,
// there should be no more searchers warming in the background
@@ -1185,7 +1164,10 @@ public final class SolrCore implements S
if(!registry.containsKey(name)){
T searchComp = resourceLoader.newInstance(c.getName(), c);
if (searchComp instanceof NamedListInitializedPlugin){
- ((NamedListInitializedPlugin)searchComp).init( new NamedList() );
+ ((NamedListInitializedPlugin)searchComp).init( new NamedList<String>() );
+ }
+ if(searchComp instanceof SearchComponent) {
+ ((SearchComponent)searchComp).setName(name);
}
registry.put(name, searchComp);
if (searchComp instanceof SolrInfoMBean){
@@ -1589,6 +1571,7 @@ public final class SolrCore implements S
if (currSearcher != null) {
future = searcherExecutor.submit(
new Callable() {
+ @Override
public Object call() throws Exception {
try {
newSearcher.warm(currSearcher);
@@ -1604,6 +1587,7 @@ public final class SolrCore implements S
if (currSearcher==null && firstSearcherListeners.size() > 0) {
future = searcherExecutor.submit(
new Callable() {
+ @Override
public Object call() throws Exception {
try {
for (SolrEventListener listener : firstSearcherListeners) {
@@ -1621,6 +1605,7 @@ public final class SolrCore implements S
if (currSearcher!=null && newSearcherListeners.size() > 0) {
future = searcherExecutor.submit(
new Callable() {
+ @Override
public Object call() throws Exception {
try {
for (SolrEventListener listener : newSearcherListeners) {
@@ -1641,6 +1626,7 @@ public final class SolrCore implements S
if (!alreadyRegistered) {
future = searcherExecutor.submit(
new Callable() {
+ @Override
public Object call() throws Exception {
try {
// registerSearcher will decrement onDeckSearchers and
@@ -2184,31 +2170,38 @@ public final class SolrCore implements S
// SolrInfoMBean stuff: Statistics and Module Info
/////////////////////////////////////////////////////////////////////
+ @Override
public String getVersion() {
return SolrCore.version;
}
+ @Override
public String getDescription() {
return "SolrCore";
}
+ @Override
public Category getCategory() {
return Category.CORE;
}
+ @Override
public String getSource() {
return "$URL$";
}
+ @Override
public URL[] getDocs() {
return null;
}
+ @Override
public NamedList getStatistics() {
NamedList<Object> lst = new SimpleOrderedMap<Object>();
lst.add("coreName", name==null ? "(null)" : name);
lst.add("startTime", new Date(startTime));
lst.add("refCount", getOpenCount());
+ lst.add("indexDir", getIndexDir());
CoreDescriptor cd = getCoreDescriptor();
if (cd != null) {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java Fri Jan 18 18:30:54 2013
@@ -48,6 +48,7 @@ public class SolrDeletionPolicy implemen
private int maxCommitsToKeep = 1;
private int maxOptimizedCommitsToKeep = 0;
+ @Override
public void init(NamedList args) {
String keepOptimizedOnlyString = (String) args.get("keepOptimizedOnly");
String maxCommitsToKeepString = (String) args.get("maxCommitsToKeep");
@@ -109,6 +110,7 @@ public class SolrDeletionPolicy implemen
/**
* Internal use for Lucene... do not explicitly call.
*/
+ @Override
public void onInit(List commits) throws IOException {
log.info("SolrDeletionPolicy.onInit: commits:" + str(commits));
updateCommits((List<IndexCommit>) commits);
@@ -117,6 +119,7 @@ public class SolrDeletionPolicy implemen
/**
* Internal use for Lucene... do not explicitly call.
*/
+ @Override
public void onCommit(List commits) throws IOException {
log.info("SolrDeletionPolicy.onCommit: commits:" + str(commits));
updateCommits((List<IndexCommit>) commits);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Fri Jan 18 18:30:54 2013
@@ -159,6 +159,7 @@ public class SolrResourceLoader implemen
if (file.canRead()) {
this.classLoader = replaceClassLoader(classLoader, file.getParentFile(),
new FileFilter() {
+ @Override
public boolean accept(File pathname) {
return pathname.equals(file);
}
@@ -288,6 +289,7 @@ public class SolrResourceLoader implemen
* Override this method to customize loading resources.
*@return the stream for the named resource
*/
+ @Override
public InputStream openResource(String resource) throws IOException {
InputStream is=null;
try {
@@ -449,6 +451,7 @@ public class SolrResourceLoader implemen
static final String empty[] = new String[0];
+ @Override
public <T> T newInstance(String name, Class<T> expectedType) {
return newInstance(name, expectedType, empty);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java Fri Jan 18 18:30:54 2013
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
/**
* Directory provider which mimics original Solr
@@ -34,7 +35,7 @@ import org.apache.lucene.store.FSDirecto
public class StandardDirectoryFactory extends CachingDirectoryFactory {
@Override
- protected Directory create(String path) throws IOException {
+ protected Directory create(String path, DirContext dirContext) throws IOException {
return FSDirectory.open(new File(path));
}
@@ -43,23 +44,36 @@ public class StandardDirectoryFactory ex
return new File(path).getCanonicalPath();
}
+ public boolean isPersistent() {
+ return true;
+ }
+
@Override
public void remove(Directory dir) throws IOException {
CacheValue val = byDirectoryCache.get(dir);
if (val == null) {
- throw new NullPointerException("Unknown directory " + dir);
+ throw new IllegalArgumentException("Unknown directory " + dir);
}
File dirFile = new File(val.path);
FileUtils.deleteDirectory(dirFile);
}
+
+ @Override
+ public void remove(String path) throws IOException {
+ String fullPath = new File(path).getAbsolutePath();
+ File dirFile = new File(fullPath);
+ FileUtils.deleteDirectory(dirFile);
+ }
+
/**
* Override for more efficient moves.
*
* @throws IOException
* If there is a low-level I/O error.
*/
- public void move(Directory fromDir, Directory toDir, String fileName)
+ @Override
+ public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
throws IOException {
if (fromDir instanceof FSDirectory && toDir instanceof FSDirectory) {
File dir1 = ((FSDirectory) fromDir).getDirectory();
@@ -72,6 +86,7 @@ public class StandardDirectoryFactory ex
}
}
- super.move(fromDir, toDir, fileName);
+ super.move(fromDir, toDir, fileName, ioContext);
}
+
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java Fri Jan 18 18:30:54 2013
@@ -212,6 +212,7 @@ public abstract class AnalysisRequestHan
// sort the tokens by absoulte position
ArrayUtil.mergeSort(tokens, new Comparator<AttributeSource>() {
+ @Override
public int compare(AttributeSource a, AttributeSource b) {
return arrayCompare(
a.getAttribute(TokenTrackingAttribute.class).getPositions(),
@@ -255,6 +256,7 @@ public abstract class AnalysisRequestHan
}
token.reflectWith(new AttributeReflector() {
+ @Override
public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
// leave out position and bytes term
if (TermToBytesRefAttribute.class.isAssignableFrom(attClass))
@@ -365,17 +367,20 @@ public abstract class AnalysisRequestHan
private int position = 0;
private transient int[] cachedPositions = null;
+ @Override
public void freezeStage() {
this.basePositions = getPositions();
this.position = 0;
this.cachedPositions = null;
}
+ @Override
public void setActPosition(int pos) {
this.position = pos;
this.cachedPositions = null;
}
+ @Override
public int[] getPositions() {
if (cachedPositions == null) {
cachedPositions = ArrayUtils.add(basePositions, position);
@@ -383,6 +388,7 @@ public abstract class AnalysisRequestHan
return cachedPositions;
}
+ @Override
public void reset(int[] basePositions, int position) {
this.basePositions = basePositions;
this.position = position;
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java Fri Jan 18 18:30:54 2013
@@ -29,11 +29,9 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.*;
import org.apache.lucene.queries.mlt.MoreLikeThis;
import org.apache.solr.common.SolrException;
@@ -78,7 +76,7 @@ public class MoreLikeThisHandler extends
SolrParams params = req.getParams();
// Set field flags
- ReturnFields returnFields = new ReturnFields( req );
+ ReturnFields returnFields = new SolrReturnFields( req );
rsp.setReturnFields( returnFields );
int flags = 0;
if (returnFields.wantsScore()) {
@@ -108,7 +106,7 @@ public class MoreLikeThisHandler extends
}
}
}
- } catch (ParseException e) {
+ } catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
@@ -259,6 +257,7 @@ public class MoreLikeThisHandler extends
public float boost;
public static Comparator<InterestingTerm> BOOST_ORDER = new Comparator<InterestingTerm>() {
+ @Override
public int compare(InterestingTerm t1, InterestingTerm t2) {
float d = t1.boost - t2.boost;
if( d == 0 ) {
@@ -300,8 +299,10 @@ public class MoreLikeThisHandler extends
mlt.setAnalyzer( searcher.getSchema().getAnalyzer() );
// configurable params
+
mlt.setMinTermFreq( params.getInt(MoreLikeThisParams.MIN_TERM_FREQ, MoreLikeThis.DEFAULT_MIN_TERM_FREQ));
mlt.setMinDocFreq( params.getInt(MoreLikeThisParams.MIN_DOC_FREQ, MoreLikeThis.DEFAULT_MIN_DOC_FREQ));
+ mlt.setMaxDocFreq( params.getInt(MoreLikeThisParams.MAX_DOC_FREQ, MoreLikeThis.DEFAULT_MAX_DOC_FREQ));
mlt.setMinWordLen( params.getInt(MoreLikeThisParams.MIN_WORD_LEN, MoreLikeThis.DEFAULT_MIN_WORD_LENGTH));
mlt.setMaxWordLen( params.getInt(MoreLikeThisParams.MAX_WORD_LEN, MoreLikeThis.DEFAULT_MAX_WORD_LENGTH));
mlt.setMaxQueryTerms( params.getInt(MoreLikeThisParams.MAX_QUERY_TERMS, MoreLikeThis.DEFAULT_MAX_QUERY_TERMS));
@@ -400,6 +401,33 @@ public class MoreLikeThisHandler extends
return mlt;
}
+ public NamedList<BooleanQuery> getMoreLikeTheseQuery(DocList docs)
+ throws IOException {
+ IndexSchema schema = searcher.getSchema();
+ NamedList<BooleanQuery> result = new NamedList<BooleanQuery>();
+ DocIterator iterator = docs.iterator();
+ while (iterator.hasNext()) {
+ int id = iterator.nextDoc();
+ String uniqueId = schema.printableUniqueKey(reader.document(id));
+
+ BooleanQuery mltquery = (BooleanQuery) mlt.like(id);
+ if (mltquery.clauses().size() == 0) {
+ return result;
+ }
+ mltquery = (BooleanQuery) getBoostedQuery(mltquery);
+
+ // exclude current document from results
+ BooleanQuery mltQuery = new BooleanQuery();
+ mltQuery.add(mltquery, BooleanClause.Occur.MUST);
+
+ mltQuery.add(
+ new TermQuery(new Term(uniqueKeyField.getName(), uniqueId)), BooleanClause.Occur.MUST_NOT);
+ result.add(uniqueId, mltQuery);
+ }
+
+ return result;
+ }
+
private void fillInterestingTermsFromMLTQuery( Query query, List<InterestingTerm> terms )
{
List clauses = ((BooleanQuery)query).clauses();
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java Fri Jan 18 18:30:54 2013
@@ -134,12 +134,14 @@ public class PingRequestHandler extends
private String healthFileName = null;
private File healthcheck = null;
+ @Override
public void init(NamedList args) {
super.init(args);
Object tmp = args.get(HEALTHCHECK_FILE_PARAM);
healthFileName = (null == tmp ? null : tmp.toString());
}
+ @Override
public void inform( SolrCore core ) {
if (null != healthFileName) {
healthcheck = new File(healthFileName);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Fri Jan 18 18:30:54 2013
@@ -60,6 +60,7 @@ import org.apache.solr.core.IndexDeletio
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrDeletionPolicy;
import org.apache.solr.core.SolrEventListener;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.BinaryQueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
@@ -361,7 +362,7 @@ public class ReplicationHandler extends
// use a set to workaround possible Lucene bug which returns same file
// name multiple times
Collection<String> files = new HashSet<String>(commit.getFileNames());
- dir = core.getDirectoryFactory().get(core.getNewIndexDir(), null);
+ dir = core.getDirectoryFactory().get(core.getNewIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
try {
for (String fileName : files) {
@@ -467,7 +468,7 @@ public class ReplicationHandler extends
Directory dir;
long size = 0;
try {
- dir = core.getDirectoryFactory().get(core.getIndexDir(), null);
+ dir = core.getDirectoryFactory().get(core.getNewIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
try {
size = DirectoryFactory.sizeOfDirectory(dir);
} finally {
@@ -777,6 +778,7 @@ public class ReplicationHandler extends
// }
// }
+ @Override
@SuppressWarnings("unchecked")
public void inform(SolrCore core) {
this.core = core;
@@ -927,19 +929,23 @@ public class ReplicationHandler extends
*/
private void registerFileStreamResponseWriter() {
core.registerResponseWriter(FILE_STREAM, new BinaryQueryResponseWriter() {
+ @Override
public void write(OutputStream out, SolrQueryRequest request, SolrQueryResponse resp) throws IOException {
DirectoryFileStream stream = (DirectoryFileStream) resp.getValues().get(FILE_STREAM);
stream.write(out);
}
+ @Override
public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) {
throw new RuntimeException("This is a binary writer , Cannot write to a characterstream");
}
+ @Override
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
return "application/octet-stream";
}
+ @Override
public void init(NamedList args) { /*no op*/ }
});
@@ -955,11 +961,13 @@ public class ReplicationHandler extends
*/
private SolrEventListener getEventListener(final boolean snapshoot, final boolean getCommit) {
return new SolrEventListener() {
+ @Override
public void init(NamedList args) {/*no op*/ }
/**
* This refreshes the latest replicateable index commit and optionally can create Snapshots as well
*/
+ @Override
public void postCommit() {
IndexCommit currentCommitPoint = core.getDeletionPolicy().getLatestCommit();
@@ -992,6 +1000,7 @@ public class ReplicationHandler extends
}
}
+ @Override
public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) { /*no op*/}
@Override
@@ -1062,7 +1071,7 @@ public class ReplicationHandler extends
while (true) {
offset = offset == -1 ? 0 : offset;
int read = (int) Math.min(buf.length, filelen - offset);
- in.readBytes(buf, offset == -1 ? 0 : (int) offset, read);
+ in.readBytes(buf, 0, read);
fos.writeInt((int) read);
if (useChecksum) {
@@ -1082,6 +1091,8 @@ public class ReplicationHandler extends
fos.close();
break;
}
+ offset += read;
+ in.seek(offset);
}
} catch (IOException e) {
LOG.warn("Exception while writing response for params: " + params, e);
@@ -1108,6 +1119,7 @@ public class ReplicationHandler extends
super(solrParams);
}
+ @Override
public void write(OutputStream out) throws IOException {
String fileName = params.get(FILE);
String cfileName = params.get(CONF_FILE_SHORT);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java Fri Jan 18 18:30:54 2013
@@ -17,12 +17,6 @@
package org.apache.solr.handler;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Timer;
-import com.yammer.metrics.core.TimerContext;
-import com.yammer.metrics.stats.Snapshot;
-import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -32,7 +26,11 @@ import org.apache.solr.core.SolrInfoMBea
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.SyntaxError;
import org.apache.solr.util.SolrPluginUtils;
+import org.apache.solr.util.stats.Snapshot;
+import org.apache.solr.util.stats.Timer;
+import org.apache.solr.util.stats.TimerContext;
import java.net.URL;
import java.util.concurrent.atomic.AtomicLong;
@@ -49,20 +47,11 @@ public abstract class RequestHandlerBase
protected boolean httpCaching = true;
// Statistics
- private static final AtomicLong handlerNumber = new AtomicLong();
- private final Counter numRequests;
- private final Counter numErrors;
- private final Counter numTimeouts;
- private final Timer requestTimes;
- long handlerStart = System.currentTimeMillis();
-
- public RequestHandlerBase() {
- String scope = new String("metrics-scope-" + handlerNumber.getAndIncrement());
- numRequests = Metrics.newCounter(RequestHandlerBase.class, "numRequests", scope);
- numErrors = Metrics.newCounter(RequestHandlerBase.class, "numErrors", scope);
- numTimeouts = Metrics.newCounter(RequestHandlerBase.class, "numTimeouts", scope);
- requestTimes = Metrics.newTimer(RequestHandlerBase.class, "requestTimes", scope);
- }
+ private final AtomicLong numRequests = new AtomicLong();
+ private final AtomicLong numErrors = new AtomicLong();
+ private final AtomicLong numTimeouts = new AtomicLong();
+ private final Timer requestTimes = new Timer();
+ private final long handlerStart = System.currentTimeMillis();
/**
* Initializes the {@link org.apache.solr.request.SolrRequestHandler} by creating three {@link org.apache.solr.common.params.SolrParams} named.
@@ -103,6 +92,7 @@ public abstract class RequestHandlerBase
*
* See also the example solrconfig.xml located in the Solr codebase (example/solr/conf).
*/
+ @Override
public void init(NamedList args) {
initArgs = args;
@@ -135,8 +125,9 @@ public abstract class RequestHandlerBase
public abstract void handleRequestBody( SolrQueryRequest req, SolrQueryResponse rsp ) throws Exception;
+ @Override
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
- numRequests.inc();
+ numRequests.incrementAndGet();
TimerContext timer = requestTimes.time();
try {
SolrPluginUtils.setDefaults(req,defaults,appends,invariants);
@@ -148,7 +139,7 @@ public abstract class RequestHandlerBase
Object partialResults = header.get("partialResults");
boolean timedOut = partialResults == null ? false : (Boolean)partialResults;
if( timedOut ) {
- numTimeouts.inc();
+ numTimeouts.incrementAndGet();
rsp.setHttpCaching(false);
}
}
@@ -163,13 +154,13 @@ public abstract class RequestHandlerBase
}
} else {
SolrException.log(SolrCore.log,e);
- if (e instanceof ParseException) {
+ if (e instanceof SyntaxError) {
e = new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
rsp.setException(e);
- numErrors.inc();
+ numErrors.incrementAndGet();
}
finally {
timer.stop();
@@ -178,37 +169,44 @@ public abstract class RequestHandlerBase
//////////////////////// SolrInfoMBeans methods //////////////////////
+ @Override
public String getName() {
return this.getClass().getName();
}
+ @Override
public abstract String getDescription();
+ @Override
public abstract String getSource();
+ @Override
public String getVersion() {
return getClass().getPackage().getSpecificationVersion();
}
+ @Override
public Category getCategory() {
return Category.QUERYHANDLER;
}
+ @Override
public URL[] getDocs() {
return null; // this can be overridden, but not required
}
+ @Override
public NamedList<Object> getStatistics() {
NamedList<Object> lst = new SimpleOrderedMap<Object>();
- lst.add("handlerStart",handlerStart);
- lst.add("requests", numRequests.count());
- lst.add("errors", numErrors.count());
- lst.add("timeouts", numTimeouts.count());
- lst.add("totalTime",requestTimes.sum());
- lst.add("avgRequestsPerSecond", requestTimes.meanRate());
- lst.add("5minRateReqsPerSecond", requestTimes.fiveMinuteRate());
- lst.add("15minRateReqsPerSecond", requestTimes.fifteenMinuteRate());
- lst.add("avgTimePerRequest", requestTimes.mean());
Snapshot snapshot = requestTimes.getSnapshot();
+ lst.add("handlerStart",handlerStart);
+ lst.add("requests", numRequests.longValue());
+ lst.add("errors", numErrors.longValue());
+ lst.add("timeouts", numTimeouts.longValue());
+ lst.add("totalTime", requestTimes.getSum());
+ lst.add("avgRequestsPerSecond", requestTimes.getMeanRate());
+ lst.add("5minRateReqsPerSecond", requestTimes.getFiveMinuteRate());
+ lst.add("15minRateReqsPerSecond", requestTimes.getFifteenMinuteRate());
+ lst.add("avgTimePerRequest", requestTimes.getMean());
lst.add("medianRequestTime", snapshot.getMedian());
lst.add("75thPcRequestTime", snapshot.get75thPercentile());
lst.add("95thPcRequestTime", snapshot.get95thPercentile());
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Fri Jan 18 18:30:54 2013
@@ -36,10 +36,7 @@ import static org.apache.solr.handler.Re
import static org.apache.solr.handler.ReplicationHandler.OFFSET;
import static org.apache.solr.handler.ReplicationHandler.SIZE;
-import java.io.EOFException;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -74,7 +71,6 @@ import org.apache.http.client.HttpClient
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.solr.client.solrj.SolrServer;
@@ -86,9 +82,11 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CachingDirectoryFactory.CloseListener;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
@@ -99,6 +97,8 @@ import org.apache.solr.search.SolrIndexS
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.FileUtils;
+import org.apache.solr.util.PropertiesInputStream;
+import org.apache.solr.util.PropertiesOutputStream;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,6 +111,8 @@ import org.slf4j.LoggerFactory;
* @since solr 1.4
*/
public class SnapPuller {
+ public static final String INDEX_PROPERTIES = "index.properties";
+
private static final Logger LOG = LoggerFactory.getLogger(SnapPuller.class.getName());
private final String masterUrl;
@@ -211,6 +213,7 @@ public class SnapPuller {
private void startExecutorService() {
Runnable task = new Runnable() {
+ @Override
public void run() {
if (pollDisabled.get()) {
LOG.info("Poll disabled");
@@ -296,7 +299,9 @@ public class SnapPuller {
successfulInstall = false;
replicationStartTime = System.currentTimeMillis();
Directory tmpIndexDir = null;
+ String tmpIndex = null;
Directory indexDir = null;
+ String indexDirPath = null;
boolean deleteTmpIdxDir = true;
try {
//get the current 'replicateable' index version in the master
@@ -365,15 +370,18 @@ public class SnapPuller {
filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
// if the generateion of master is older than that of the slave , it means they are not compatible to be copied
// then a new index direcory to be created and all the files need to be copied
- boolean isFullCopyNeeded = IndexDeletionPolicyWrapper.getCommitTimestamp(commit) >= latestVersion || forceReplication;
-
+ boolean isFullCopyNeeded = IndexDeletionPolicyWrapper
+ .getCommitTimestamp(commit) >= latestVersion
+ || commit.getGeneration() >= latestGeneration || forceReplication;
+
String tmpIdxDirName = "index." + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());
- String tmpIndex = createTempindexDir(core, tmpIdxDirName);
+ tmpIndex = createTempindexDir(core, tmpIdxDirName);
- tmpIndexDir = core.getDirectoryFactory().get(tmpIndex, null);
+ tmpIndexDir = core.getDirectoryFactory().get(tmpIndex, DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
// make sure it's the newest known index dir...
- indexDir = core.getDirectoryFactory().get(core.getNewIndexDir(), null);
+ indexDirPath = core.getNewIndexDir();
+ indexDir = core.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
Directory oldDirectory = null;
try {
@@ -422,12 +430,15 @@ public class SnapPuller {
if (isFullCopyNeeded) {
// we have to do this before commit
final Directory freezeIndexDir = indexDir;
+ final String freezeIndexDirPath = indexDirPath;
core.getDirectoryFactory().addCloseListener(oldDirectory, new CloseListener(){
@Override
public void preClose() {
LOG.info("removing old index files " + freezeIndexDir);
- DirectoryFactory.empty(freezeIndexDir);
+ if (core.getDirectoryFactory().exists(freezeIndexDirPath)) {
+ DirectoryFactory.empty(freezeIndexDir);
+ }
}
@Override
@@ -466,7 +477,9 @@ public class SnapPuller {
} finally {
if (deleteTmpIdxDir) {
LOG.info("removing temporary index download directory files " + tmpIndexDir);
- DirectoryFactory.empty(tmpIndexDir);
+ if (tmpIndex != null && core.getDirectoryFactory().exists(tmpIndex)) {
+ DirectoryFactory.empty(tmpIndexDir);
+ }
}
}
} finally {
@@ -519,9 +532,9 @@ public class SnapPuller {
/**
* Helper method to record the last replication's details so that we can show them on the statistics page across
* restarts.
+ * @throws IOException on IO error
*/
- private void logReplicationTimeAndConfFiles(Collection<Map<String, Object>> modifiedConfFiles, boolean successfulInstall) {
- FileOutputStream outFile = null;
+ private void logReplicationTimeAndConfFiles(Collection<Map<String, Object>> modifiedConfFiles, boolean successfulInstall) throws IOException {
List<String> confFiles = new ArrayList<String>();
if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty())
for (Map<String, Object> map1 : modifiedConfFiles)
@@ -530,7 +543,10 @@ public class SnapPuller {
Properties props = replicationHandler.loadReplicationProperties();
long replicationTime = System.currentTimeMillis();
long replicationTimeTaken = (replicationTime - getReplicationStartTime()) / 1000;
+ Directory dir = null;
try {
+ dir = solrCore.getDirectoryFactory().get(solrCore.getDataDir(), DirContext.META_DATA, solrCore.getSolrConfig().indexConfig.lockType);
+
int indexCount = 1, confFilesCount = 1;
if (props.containsKey(TIMES_INDEX_REPLICATED)) {
indexCount = Integer.valueOf(props.getProperty(TIMES_INDEX_REPLICATED)) + 1;
@@ -560,15 +576,21 @@ public class SnapPuller {
sb = readToStringBuffer(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST));
props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString());
}
- File f = new File(solrCore.getDataDir(), REPLICATION_PROPERTIES);
- outFile = new FileOutputStream(f);
- props.store(outFile, "Replication details");
- outFile.close();
+
+ final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ OutputStream outFile = new PropertiesOutputStream(out);
+ try {
+ props.store(outFile, "Replication details");
+ dir.sync(Collections.singleton(REPLICATION_PROPERTIES));
+ } finally {
+ IOUtils.closeQuietly(outFile);
+ }
} catch (Exception e) {
LOG.warn("Exception while updating statistics", e);
- }
- finally {
- IOUtils.closeQuietly(outFile);
+ } finally {
+ if (dir != null) {
+ solrCore.getDirectoryFactory().release(dir);
+ }
}
}
@@ -706,7 +728,7 @@ public class SnapPuller {
String indexDir = solrCore.getIndexDir();
// it's okay to use null for lock factory since we know this dir will exist
- Directory dir = solrCore.getDirectoryFactory().get(indexDir, null);
+ Directory dir = solrCore.getDirectoryFactory().get(indexDir, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
try {
for (Map<String,Object> file : filesToDownload) {
if (!dir.fileExists((String) file.get(NAME)) || downloadCompleteIndex) {
@@ -758,7 +780,7 @@ public class SnapPuller {
return false;
}
try {
- solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname);
+ solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname, DirectoryFactory.IOCONTEXT_NO_CACHE);
success = true;
} catch (IOException e) {
SolrException.log(LOG, "Could not move file", e);
@@ -829,68 +851,38 @@ public class SnapPuller {
Properties p = new Properties();
Directory dir = null;
try {
- dir = solrCore.getDirectoryFactory().get(solrCore.getDataDir(), null);
- if (dir.fileExists("index.properties")){
- final IndexInput input = dir.openInput("index.properties", IOContext.DEFAULT);
+ dir = solrCore.getDirectoryFactory().get(solrCore.getDataDir(), DirContext.META_DATA, solrCore.getSolrConfig().indexConfig.lockType);
+ if (dir.fileExists(SnapPuller.INDEX_PROPERTIES)){
+ final IndexInput input = dir.openInput(SnapPuller.INDEX_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
- final InputStream is = new InputStream() {
-
- @Override
- public int read() throws IOException {
- byte next;
- try {
- next = input.readByte();
- } catch (EOFException e) {
- return -1;
- }
- return next;
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- input.close();
- }
- };
-
+ final InputStream is = new PropertiesInputStream(input);
try {
p.load(is);
} catch (Exception e) {
- LOG.error("Unable to load index.properties", e);
+ LOG.error("Unable to load " + SnapPuller.INDEX_PROPERTIES, e);
} finally {
IOUtils.closeQuietly(is);
}
}
try {
- dir.deleteFile("index.properties");
+ dir.deleteFile(SnapPuller.INDEX_PROPERTIES);
} catch (IOException e) {
// no problem
}
- final IndexOutput out = dir.createOutput("index.properties", IOContext.DEFAULT);
+ final IndexOutput out = dir.createOutput(SnapPuller.INDEX_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
p.put("index", tmpIdxDirName);
OutputStream os = null;
try {
- os = new OutputStream() {
-
- @Override
- public void write(int b) throws IOException {
- out.writeByte((byte) b);
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- out.close();
- }
- };
- p.store(os, "index properties");
+ os = new PropertiesOutputStream(out);
+ p.store(os, SnapPuller.INDEX_PROPERTIES);
+ dir.sync(Collections.singleton(INDEX_PROPERTIES));
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Unable to write index.properties", e);
+ "Unable to write " + SnapPuller.INDEX_PROPERTIES, e);
} finally {
IOUtils.closeQuietly(os);
}
- return true;
+ return true;
} catch (IOException e1) {
throw new RuntimeException(e1);
@@ -1088,7 +1080,7 @@ public class SnapPuller {
indexGen = latestGen;
- outStream = copy2Dir.createOutput(saveAs, IOContext.DEFAULT);
+ outStream = copy2Dir.createOutput(saveAs, DirectoryFactory.IOCONTEXT_NO_CACHE);
if (includeChecksum)
checksum = new Adler32();
@@ -1118,6 +1110,7 @@ public class SnapPuller {
cleanup();
//if cleanup suceeds . The file is downloaded fully. do an fsync
fsyncService.submit(new Runnable(){
+ @Override
public void run() {
try {
copy2Dir.sync(Collections.singleton(saveAs));
@@ -1379,6 +1372,7 @@ public class SnapPuller {
cleanup();
//if cleanup suceeds . The file is downloaded fully. do an fsync
fsyncService.submit(new Runnable(){
+ @Override
public void run() {
try {
FileUtils.sync(file);
@@ -1593,7 +1587,22 @@ public class SnapPuller {
}
public void destroy() {
- if (executorService != null) executorService.shutdown();
+ try {
+ if (executorService != null) executorService.shutdown();
+ } catch (Throwable e) {
+ SolrException.log(LOG, e);
+ }
+ try {
+ abortPull();
+ } catch (Throwable e) {
+ SolrException.log(LOG, e);
+ }
+ try {
+ if (executorService != null) ExecutorUtil
+ .shutdownNowAndAwaitTermination(executorService);
+ } catch (Throwable e) {
+ SolrException.log(LOG, e);
+ }
}
String getMasterUrl() {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapShooter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapShooter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/SnapShooter.java Fri Jan 18 18:30:54 2013
@@ -31,16 +31,18 @@ import java.util.regex.Pattern;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* <p/> Provides functionality equivalent to the snapshooter script </p>
+ * This is no longer used in standard replication.
*
*
* @since solr 1.4
@@ -79,7 +81,7 @@ public class SnapShooter {
}
void createSnapshot(final IndexCommit indexCommit, int numberToKeep, ReplicationHandler replicationHandler) {
-
+ LOG.info("Creating backup snapshot...");
NamedList<Object> details = new NamedList<Object>();
details.add("startTime", new Date().toString());
File snapShotDir = null;
@@ -101,7 +103,7 @@ public class SnapShooter {
Collection<String> files = indexCommit.getFileNames();
FileCopier fileCopier = new FileCopier();
- Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), null);
+ Directory dir = solrCore.getDirectoryFactory().get(solrCore.getNewIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
try {
fileCopier.copyFiles(dir, files, snapShotDir);
} finally {
@@ -163,6 +165,7 @@ public class SnapShooter {
}
}
}
+ @Override
public int compareTo(OldBackupDirectory that) {
return that.timestamp.compareTo(this.timestamp);
}
@@ -200,7 +203,7 @@ public class SnapShooter {
throw new IOException(message);
}
- sourceDir.copy(destDir, indexFile, indexFile, IOContext.DEFAULT);
+ sourceDir.copy(destDir, indexFile, indexFile, DirectoryFactory.IOCONTEXT_NO_CACHE);
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java Fri Jan 18 18:30:54 2013
@@ -51,10 +51,12 @@ public class AdminHandlers implements So
/**
* Save the args and pass them to each standard handler
*/
+ @Override
public void init(NamedList args) {
this.initArgs = args;
}
+ @Override
public void inform(SolrCore core)
{
String path = null;
@@ -101,6 +103,7 @@ public class AdminHandlers implements So
}
+ @Override
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"The AdminHandler should never be called directly" );
@@ -108,30 +111,37 @@ public class AdminHandlers implements So
//////////////////////// SolrInfoMBeans methods //////////////////////
+ @Override
public String getDescription() {
return "Register Standard Admin Handlers";
}
+ @Override
public String getVersion() {
return getClass().getPackage().getSpecificationVersion();
}
+ @Override
public String getSource() {
return "$URL$";
}
+ @Override
public Category getCategory() {
return Category.QUERYHANDLER;
}
+ @Override
public URL[] getDocs() {
return null;
}
+ @Override
public String getName() {
return this.getClass().getName();
}
+ @Override
public NamedList getStatistics() {
return null;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Fri Jan 18 18:30:54 2013
@@ -18,6 +18,8 @@ package org.apache.solr.handler.admin;
*/
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -26,6 +28,7 @@ import org.apache.solr.client.solrj.requ
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.OverseerCollectionProcessor;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -147,6 +150,8 @@ public class CollectionsHandler extends
ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(leaderProps);
HttpSolrServer server = new HttpSolrServer(nodeProps.getBaseUrl());
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
RequestSyncShard reqSyncShard = new CoreAdminRequest.RequestSyncShard();
reqSyncShard.setCollection(collection);
reqSyncShard.setShard(shard);
@@ -176,14 +181,32 @@ public class CollectionsHandler extends
private void handleCreateAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws InterruptedException, KeeperException {
log.info("Creating Collection : " + req.getParamString());
- Integer numReplicas = req.getParams().getInt(OverseerCollectionProcessor.REPLICATION_FACTOR, 0);
+ Integer numReplicas = req.getParams().getInt(OverseerCollectionProcessor.REPLICATION_FACTOR, 1);
String name = req.getParams().required().get("name");
String configName = req.getParams().get("collection.configName");
- String numShards = req.getParams().get("numShards");
+ String numShards = req.getParams().get(OverseerCollectionProcessor.NUM_SLICES);
+ String maxShardsPerNode = req.getParams().get(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE);
+ String createNodeSetStr = req.getParams().get(OverseerCollectionProcessor.CREATE_NODE_SET);
+
+ if (name == null) {
+ log.error("Collection name is required to create a new collection");
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Collection name is required to create a new collection");
+ }
- ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
- OverseerCollectionProcessor.CREATECOLLECTION, OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas.toString(), "name", name,
- "collection.configName", configName, "numShards", numShards);
+ Map<String,Object> props = new HashMap<String,Object>();
+ props.put(Overseer.QUEUE_OPERATION,
+ OverseerCollectionProcessor.CREATECOLLECTION);
+ props.put(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas.toString());
+ props.put("name", name);
+ if (configName != null) {
+ props.put("collection.configName", configName);
+ }
+ props.put(OverseerCollectionProcessor.NUM_SLICES, numShards);
+ props.put(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
+ props.put(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
+
+ ZkNodeProps m = new ZkNodeProps(props);
// TODO: what if you want to block until the collection is available?
coreContainer.getZkController().getOverseerCollectionQueue().offer(ZkStateReader.toJSON(m));
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Fri Jan 18 18:30:54 2013
@@ -29,6 +29,8 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
+import javax.xml.parsers.ParserConfigurationException;
+
import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.store.Directory;
@@ -39,7 +41,7 @@ import org.apache.solr.cloud.ZkControlle
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.HashPartitioner;
+import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -55,6 +57,7 @@ import org.apache.solr.core.CoreContaine
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
@@ -71,8 +74,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
-import javax.xml.parsers.ParserConfigurationException;
-
/**
*
* @since solr 1.3
@@ -216,7 +217,7 @@ public class CoreAdminHandler extends Re
}
/** Creates a new core and registers it. The returned core will have it's reference count incremented an extra time and close() should be called when finished. */
- private SolrCore createCore(SolrCore current, int ord, HashPartitioner.Range newRange) throws IOException, SAXException, ParserConfigurationException {
+ private SolrCore createCore(SolrCore current, int ord, DocRouter.Range newRange) throws IOException, SAXException, ParserConfigurationException {
CoreDescriptor currCoreD = current.getCoreDescriptor();
CloudDescriptor currCloudD = currCoreD.getCloudDescriptor();
@@ -268,7 +269,7 @@ public class CoreAdminHandler extends Re
// partitions=N (split into N partitions, leaving it up to solr what the ranges are and where to put them)
// path - multiValued param, or comma separated param? Only creates indexes, not cores
- List<HashPartitioner.Range> ranges = null;
+ List<DocRouter.Range> ranges = null;
// boolean closeDirectories = true;
// DirectoryFactory dirFactory = null;
@@ -290,9 +291,9 @@ public class CoreAdminHandler extends Re
// split on every other doc rather than hash.
// TODO (cloud): get from the current core
- HashPartitioner.Range currentRange = new HashPartitioner.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
+ DocRouter.Range currentRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
- HashPartitioner hp = new HashPartitioner();
+ DocRouter hp = DocRouter.DEFAULT; // TODO: get actual doc router for collection if available
ranges = hp.partitionRange(partitions, currentRange);
if (pathsArr == null) {
@@ -367,7 +368,7 @@ public class CoreAdminHandler extends Re
dirsToBeReleased = new Directory[dirNames.length];
DirectoryFactory dirFactory = core.getDirectoryFactory();
for (int i = 0; i < dirNames.length; i++) {
- Directory dir = dirFactory.get(dirNames[i], core.getSolrConfig().indexConfig.lockType);
+ Directory dir = dirFactory.get(dirNames[i], DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
dirsToBeReleased[i] = dir;
// TODO: why doesn't this use the IR factory? what is going on here?
readersToBeClosed[i] = DirectoryReader.open(dir);
@@ -478,6 +479,10 @@ public class CoreAdminHandler extends Re
opts = params.get(CoreAdminParams.DATA_DIR);
if (opts != null)
dcore.setDataDir(opts);
+
+ opts = params.get(CoreAdminParams.ULOG_DIR);
+ if (opts != null)
+ dcore.setUlogDir(opts);
CloudDescriptor cd = dcore.getCloudDescriptor();
if (cd != null) {
@@ -610,26 +615,10 @@ public class CoreAdminHandler extends Re
@Override
public void postClose(SolrCore core) {
- Directory dir = null;
try {
- dir = core.getDirectoryFactory().get(core.getIndexDir(), null);
- core.getDirectoryFactory().remove(dir);
- core.getDirectoryFactory().doneWithDirectory(dir);
+ core.getDirectoryFactory().remove(core.getIndexDir());
} catch (IOException e) {
throw new RuntimeException(e);
- } finally {
- if (dir != null) {
- try {
- core.getDirectoryFactory().release(dir);
- } catch (IOException e) {
- log.error("IOException trying to release directory", e);
- }
- }
- }
- try {
- core.getDirectoryFactory().remove(dir);
- } catch (IOException e) {
- log.error("IOException trying to remove directory", e);
}
}
});
@@ -700,6 +689,8 @@ public class CoreAdminHandler extends Re
SolrParams params = req.getParams();
String cname = params.get(CoreAdminParams.CORE);
+ String indexInfo = params.get(CoreAdminParams.INDEX_INFO);
+ boolean isIndexInfoNeeded = Boolean.parseBoolean(null == indexInfo ? "true" : indexInfo);
boolean doPersist = false;
NamedList<Object> status = new SimpleOrderedMap<Object>();
Map<String,Exception> allFailures = coreContainer.getCoreInitFailures();
@@ -707,7 +698,7 @@ public class CoreAdminHandler extends Re
if (cname == null) {
rsp.add("defaultCoreName", coreContainer.getDefaultCoreName());
for (String name : coreContainer.getCoreNames()) {
- status.add(name, getCoreStatus(coreContainer, name));
+ status.add(name, getCoreStatus(coreContainer, name, isIndexInfoNeeded));
}
rsp.add("initFailures", allFailures);
} else {
@@ -715,7 +706,7 @@ public class CoreAdminHandler extends Re
? Collections.singletonMap(cname, allFailures.get(cname))
: Collections.emptyMap();
rsp.add("initFailures", failures);
- status.add(cname, getCoreStatus(coreContainer, cname));
+ status.add(cname, getCoreStatus(coreContainer, cname, isIndexInfoNeeded));
}
rsp.add("status", status);
doPersist = false; // no state change
@@ -789,33 +780,41 @@ public class CoreAdminHandler extends Re
SolrQueryResponse rsp) throws IOException {
final SolrParams params = req.getParams();
log.info("It has been requested that we recover");
- String cname = params.get(CoreAdminParams.CORE);
- if (cname == null) {
- cname = "";
- }
- SolrCore core = null;
- try {
- core = coreContainer.getCore(cname);
- if (core != null) {
- // try to publish as recovering right away
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ String cname = params.get(CoreAdminParams.CORE);
+ if (cname == null) {
+ cname = "";
+ }
+ SolrCore core = null;
try {
- coreContainer.getZkController().publish(core.getCoreDescriptor(), ZkStateReader.RECOVERING);
- } catch (KeeperException e) {
- SolrException.log(log, "", e);
- } catch (InterruptedException e) {
- SolrException.log(log, "", e);
+ core = coreContainer.getCore(cname);
+ if (core != null) {
+ // try to publish as recovering right away
+ try {
+ coreContainer.getZkController().publish(core.getCoreDescriptor(), ZkStateReader.RECOVERING);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ SolrException.log(log, "", e);
+ } catch (Throwable t) {
+ SolrException.log(log, "", t);
+ }
+
+ core.getUpdateHandler().getSolrCoreState().doRecovery(coreContainer, cname);
+ } else {
+ SolrException.log(log, "Cound not find core to call recovery:" + cname);
+ }
+ } finally {
+ // no recoveryStrat close for now
+ if (core != null) {
+ core.close();
+ }
}
-
- core.getUpdateHandler().getSolrCoreState().doRecovery(coreContainer, cname);
- } else {
- SolrException.log(log, "Cound not find core to call recovery:" + cname);
- }
- } finally {
- // no recoveryStrat close for now
- if (core != null) {
- core.close();
}
- }
+ };
+
+ thread.start();
}
protected void handleRequestSyncAction(SolrQueryRequest req,
@@ -880,7 +879,7 @@ public class CoreAdminHandler extends Re
}
protected void handleWaitForStateAction(SolrQueryRequest req,
- SolrQueryResponse rsp) throws IOException, InterruptedException {
+ SolrQueryResponse rsp) throws IOException, InterruptedException, KeeperException {
final SolrParams params = req.getParams();
String cname = params.get(CoreAdminParams.CORE);
@@ -916,6 +915,12 @@ public class CoreAdminHandler extends Re
// to accept updates
CloudDescriptor cloudDescriptor = core.getCoreDescriptor()
.getCloudDescriptor();
+
+ if (retry == 15 || retry == 60) {
+ // force a cluster state update
+ coreContainer.getZkController().getZkStateReader().updateClusterState(true);
+ }
+
ClusterState clusterState = coreContainer.getZkController()
.getClusterState();
String collection = cloudDescriptor.getCollectionName();
@@ -939,13 +944,41 @@ public class CoreAdminHandler extends Re
}
}
- if (retry++ == 30) {
+ if (retry++ == 120) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"I was asked to wait on state " + waitForState + " for "
+ nodeName
- + " but I still do not see the request state. I see state: "
+ + " but I still do not see the requested state. I see state: "
+ state + " live:" + live);
}
+
+ if (coreContainer.isShutDown()) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Solr is shutting down");
+ }
+
+ // solrcloud_debug
+// try {;
+// LocalSolrQueryRequest r = new LocalSolrQueryRequest(core, new
+// ModifiableSolrParams());
+// CommitUpdateCommand commitCmd = new CommitUpdateCommand(r, false);
+// commitCmd.softCommit = true;
+// core.getUpdateHandler().commit(commitCmd);
+// RefCounted<SolrIndexSearcher> searchHolder =
+// core.getNewestSearcher(false);
+// SolrIndexSearcher searcher = searchHolder.get();
+// try {
+// System.out.println(core.getCoreDescriptor().getCoreContainer().getZkController().getNodeName()
+// + " to replicate "
+// + searcher.search(new MatchAllDocsQuery(), 1).totalHits + " gen:" +
+// core.getDeletionPolicy().getLatestCommit().getGeneration() + " data:" +
+// core.getDataDir());
+// } finally {
+// searchHolder.decref();
+// }
+// } catch (Exception e) {
+//
+// }
} finally {
if (core != null) {
core.close();
@@ -955,32 +988,9 @@ public class CoreAdminHandler extends Re
}
- // solrcloud_debug
- // try {;
- // LocalSolrQueryRequest r = new LocalSolrQueryRequest(core, new
- // ModifiableSolrParams());
- // CommitUpdateCommand commitCmd = new CommitUpdateCommand(r, false);
- // commitCmd.softCommit = true;
- // core.getUpdateHandler().commit(commitCmd);
- // RefCounted<SolrIndexSearcher> searchHolder =
- // core.getNewestSearcher(false);
- // SolrIndexSearcher searcher = searchHolder.get();
- // try {
- // System.out.println(core.getCoreDescriptor().getCoreContainer().getZkController().getNodeName()
- // + " to replicate "
- // + searcher.search(new MatchAllDocsQuery(), 1).totalHits + " gen:" +
- // core.getDeletionPolicy().getLatestCommit().getGeneration() + " data:" +
- // core.getDataDir());
- // } finally {
- // searchHolder.decref();
- // }
- // } catch (Exception e) {
- //
- // }
-
}
- protected NamedList<Object> getCoreStatus(CoreContainer cores, String cname) throws IOException {
+ protected NamedList<Object> getCoreStatus(CoreContainer cores, String cname, boolean isIndexInfoNeeded) throws IOException {
NamedList<Object> info = new SimpleOrderedMap<Object>();
SolrCore core = cores.getCore(cname);
if (core != null) {
@@ -993,15 +1003,17 @@ public class CoreAdminHandler extends Re
info.add("schema", core.getSchemaResource());
info.add("startTime", new Date(core.getStartTime()));
info.add("uptime", System.currentTimeMillis() - core.getStartTime());
- RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
- try {
- SimpleOrderedMap<Object> indexInfo = LukeRequestHandler.getIndexInfo(searcher.get().getIndexReader());
- long size = getIndexSize(core);
- indexInfo.add("sizeInBytes", size);
- indexInfo.add("size", NumberUtils.readableSize(size));
- info.add("index", indexInfo);
- } finally {
- searcher.decref();
+ if (isIndexInfoNeeded) {
+ RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
+ try {
+ SimpleOrderedMap<Object> indexInfo = LukeRequestHandler.getIndexInfo(searcher.get().getIndexReader());
+ long size = getIndexSize(core);
+ indexInfo.add("sizeInBytes", size);
+ indexInfo.add("size", NumberUtils.readableSize(size));
+ info.add("index", indexInfo);
+ } finally {
+ searcher.decref();
+ }
}
} finally {
core.close();
@@ -1014,7 +1026,12 @@ public class CoreAdminHandler extends Re
Directory dir;
long size = 0;
try {
- dir = core.getDirectoryFactory().get(core.getIndexDir(), null);
+ if (!core.getDirectoryFactory().exists(core.getIndexDir())) {
+ dir = core.getDirectoryFactory().get(core.getNewIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
+ } else {
+ dir = core.getDirectoryFactory().get(core.getIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
+ }
+
try {
size = DirectoryFactory.sizeOfDirectory(dir);
} finally {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Fri Jan 18 18:30:54 2013
@@ -389,7 +389,7 @@ public class LukeRequestHandler extends
if (text == null) { // Ran off the end of the terms enum without finding any live docs with that field in them.
return null;
}
- docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+ docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE);
if (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
return reader.document(docsEnum.docID());
}
@@ -547,6 +547,7 @@ public class LukeRequestHandler extends
indexInfo.add("numDocs", reader.numDocs());
indexInfo.add("maxDoc", reader.maxDoc());
+ indexInfo.add("deletedDocs", reader.maxDoc() - reader.numDocs());
indexInfo.add("version", reader.getVersion()); // TODO? Is this different then: IndexReader.getCurrentVersion( dir )?
indexInfo.add("segmentCount", reader.leaves().size());
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java Fri Jan 18 18:30:54 2013
@@ -154,8 +154,9 @@ public class ShowFileRequestHandler exte
// Make sure the file exists, is readable and is not a hidden file
if (!zkClient.exists(adminFile, true)) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find: "
- + adminFile);
+ rsp.setException(new SolrException(ErrorCode.NOT_FOUND, "Can not find: "
+ + adminFile));
+ return;
}
// Show a directory listing
@@ -210,7 +211,8 @@ public class ShowFileRequestHandler exte
try {
configdir = new File( loader.getClassLoader().getResource(loader.getConfigDir()).toURI() );
} catch (URISyntaxException e) {
- throw new SolrException( ErrorCode.FORBIDDEN, "Can not access configuration directory!");
+ rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Can not access configuration directory!", e));
+ return;
}
}
String fname = req.getParams().get("file", null);
@@ -232,12 +234,16 @@ public class ShowFileRequestHandler exte
// Make sure the file exists, is readable and is not a hidden file
if( !adminFile.exists() ) {
- throw new SolrException( ErrorCode.BAD_REQUEST, "Can not find: "+adminFile.getName()
- + " ["+adminFile.getAbsolutePath()+"]" );
+ rsp.setException(new SolrException
+ ( ErrorCode.NOT_FOUND, "Can not find: "+adminFile.getName()
+ + " ["+adminFile.getAbsolutePath()+"]" ));
+ return;
}
if( !adminFile.canRead() || adminFile.isHidden() ) {
- throw new SolrException( ErrorCode.BAD_REQUEST, "Can not show: "+adminFile.getName()
- + " ["+adminFile.getAbsolutePath()+"]" );
+ rsp.setException(new SolrException
+ ( ErrorCode.NOT_FOUND, "Can not show: "+adminFile.getName()
+ + " ["+adminFile.getAbsolutePath()+"]" ));
+ return;
}
// Show a directory listing