You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2008/10/13 11:12:43 UTC
svn commit: r703981 - in /lucene/solr/trunk: ./
src/java/org/apache/solr/core/ src/java/org/apache/solr/search/
src/java/org/apache/solr/update/ src/test/org/apache/solr/core/
Author: shalin
Date: Mon Oct 13 02:12:42 2008
New Revision: 703981
URL: http://svn.apache.org/viewvc?rev=703981&view=rev
Log:
SOLR-658 -- Allow Solr to load index from arbitrary directory in dataDir
Added:
lucene/solr/trunk/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (with props)
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java
lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/solr/trunk/src/java/org/apache/solr/update/UpdateHandler.java
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=703981&r1=703980&r2=703981&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Mon Oct 13 02:12:42 2008
@@ -45,6 +45,9 @@
3. SOLR-657: Replace deprecated calls with the non-deprecated equivalents
(Lars Kotthoff via ryan)
+
+ 4. SOLR-658: Allow Solr to load index from arbitrary directory in dataDir
+ (Noble Paul, Akshay Ukey via shalin)
Optimizations
Modified: lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java?rev=703981&r1=703980&r2=703981&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java Mon Oct 13 02:12:42 2008
@@ -51,6 +51,7 @@
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.NamedListPluginLoader;
import org.apache.solr.util.plugin.SolrCoreAware;
+import org.apache.commons.io.IOUtils;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
@@ -61,7 +62,9 @@
import javax.xml.xpath.XPathExpressionException;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -174,11 +177,47 @@
public String getDataDir() {
return dataDir;
}
-
+
public String getIndexDir() {
- return dataDir + "index/";
+ if (_searcher == null)
+ return dataDir + "index/";
+ SolrIndexSearcher searcher = _searcher.get();
+ return searcher.getIndexDir() == null ? dataDir + "index/" : searcher.getIndexDir();
}
-
+
+
+ /**
+ * Returns the indexdir as given in index.properties. If index.properties exists in dataDir and
+ * there is a property <i>index</i> available and it points to a valid directory
+ * in dataDir that is returned Else dataDir/index is returned. Only called for creating new indexSearchers
+ * and indexwriters. Use the getIndexDir() method to know the active index directory
+ *
+ * @return the indexdir as given in index.properties
+ */
+ public String getNewIndexDir() {
+ String result = dataDir + "index/";
+ File propsFile = new File(dataDir + "index.properties");
+ if (propsFile.exists()) {
+ Properties p = new Properties();
+ InputStream is = null;
+ try {
+ is = new FileInputStream(propsFile);
+ p.load(is);
+ } catch (IOException e) {
+ /*no op*/
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ String s = p.getProperty("index");
+ if (s != null && s.trim().length() > 0) {
+ File tmp = new File(dataDir + s);
+ if (tmp.exists() && tmp.isDirectory())
+ result = dataDir + s;
+ }
+ }
+ return result;
+ }
+
public String getName() {
return name;
}
@@ -290,7 +329,7 @@
// currently only called with SolrCore.class lock held
void initIndex() {
try {
- File dirFile = new File(getIndexDir());
+ File dirFile = new File(getNewIndexDir());
boolean indexExists = dirFile.canRead();
boolean firstTime = dirs.add(dirFile.getCanonicalPath());
boolean removeLocks = solrConfig.getBool("mainIndex/unlockOnStartup", false);
@@ -971,15 +1010,20 @@
newestSearcher = getNewestSearcher(false);
if (newestSearcher != null) {
IndexReader currentReader = newestSearcher.get().getReader();
- IndexReader newReader = currentReader.reopen();
+ String newIndexDir = getNewIndexDir();
+ if(new File(getIndexDir()).equals(new File(newIndexDir))) {
+ IndexReader newReader = currentReader.reopen();
+
+ if(newReader == currentReader) {
+ currentReader.incRef();
+ }
- if(newReader == currentReader) {
- currentReader.incRef();
+ tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true);
+ } else {
+ tmp = new SolrIndexSearcher(this, schema, "main", newIndexDir, true);
}
-
- tmp = new SolrIndexSearcher(this, schema, "main", newReader, true, true);
} else {
- tmp = new SolrIndexSearcher(this, schema, "main", IndexReader.open(FSDirectory.getDirectory(getIndexDir()), true), true, true);
+ tmp = new SolrIndexSearcher(this, schema, "main", getNewIndexDir(), true);
}
} catch (Throwable th) {
synchronized(searcherLock) {
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=703981&r1=703980&r2=703981&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java Mon Oct 13 02:12:42 2008
@@ -23,6 +23,7 @@
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -57,6 +58,7 @@
private static Logger log = LoggerFactory.getLogger(SolrIndexSearcher.class);
private final SolrCore core;
private final IndexSchema schema;
+ private String indexDir;
private final String name;
private long openTime = System.currentTimeMillis();
@@ -111,6 +113,11 @@
log.info("Opening " + this.name);
+ if (r.directory() instanceof FSDirectory) {
+ FSDirectory fsDirectory = (FSDirectory) r.directory();
+ indexDir = fsDirectory.getFile().getAbsolutePath();
+ }
+
reader = r;
searcher = new IndexSearcher(r);
this.closeReader = closeReader;
@@ -311,6 +318,14 @@
return searcher.docFreq(term);
}
+ /**
+ * @return the indexDir on which this searcher is opened
+ * @see org.apache.solr.search.SolrIndexSearcher#SolrIndexSearcher(org.apache.solr.core.SolrCore, org.apache.solr.schema.IndexSchema, String, String, boolean)
+ */
+ public String getIndexDir() {
+ return indexDir;
+ }
+
/* ********************** Document retrieval *************************/
/* Future optimizations (yonik)
Modified: lucene/solr/trunk/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/UpdateHandler.java?rev=703981&r1=703980&r2=703981&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/UpdateHandler.java Mon Oct 13 02:12:42 2008
@@ -120,7 +120,7 @@
}
protected SolrIndexWriter createMainIndexWriter(String name, boolean removeAllExisting) throws IOException {
- return new SolrIndexWriter(name,core.getIndexDir(), removeAllExisting, schema, core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy());
+ return new SolrIndexWriter(name,core.getNewIndexDir(), removeAllExisting, schema, core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy());
}
protected final Term idTerm(String readableId) {
Added: lucene/solr/trunk/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/core/TestArbitraryIndexDir.java?rev=703981&view=auto
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (added)
+++ lucene/solr/trunk/src/test/org/apache/solr/core/TestArbitraryIndexDir.java Mon Oct 13 02:12:42 2008
@@ -0,0 +1,102 @@
+package org.apache.solr.core;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.TestHarness;
+import org.junit.Test;
+import org.xml.sax.SAXException;
+
+/**
+ * @version $Id$
+ */
+public class TestArbitraryIndexDir extends AbstractSolrTestCase{
+
+ public void setUp() throws Exception {
+ dataDir = new File(System.getProperty("java.io.tmpdir")
+ + System.getProperty("file.separator")
+ + getClass().getName() + "-" + System.currentTimeMillis() + System.getProperty("file.separator") + "solr"
+ + System.getProperty("file.separator") + "data");
+ dataDir.mkdirs();
+
+ solrConfig = h.createConfig(getSolrConfigFile());
+ h = new TestHarness( dataDir.getAbsolutePath(),
+ solrConfig,
+ getSchemaFile());
+ lrf = h.getRequestFactory
+ ("standard",0,20,"version","2.2");
+ }
+
+ public void tearDown() throws Exception {
+ super.tearDown();
+
+ }
+
+ @Override
+ public String getSchemaFile() {
+ return "schema.xml";
+ }
+
+ @Override
+ public String getSolrConfigFile() {
+ return "solrconfig.xml";
+ }
+
+ @Test
+ public void testLoadNewIndexDir() throws IOException, ParserConfigurationException, SAXException, ParseException{
+ //add a doc in original index dir
+ assertU(adoc("id", String.valueOf(1),
+ "name", "name"+String.valueOf(1)));
+ //create a new index dir and index.properties file
+ File idxprops = new File(h.getCore().getDataDir() + "index.properties");
+ Properties p = new Properties();
+ File newDir = new File(h.getCore().getDataDir() + "index_temp");
+ newDir.mkdirs();
+ p.put("index", newDir.getName());
+ FileOutputStream os = null;
+ try {
+ os = new FileOutputStream(idxprops);
+ p.store(os, "index properties");
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to write index.properties", e);
+ }
+
+ //add a doc in the new index dir
+ Directory dir = FSDirectory.getDirectory(newDir);
+ IndexWriter iw = new IndexWriter(dir, new StandardAnalyzer(), new MaxFieldLength(1000));
+ Document doc = new Document();
+ doc.add(new Field("id", "2", Field.Store.YES, Field.Index.TOKENIZED));
+ doc.add(new Field("name", "name2", Field.Store.YES, Field.Index.TOKENIZED));
+ iw.addDocument(doc);
+ iw.commit();
+ iw.close();
+
+ //commit will cause searcher to open with the new index dir
+ assertU(commit());
+ //new index dir contains just 1 doc.
+ assertQ("return doc with id 2",
+ req("id:2"),
+ "*[count(//doc)=1]"
+ );
+ newDir.delete();
+ }
+}
Propchange: lucene/solr/trunk/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL