You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/19 17:59:32 UTC
svn commit: r1363400 [27/31] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/copyright/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/maven/ dev-tools/maven/lucene...
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Thu Jul 19 15:58:54 2012
@@ -22,7 +22,9 @@ package org.apache.solr.update;
import java.io.IOException;
import java.net.URL;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -42,11 +44,9 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;
@@ -93,7 +93,7 @@ public class DirectUpdateHandler2 extend
protected final CommitTracker commitTracker;
protected final CommitTracker softCommitTracker;
- public DirectUpdateHandler2(SolrCore core) throws IOException {
+ public DirectUpdateHandler2(SolrCore core) {
super(core);
solrCoreState = new DefaultSolrCoreState(core.getDirectoryFactory());
@@ -109,7 +109,7 @@ public class DirectUpdateHandler2 extend
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
}
- public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) throws IOException {
+ public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
super(core);
if (updateHandler instanceof DirectUpdateHandler2) {
this.solrCoreState = ((DirectUpdateHandler2) updateHandler).solrCoreState;
@@ -137,7 +137,12 @@ public class DirectUpdateHandler2 extend
private void deleteAll() throws IOException {
SolrCore.log.info(core.getLogId()+"REMOVING ALL DOCUMENTS FROM INDEX");
- solrCoreState.getIndexWriter(core).deleteAll();
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
+ try {
+ iw.get().deleteAll();
+ } finally {
+ iw.decref();
+ }
}
protected void rollbackWriter() throws IOException {
@@ -148,84 +153,124 @@ public class DirectUpdateHandler2 extend
@Override
public int addDoc(AddUpdateCommand cmd) throws IOException {
- IndexWriter writer = solrCoreState.getIndexWriter(core);
- addCommands.incrementAndGet();
- addCommandsCumulative.incrementAndGet();
- int rc=-1;
-
- // if there is no ID field, don't overwrite
- if( idField == null ) {
- cmd.overwrite = false;
- }
-
-
+ int rc = -1;
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
-
- if (cmd.overwrite) {
- Term updateTerm;
- Term idTerm = new Term(idField.getName(), cmd.getIndexedId());
- boolean del = false;
- if (cmd.updateTerm == null) {
- updateTerm = idTerm;
+ IndexWriter writer = iw.get();
+ addCommands.incrementAndGet();
+ addCommandsCumulative.incrementAndGet();
+
+ // if there is no ID field, don't overwrite
+ if (idField == null) {
+ cmd.overwrite = false;
+ }
+
+ try {
+
+ if (cmd.overwrite) {
+
+ // Check for delete by query commands newer (i.e. reordered). This
+ // should always be null on a leader
+ List<UpdateLog.DBQ> deletesAfter = null;
+ if (ulog != null && cmd.version > 0) {
+ deletesAfter = ulog.getDBQNewer(cmd.version);
+ }
+
+ if (deletesAfter != null) {
+ log.info("Reordered DBQs detected. Update=" + cmd + " DBQs="
+ + deletesAfter);
+ List<Query> dbqList = new ArrayList<Query>(deletesAfter.size());
+ for (UpdateLog.DBQ dbq : deletesAfter) {
+ try {
+ DeleteUpdateCommand tmpDel = new DeleteUpdateCommand(cmd.req);
+ tmpDel.query = dbq.q;
+ tmpDel.version = -dbq.version;
+ dbqList.add(getQuery(tmpDel));
+ } catch (Exception e) {
+ log.error("Exception parsing reordered query : " + dbq, e);
+ }
+ }
+
+ addAndDelete(cmd, dbqList);
+ } else {
+ // normal update
+
+ Term updateTerm;
+ Term idTerm = new Term(idField.getName(), cmd.getIndexedId());
+ boolean del = false;
+ if (cmd.updateTerm == null) {
+ updateTerm = idTerm;
+ } else {
+ del = true;
+ updateTerm = cmd.updateTerm;
+ }
+
+ Document luceneDocument = cmd.getLuceneDocument();
+ // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer);
+ writer.updateDocument(updateTerm, luceneDocument,
+ schema.getAnalyzer());
+ // SolrCore.verbose("updateDocument",updateTerm,"DONE");
+
+ if (del) { // ensure id remains unique
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(new BooleanClause(new TermQuery(updateTerm),
+ Occur.MUST_NOT));
+ bq.add(new BooleanClause(new TermQuery(idTerm), Occur.MUST));
+ writer.deleteDocuments(bq);
+ }
+
+ // Add to the transaction log *after* successfully adding to the
+ // index, if there was no error.
+ // This ordering ensures that if we log it, it's definitely been
+ // added to the the index.
+ // This also ensures that if a commit sneaks in-between, that we
+ // know everything in a particular
+ // log version was definitely committed.
+ if (ulog != null) ulog.add(cmd);
+ }
+
} else {
- del = true;
- updateTerm = cmd.updateTerm;
+ // allow duplicates
+ writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer());
+ if (ulog != null) ulog.add(cmd);
}
-
- Document luceneDocument = cmd.getLuceneDocument();
- // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer);
- writer.updateDocument(updateTerm, luceneDocument);
- // SolrCore.verbose("updateDocument",updateTerm,"DONE");
-
- if(del) { // ensure id remains unique
- BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause(new TermQuery(updateTerm), Occur.MUST_NOT));
- bq.add(new BooleanClause(new TermQuery(idTerm), Occur.MUST));
- writer.deleteDocuments(bq);
+
+ if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
+ commitTracker.addedDocument(-1);
+ softCommitTracker.addedDocument(cmd.commitWithin);
+ }
+
+ rc = 1;
+ } finally {
+ if (rc != 1) {
+ numErrors.incrementAndGet();
+ numErrorsCumulative.incrementAndGet();
+ } else {
+ numDocsPending.incrementAndGet();
}
- } else {
- // allow duplicates
- writer.addDocument(cmd.getLuceneDocument());
- }
-
- // Add to the transaction log *after* successfully adding to the index, if there was no error.
- // This ordering ensures that if we log it, it's definitely been added to the the index.
- // This also ensures that if a commit sneaks in-between, that we know everything in a particular
- // log version was definitely committed.
- if (ulog != null) ulog.add(cmd);
-
- if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
- commitTracker.addedDocument( -1 );
- softCommitTracker.addedDocument( cmd.commitWithin );
}
-
- rc = 1;
+
} finally {
- if (rc!=1) {
- numErrors.incrementAndGet();
- numErrorsCumulative.incrementAndGet();
- } else {
- numDocsPending.incrementAndGet();
- }
+ iw.decref();
}
-
+
return rc;
}
-
+
private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
- softCommitTracker.deletedDocument( cmd.commitWithin );
-
+ softCommitTracker.deletedDocument(cmd.commitWithin);
+
if (commitTracker.getTimeUpperBound() > 0) {
commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
}
-
+
if (softCommitTracker.getTimeUpperBound() > 0) {
- softCommitTracker.scheduleCommitWithin(softCommitTracker.getTimeUpperBound());
+ softCommitTracker.scheduleCommitWithin(softCommitTracker
+ .getTimeUpperBound());
}
}
}
-
// we don't return the number of docs deleted because it's not always possible to quickly know that info.
@Override
@@ -233,11 +278,14 @@ public class DirectUpdateHandler2 extend
deleteByIdCommands.incrementAndGet();
deleteByIdCommandsCumulative.incrementAndGet();
- IndexWriter writer = solrCoreState.getIndexWriter(core);
Term deleteTerm = new Term(idField.getName(), cmd.getIndexedId());
-
// SolrCore.verbose("deleteDocuments",deleteTerm,writer);
- writer.deleteDocuments(deleteTerm);
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
+ try {
+ iw.get().deleteDocuments(deleteTerm);
+ } finally {
+ iw.decref();
+ }
// SolrCore.verbose("deleteDocuments",deleteTerm,"DONE");
if (ulog != null) ulog.delete(cmd);
@@ -245,6 +293,43 @@ public class DirectUpdateHandler2 extend
updateDeleteTrackers(cmd);
}
+
+ public void clearIndex() throws IOException {
+ deleteAll();
+ if (ulog != null) {
+ ulog.deleteAll();
+ }
+ }
+
+
+ private Query getQuery(DeleteUpdateCommand cmd) {
+ Query q;
+ try {
+ // move this higher in the stack?
+ QParser parser = QParser.getParser(cmd.getQuery(), "lucene", cmd.req);
+ q = parser.getQuery();
+ q = QueryUtils.makeQueryable(q);
+
+ // Make sure not to delete newer versions
+ if (ulog != null && cmd.getVersion() != 0 && cmd.getVersion() != -Long.MAX_VALUE) {
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(q, Occur.MUST);
+ SchemaField sf = ulog.getVersionInfo().getVersionField();
+ ValueSource vs = sf.getType().getValueSource(sf, null);
+ ValueSourceRangeFilter filt = new ValueSourceRangeFilter(vs, null, Long.toString(Math.abs(cmd.getVersion())), true, true);
+ FunctionRangeQuery range = new FunctionRangeQuery(filt);
+ bq.add(range, Occur.MUST);
+ q = bq;
+ }
+
+ return q;
+
+ } catch (ParseException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+ }
+
+
// we don't return the number of docs deleted because it's not always possible to quickly know that info.
@Override
public void deleteByQuery(DeleteUpdateCommand cmd) throws IOException {
@@ -252,35 +337,19 @@ public class DirectUpdateHandler2 extend
deleteByQueryCommandsCumulative.incrementAndGet();
boolean madeIt=false;
try {
- Query q;
- try {
- // TODO: move this higher in the stack?
- QParser parser = QParser.getParser(cmd.query, "lucene", cmd.req);
- q = parser.getQuery();
- q = QueryUtils.makeQueryable(q);
-
- // peer-sync can cause older deleteByQueries to be executed and could
- // delete newer documents. We prevent this by adding a clause restricting
- // version.
- if ((cmd.getFlags() & UpdateCommand.PEER_SYNC) != 0) {
- BooleanQuery bq = new BooleanQuery();
- bq.add(q, Occur.MUST);
- SchemaField sf = core.getSchema().getField(VersionInfo.VERSION_FIELD);
- ValueSource vs = sf.getType().getValueSource(sf, null);
- ValueSourceRangeFilter filt = new ValueSourceRangeFilter(vs, null, Long.toString(Math.abs(cmd.version)), true, true);
- FunctionRangeQuery range = new FunctionRangeQuery(filt);
- bq.add(range, Occur.MUST);
- q = bq;
- }
-
-
-
- } catch (ParseException e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
- }
+ Query q = getQuery(cmd);
boolean delAll = MatchAllDocsQuery.class == q.getClass();
+ // currently for testing purposes. Do a delete of complete index w/o worrying about versions, don't log, clean up most state in update log, etc
+ if (delAll && cmd.getVersion() == -Long.MAX_VALUE) {
+ synchronized (this) {
+ deleteAll();
+ ulog.deleteAll();
+ return;
+ }
+ }
+
//
// synchronized to prevent deleteByQuery from running during the "open new searcher"
// part of a commit. DBQ needs to signal that a fresh reader will be needed for
@@ -291,7 +360,12 @@ public class DirectUpdateHandler2 extend
if (delAll) {
deleteAll();
} else {
- solrCoreState.getIndexWriter(core).deleteDocuments(q);
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
+ try {
+ iw.get().deleteDocuments(q);
+ } finally {
+ iw.decref();
+ }
}
if (ulog != null) ulog.deleteByQuery(cmd);
@@ -309,6 +383,37 @@ public class DirectUpdateHandler2 extend
}
}
+
+
+ /** Add a document execute the deletes as atomically as possible */
+ private void addAndDelete(AddUpdateCommand cmd, List<Query> dbqList)
+ throws IOException {
+ Document luceneDocument = cmd.getLuceneDocument();
+ Term idTerm = new Term(idField.getName(), cmd.getIndexedId());
+
+ // see comment in deleteByQuery
+ synchronized (this) {
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
+ try {
+ IndexWriter writer = iw.get();
+ writer.updateDocument(idTerm, luceneDocument, core.getSchema()
+ .getAnalyzer());
+
+ for (Query q : dbqList) {
+ writer.deleteDocuments(q);
+ }
+ } finally {
+ iw.decref();
+ }
+
+ if (ulog != null) ulog.add(cmd, true);
+ }
+
+ }
+
+
+
+
@Override
public int mergeIndexes(MergeIndexesCommand cmd) throws IOException {
mergeIndexesCommands.incrementAndGet();
@@ -318,7 +423,12 @@ public class DirectUpdateHandler2 extend
IndexReader[] readers = cmd.readers;
if (readers != null && readers.length > 0) {
- solrCoreState.getIndexWriter(core).addIndexes(readers);
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
+ try {
+ iw.get().addIndexes(readers);
+ } finally {
+ iw.decref();
+ }
rc = 1;
} else {
rc = 0;
@@ -341,9 +451,12 @@ public class DirectUpdateHandler2 extend
try {
log.info("start "+cmd);
- IndexWriter writer = solrCoreState.getIndexWriter(core);
-
- writer.prepareCommit();
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
+ try {
+ iw.get().prepareCommit();
+ } finally {
+ iw.decref();
+ }
log.info("end_prepareCommit");
@@ -361,7 +474,6 @@ public class DirectUpdateHandler2 extend
return;
}
- IndexWriter writer = solrCoreState.getIndexWriter(core);
if (cmd.optimize) {
optimizeCommands.incrementAndGet();
} else {
@@ -396,27 +508,35 @@ public class DirectUpdateHandler2 extend
commitTracker.cancelPendingCommit();
}
-
- if (cmd.optimize) {
- writer.forceMerge(cmd.maxOptimizeSegments);
- } else if (cmd.expungeDeletes) {
- writer.forceMergeDeletes();
- }
-
- if (!cmd.softCommit) {
- synchronized (this) { // sync is currently needed to prevent preCommit from being called between preSoft and postSoft... see postSoft comments.
- if (ulog != null) ulog.preCommit(cmd);
+ RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
+ try {
+ IndexWriter writer = iw.get();
+ if (cmd.optimize) {
+ writer.forceMerge(cmd.maxOptimizeSegments);
+ } else if (cmd.expungeDeletes) {
+ writer.forceMergeDeletes();
}
-
- // SolrCore.verbose("writer.commit() start writer=",writer);
- final Map<String,String> commitData = new HashMap<String,String>();
- commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));
- writer.commit(commitData);
- // SolrCore.verbose("writer.commit() end");
- numDocsPending.set(0);
- callPostCommitCallbacks();
- } else {
- callPostSoftCommitCallbacks();
+
+ if (!cmd.softCommit) {
+ synchronized (this) { // sync is currently needed to prevent preCommit
+ // from being called between preSoft and
+ // postSoft... see postSoft comments.
+ if (ulog != null) ulog.preCommit(cmd);
+ }
+
+ // SolrCore.verbose("writer.commit() start writer=",writer);
+ final Map<String,String> commitData = new HashMap<String,String>();
+ commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY,
+ String.valueOf(System.currentTimeMillis()));
+ writer.commit(commitData);
+ // SolrCore.verbose("writer.commit() end");
+ numDocsPending.set(0);
+ callPostCommitCallbacks();
+ } else {
+ callPostSoftCommitCallbacks();
+ }
+ } finally {
+ iw.decref();
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Thu Jul 19 15:58:54 2012
@@ -172,7 +172,7 @@ public class SolrCmdDistributor {
}
private void doDelete(DeleteUpdateCommand cmd, List<Node> nodes,
- ModifiableSolrParams params) throws IOException {
+ ModifiableSolrParams params) {
flushAdds(1);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java Thu Jul 19 15:58:54 2012
@@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.util.RefCounted;
/**
* The state in this class can be easily shared between SolrCores across
@@ -46,7 +47,7 @@ public abstract class SolrCoreState {
*
* @throws IOException
*/
- public abstract IndexWriter getIndexWriter(SolrCore core) throws IOException;
+ public abstract RefCounted<IndexWriter> getIndexWriter(SolrCore core) throws IOException;
/**
* Decrement the number of references to this state. When then number of
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Thu Jul 19 15:58:54 2012
@@ -155,7 +155,11 @@ public class SolrIndexConfig {
}
public IndexWriterConfig toIndexWriterConfig(IndexSchema schema) {
- IndexWriterConfig iwc = new IndexWriterConfig(luceneVersion, schema.getAnalyzer());
+ // so that we can update the analyzer on core reload, we pass null
+ // for the default analyzer, and explicitly pass an analyzer on
+ // appropriate calls to IndexWriter
+
+ IndexWriterConfig iwc = new IndexWriterConfig(luceneVersion, null);
if (maxBufferedDocs != -1)
iwc.setMaxBufferedDocs(maxBufferedDocs);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Thu Jul 19 15:58:54 2012
@@ -75,7 +75,7 @@ public class SolrIndexWriter extends Ind
File parent = f.getParentFile();
if (parent != null) parent.mkdirs();
FileOutputStream fos = new FileOutputStream(f, true);
- return new PrintStreamInfoStream(new PrintStream(fos, true));
+ return new PrintStreamInfoStream(new PrintStream(fos, true, "UTF-8"));
} else {
return InfoStream.NO_OUTPUT;
}
@@ -122,10 +122,11 @@ public class SolrIndexWriter extends Ind
final InfoStream infoStream = isClosed ? null : getConfig().getInfoStream();
try {
super.close();
+ } finally {
if(infoStream != null) {
infoStream.close();
}
- } finally {
+
isClosed = true;
directoryFactory.release(directory);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/TransactionLog.java Thu Jul 19 15:58:54 2012
@@ -18,7 +18,6 @@
package org.apache.solr.update;
import org.apache.lucene.util.BytesRef;
-import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.FastInputStream;
@@ -27,13 +26,19 @@ import org.apache.solr.common.util.JavaB
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
-import java.rmi.registry.LocateRegistry;
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -129,11 +134,11 @@ public class TransactionLog {
}
- TransactionLog(File tlogFile, Collection<String> globalStrings) throws IOException {
+ TransactionLog(File tlogFile, Collection<String> globalStrings) {
this(tlogFile, globalStrings, false);
}
- TransactionLog(File tlogFile, Collection<String> globalStrings, boolean openExisting) throws IOException {
+ TransactionLog(File tlogFile, Collection<String> globalStrings, boolean openExisting) {
try {
if (debug) {
log.debug("New TransactionLog file=" + tlogFile + ", exists=" + tlogFile.exists() + ", size=" + tlogFile.length() + ", openExisting=" + openExisting);
@@ -596,7 +601,7 @@ public class TransactionLog {
ChannelFastInputStream fis;
private LogCodec codec = new LogCodec() {
@Override
- public SolrInputDocument readSolrInputDocument(FastInputStream dis) throws IOException {
+ public SolrInputDocument readSolrInputDocument(FastInputStream dis) {
// Given that the SolrInputDocument is last in an add record, it's OK to just skip
// reading it completely.
return null;
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java Thu Jul 19 15:58:54 2012
@@ -34,6 +34,7 @@ public abstract class UpdateCommand impl
public static int REPLAY = 0x00000002; // update command is from replaying a log.
public static int PEER_SYNC = 0x00000004; // update command is a missing update being provided by a peer.
public static int IGNORE_AUTOCOMMIT = 0x00000008; // this update should not count toward triggering of autocommits.
+ public static int CLEAR_CACHES = 0x00000010; // clear caches associated with the update log. used when applying reordered DBQ updates when doing an add.
public UpdateCommand(SolrQueryRequest req) {
this.req = req;
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateLog.java Thu Jul 19 15:58:54 2012
@@ -17,10 +17,10 @@
package org.apache.solr.update;
+import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -31,7 +31,6 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessor;
@@ -106,7 +105,9 @@ public class UpdateLog implements Plugin
private TransactionLog prevMapLog2; // the transaction log used to look up entries found in prevMap
private final int numDeletesToKeep = 1000;
+ private final int numDeletesByQueryToKeep = 100;
public final int numRecordsToKeep = 100;
+
// keep track of deletes only... this is not updated on an add
private LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
protected boolean removeEldestEntry(Map.Entry eldest) {
@@ -114,6 +115,18 @@ public class UpdateLog implements Plugin
}
};
+ public class DBQ {
+ public String q; // the query string
+ public long version; // positive version of the DBQ
+
+ @Override
+ public String toString() {
+ return "DBQ{version=" + version + ",q="+q+"}";
+ }
+ }
+
+ private LinkedList<DBQ> deleteByQueries = new LinkedList<DBQ>();
+
private String[] tlogFiles;
private File tlogDir;
private Collection<String> globalStrings;
@@ -209,6 +222,16 @@ public class UpdateLog implements Plugin
DeleteUpdate du = startingUpdates.deleteList.get(i);
oldDeletes.put(new BytesRef(du.id), new LogPtr(-1,du.version));
}
+
+ // populate recent deleteByQuery commands
+ for (int i=startingUpdates.deleteByQueryList.size()-1; i>=0; i--) {
+ Update update = startingUpdates.deleteByQueryList.get(i);
+ List<Object> dbq = (List<Object>) update.log.lookup(update.pointer);
+ long version = (Long) dbq.get(1);
+ String q = (String) dbq.get(2);
+ trackDeleteByQuery(q, version);
+ }
+
} finally {
startingUpdates.close();
}
@@ -282,6 +305,11 @@ public class UpdateLog implements Plugin
public void add(AddUpdateCommand cmd) {
+ add(cmd, false);
+ }
+
+
+ public void add(AddUpdateCommand cmd, boolean clearCaches) {
// don't log if we are replaying from another log
// TODO: we currently need to log to maintain correct versioning, rtg, etc
// if ((cmd.getFlags() & UpdateCommand.REPLAY) != 0) return;
@@ -295,21 +323,45 @@ public class UpdateLog implements Plugin
pos = tlog.write(cmd, operationFlags);
}
- // TODO: in the future we could support a real position for a REPLAY update.
- // Only currently would be useful for RTG while in recovery mode though.
- LogPtr ptr = new LogPtr(pos, cmd.getVersion());
+ if (!clearCaches) {
+ // TODO: in the future we could support a real position for a REPLAY update.
+ // Only currently would be useful for RTG while in recovery mode though.
+ LogPtr ptr = new LogPtr(pos, cmd.getVersion());
+
+ // only update our map if we're not buffering
+ if ((cmd.getFlags() & UpdateCommand.BUFFERING) == 0) {
+ map.put(cmd.getIndexedId(), ptr);
+ }
- // only update our map if we're not buffering
- if ((cmd.getFlags() & UpdateCommand.BUFFERING) == 0) {
- map.put(cmd.getIndexedId(), ptr);
- }
+ if (trace) {
+ log.trace("TLOG: added id " + cmd.getPrintableId() + " to " + tlog + " " + ptr + " map=" + System.identityHashCode(map));
+ }
+
+ } else {
+ // replicate the deleteByQuery logic. See deleteByQuery for comments.
+
+ if (map != null) map.clear();
+ if (prevMap != null) prevMap.clear();
+ if (prevMap2 != null) prevMap2.clear();
+
+ try {
+ RefCounted<SolrIndexSearcher> holder = uhandler.core.openNewSearcher(true, true);
+ holder.decref();
+ } catch (Throwable e) {
+ SolrException.log(log, "Error opening realtime searcher for deleteByQuery", e);
+ }
+
+ if (trace) {
+ log.trace("TLOG: added id " + cmd.getPrintableId() + " to " + tlog + " clearCaches=true");
+ }
- if (trace) {
- log.trace("TLOG: added id " + cmd.getPrintableId() + " to " + tlog + " " + ptr + " map=" + System.identityHashCode(map));
}
+
}
}
+
+
public void delete(DeleteUpdateCommand cmd) {
BytesRef br = cmd.getIndexedId();
@@ -350,7 +402,11 @@ public class UpdateLog implements Plugin
if ((cmd.getFlags() & UpdateCommand.BUFFERING) == 0) {
// given that we just did a delete-by-query, we don't know what documents were
// affected and hence we must purge our caches.
- map.clear();
+ if (map != null) map.clear();
+ if (prevMap != null) prevMap.clear();
+ if (prevMap2 != null) prevMap2.clear();
+
+ trackDeleteByQuery(cmd.getQuery(), cmd.getVersion());
// oldDeletes.clear();
@@ -375,6 +431,75 @@ public class UpdateLog implements Plugin
}
}
+ /** currently for testing only */
+ public void deleteAll() {
+ synchronized (this) {
+
+ try {
+ RefCounted<SolrIndexSearcher> holder = uhandler.core.openNewSearcher(true, true);
+ holder.decref();
+ } catch (Throwable e) {
+ SolrException.log(log, "Error opening realtime searcher for deleteByQuery", e);
+ }
+
+ if (map != null) map.clear();
+ if (prevMap != null) prevMap.clear();
+ if (prevMap2 != null) prevMap2.clear();
+
+ oldDeletes.clear();
+ deleteByQueries.clear();
+ }
+ }
+
+
+ void trackDeleteByQuery(String q, long version) {
+ version = Math.abs(version);
+ DBQ dbq = new DBQ();
+ dbq.q = q;
+ dbq.version = version;
+
+ synchronized (this) {
+ if (deleteByQueries.isEmpty() || deleteByQueries.getFirst().version < version) {
+ // common non-reordered case
+ deleteByQueries.addFirst(dbq);
+ } else {
+ // find correct insertion point
+ ListIterator<DBQ> iter = deleteByQueries.listIterator();
+ iter.next(); // we already checked the first element in the previous "if" clause
+ while (iter.hasNext()) {
+ DBQ oldDBQ = iter.next();
+ if (oldDBQ.version < version) {
+ iter.previous();
+ break;
+ } else if (oldDBQ.version == version && oldDBQ.q.equals(q)) {
+ // a duplicate
+ return;
+ }
+ }
+ iter.add(dbq); // this also handles the case of adding at the end when hasNext() == false
+ }
+
+ if (deleteByQueries.size() > numDeletesByQueryToKeep) {
+ deleteByQueries.removeLast();
+ }
+ }
+ }
+
+ public List<DBQ> getDBQNewer(long version) {
+ synchronized (this) {
+ if (deleteByQueries.isEmpty() || deleteByQueries.getFirst().version < version) {
+ // fast common case
+ return null;
+ }
+
+ List<DBQ> dbqList = new ArrayList<DBQ>();
+ for (DBQ dbq : deleteByQueries) {
+ if (dbq.version <= version) break;
+ dbqList.add(dbq);
+ }
+ return dbqList;
+ }
+ }
private void newMap() {
prevMap2 = prevMap;
@@ -631,12 +756,8 @@ public class UpdateLog implements Plugin
private void ensureLog() {
if (tlog == null) {
- String newLogName = String.format(Locale.ENGLISH, LOG_FILENAME_PATTERN, TLOG_NAME, id);
- try {
- tlog = new TransactionLog(new File(tlogDir, newLogName), globalStrings);
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Can't open new tlog!", e);
- }
+ String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN, TLOG_NAME, id);
+ tlog = new TransactionLog(new File(tlogDir, newLogName), globalStrings);
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Thu Jul 19 15:58:54 2012
@@ -285,7 +285,10 @@ public class DistributedUpdateProcessor
boolean dropCmd = false;
if (!forwardToLeader) {
- dropCmd = versionAdd(cmd);
+ // clone the original doc
+ SolrInputDocument clonedDoc = cmd.solrDoc.deepCopy();
+ dropCmd = versionAdd(cmd, clonedDoc);
+ cmd.solrDoc = clonedDoc;
}
if (dropCmd) {
@@ -393,10 +396,11 @@ public class DistributedUpdateProcessor
/**
* @param cmd
+ * @param cloneDoc needs the version if it's assigned
* @return whether or not to drop this cmd
* @throws IOException
*/
- private boolean versionAdd(AddUpdateCommand cmd) throws IOException {
+ private boolean versionAdd(AddUpdateCommand cmd, SolrInputDocument cloneDoc) throws IOException {
BytesRef idBytes = cmd.getIndexedId();
if (vinfo == null || idBytes == null) {
@@ -443,16 +447,16 @@ public class DistributedUpdateProcessor
// realtime-get to work reliably.
// TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason?
// there may be other reasons in the future for a version on the commands
+
+ boolean checkDeleteByQueries = false;
+
if (versionsStored) {
long bucketVersion = bucket.highest;
if (leaderLogic) {
- boolean updated = getUpdatedDocument(cmd);
- if (updated && versionOnUpdate == -1) {
- versionOnUpdate = 1; // implied "doc must exist" for now...
- }
+ boolean updated = getUpdatedDocument(cmd, versionOnUpdate);
if (versionOnUpdate != 0) {
Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
@@ -469,6 +473,7 @@ public class DistributedUpdateProcessor
long version = vinfo.getNewClock();
cmd.setVersion(version);
cmd.getSolrInputDocument().setField(VersionInfo.VERSION_FIELD, version);
+ cloneDoc.setField(VersionInfo.VERSION_FIELD, version);
bucket.updateHighest(version);
} else {
// The leader forwarded us this update.
@@ -484,7 +489,7 @@ public class DistributedUpdateProcessor
// if we aren't the leader, then we need to check that updates were not re-ordered
if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
// we're OK... this update has a version higher than anything we've seen
- // in this bucket so far, so we know that no reordering has yet occured.
+ // in this bucket so far, so we know that no reordering has yet occurred.
bucket.updateHighest(versionOnUpdate);
} else {
// there have been updates higher than the current update. we need to check
@@ -494,11 +499,16 @@ public class DistributedUpdateProcessor
// This update is a repeat, or was reordered. We need to drop this update.
return true;
}
+
+ // also need to re-apply newer deleteByQuery commands
+ checkDeleteByQueries = true;
}
}
}
+ // TODO: possibly set checkDeleteByQueries as a flag on the command?
doLocalAdd(cmd);
+
} // end synchronized (bucket)
} finally {
vinfo.unlockForUpdate();
@@ -509,7 +519,7 @@ public class DistributedUpdateProcessor
// TODO: may want to switch to using optimistic locking in the future for better concurrency
// that's why this code is here... need to retry in a loop closely around/in versionAdd
- boolean getUpdatedDocument(AddUpdateCommand cmd) throws IOException {
+ boolean getUpdatedDocument(AddUpdateCommand cmd, long versionOnUpdate) throws IOException {
SolrInputDocument sdoc = cmd.getSolrInputDocument();
boolean update = false;
for (SolrInputField sif : sdoc.values()) {
@@ -525,13 +535,17 @@ public class DistributedUpdateProcessor
SolrInputDocument oldDoc = RealTimeGetComponent.getInputDocument(cmd.getReq().getCore(), id);
if (oldDoc == null) {
- // not found... allow this in the future (depending on the details of the update, or if the user explicitly sets it).
- // could also just not change anything here and let the optimistic locking throw the error
- throw new SolrException(ErrorCode.CONFLICT, "Document not found for update. id=" + cmd.getPrintableId());
+ // create a new doc by default if an old one wasn't found
+ if (versionOnUpdate <= 0) {
+ oldDoc = new SolrInputDocument();
+ } else {
+ // could just let the optimistic locking throw the error
+ throw new SolrException(ErrorCode.CONFLICT, "Document not found for update. id=" + cmd.getPrintableId());
+ }
+ } else {
+ oldDoc.remove(VERSION_FIELD);
}
- oldDoc.remove(VERSION_FIELD);
-
for (SolrInputField sif : sdoc.values()) {
Object val = sif.getValue();
if (val instanceof Map) {
@@ -654,7 +668,7 @@ public class DistributedUpdateProcessor
// FROM: we are a replica receiving a DBQ from our leader
// - log + execute the local DBQ
DistribPhase phase =
- DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
+ DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
if (zkEnabled && DistribPhase.NONE == phase) {
boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard
@@ -772,8 +786,6 @@ public class DistributedUpdateProcessor
}
-
- // TODO: need to handle reorders to replicas somehow
// forward to all replicas
if (leaderLogic && replicas != null) {
ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java Thu Jul 19 15:58:54 2012
@@ -21,7 +21,6 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
import org.apache.commons.io.IOUtils;
@@ -69,7 +68,7 @@ public final class HTMLStripFieldUpdateP
Reader in = null;
try {
in = new HTMLStripCharFilter
- (CharReader.get(new StringReader(s.toString())));
+ (new StringReader(s.toString()));
IOUtils.copy(in, result);
return result.toString();
} catch (IOException e) {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java Thu Jul 19 15:58:54 2012
@@ -71,25 +71,26 @@ public final class RegexReplaceProcessor
@Override
public void init(NamedList args) {
- String patternParam = args.remove(PATTERN_PARAM).toString();
+ Object patternParam = args.remove(PATTERN_PARAM);
if(patternParam == null) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Missing required init parameter: " + PATTERN_PARAM);
}
+
try {
- pattern = Pattern.compile(patternParam);
+ pattern = Pattern.compile(patternParam.toString());
} catch (PatternSyntaxException e) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Invalid regex: " + patternParam, e);
}
- String replacementParam = args.remove(REPLACEMENT_PARAM).toString();
+ Object replacementParam = args.remove(REPLACEMENT_PARAM);
if(replacementParam == null) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Missing required init parameter: " + REPLACEMENT_PARAM);
}
- replacement = Matcher.quoteReplacement(replacementParam);
+ replacement = Matcher.quoteReplacement(replacementParam.toString());
super.init(args);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java Thu Jul 19 15:58:54 2012
@@ -21,6 +21,8 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.Locale;
+
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
@@ -213,7 +215,7 @@ public class URLClassifyProcessor extend
}
private String landingPageSuffix(URL url) {
- String path = url.getPath().toLowerCase();
+ String path = url.getPath().toLowerCase(Locale.ROOT);
for(String suffix : landingPageSuffixes) {
if(path.endsWith(suffix)) {
return suffix;
@@ -223,6 +225,6 @@ public class URLClassifyProcessor extend
}
private String getPathWithoutSuffix(URL url) {
- return url.getPath().toLowerCase().replaceFirst(landingPageSuffix(url)+"$", "");
+ return url.getPath().toLowerCase(Locale.ROOT).replaceFirst(landingPageSuffix(url)+"$", "");
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.java Thu Jul 19 15:58:54 2012
@@ -59,7 +59,7 @@ public class UUIDUpdateProcessorFactory
UpdateRequestProcessor next ) {
return new DefaultValueUpdateProcessor(fieldName, next) {
public Object getDefaultValue() {
- return UUID.randomUUID().toString().toLowerCase(Locale.ENGLISH);
+ return UUID.randomUUID().toString().toLowerCase(Locale.ROOT);
}
};
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/DateMathParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/DateMathParser.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/DateMathParser.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/DateMathParser.java Thu Jul 19 15:58:54 2012
@@ -83,8 +83,8 @@ public class DateMathParser {
/** Default TimeZone for DateMath rounding (UTC) */
public static final TimeZone DEFAULT_MATH_TZ = UTC;
- /** Default Locale for DateMath rounding (Locale.US) */
- public static final Locale DEFAULT_MATH_LOCALE = Locale.US;
+ /** Default Locale for DateMath rounding (Locale.ROOT) */
+ public static final Locale DEFAULT_MATH_LOCALE = Locale.ROOT;
/**
* A mapping from (uppercased) String labels idenyifying time units,
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/NumberUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/NumberUtils.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/NumberUtils.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/NumberUtils.java Thu Jul 19 15:58:54 2012
@@ -18,6 +18,7 @@
package org.apache.solr.util;
import java.text.NumberFormat;
+import java.util.Locale;
import org.apache.lucene.util.BytesRef;
@@ -27,7 +28,7 @@ import org.apache.lucene.util.BytesRef;
public class NumberUtils {
public static String readableSize(long size) {
- NumberFormat formatter = NumberFormat.getNumberInstance();
+ NumberFormat formatter = NumberFormat.getNumberInstance(Locale.ROOT);
formatter.setMaximumFractionDigits(2);
if (size / (1024 * 1024 * 1024) > 0) {
return formatter.format(size * 1.0d / (1024 * 1024 * 1024)) + " GB";
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java Thu Jul 19 15:58:54 2012
@@ -218,8 +218,7 @@ public class SimplePostTool {
/**
* Opens the file and posts it's contents to the solrUrl,
- * writes to response to output.
- * @throws UnsupportedEncodingException
+ * writes to response to output.
*/
public void postFile(File file, OutputStream output, String type) {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java Thu Jul 19 15:58:54 2012
@@ -434,9 +434,10 @@ public class SolrPluginUtils {
*
* @param fieldLists - an array of Strings eg. <code>{"fieldOne^2.3", "fieldTwo", fieldThree~5^-0.4}</code>
* @param wordGrams - (0=all words, 2,3 = shingle size)
+ * @param defaultSlop - the default slop for this param
* @return - FieldParams containing the fieldname,boost,slop,and shingle size
*/
- public static List<FieldParams> parseFieldBoostsAndSlop(String[] fieldLists,int wordGrams) {
+ public static List<FieldParams> parseFieldBoostsAndSlop(String[] fieldLists,int wordGrams,int defaultSlop) {
if (null == fieldLists || 0 == fieldLists.length) {
return new ArrayList<FieldParams>();
}
@@ -454,7 +455,7 @@ public class SolrPluginUtils {
String[] fieldAndSlopVsBoost = caratPattern.split(s);
String[] fieldVsSlop = tildePattern.split(fieldAndSlopVsBoost[0]);
String field = fieldVsSlop[0];
- int slop = (2 == fieldVsSlop.length) ? Integer.valueOf(fieldVsSlop[1]) : 0;
+ int slop = (2 == fieldVsSlop.length) ? Integer.valueOf(fieldVsSlop[1]) : defaultSlop;
Float boost = (1 == fieldAndSlopVsBoost.length) ? 1 : Float.valueOf(fieldAndSlopVsBoost[1]);
FieldParams fp = new FieldParams(field,wordGrams,slop,boost);
out.add(fp);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java Thu Jul 19 15:58:54 2012
@@ -96,7 +96,7 @@ public final class SystemIdResolver impl
};
}
- URI resolveRelativeURI(String baseURI, String systemId) throws IOException,URISyntaxException {
+ URI resolveRelativeURI(String baseURI, String systemId) throws URISyntaxException {
URI uri;
// special case for backwards compatibility: if relative systemId starts with "/" (we convert that to an absolute solrres:-URI)
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java Thu Jul 19 15:58:54 2012
@@ -64,7 +64,7 @@ public class TransformerProvider {
}
/** Return a new Transformer, possibly created from our cached Templates object
- * @throws TransformerConfigurationException
+ * @throws IOException
*/
public synchronized Transformer getTransformer(SolrConfig solrConfig, String filename,int cacheLifetimeSeconds) throws IOException {
// For now, the Templates are blindly reloaded once cacheExpires is over.
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml Thu Jul 19 15:58:54 2012
@@ -29,6 +29,6 @@
If 'null' (or absent), cores will not be manageable via request handler
-->
<cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}" hostContext="solr" zkClientTimeout="12000" numShards="${numShards:3}">
- <core name="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}" instanceDir="."/>
+ <core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"/>
</cores>
</solr>
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java Thu Jul 19 15:58:54 2012
@@ -52,6 +52,7 @@ import org.apache.solr.schema.SchemaFiel
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.util.RefCounted;
import org.junit.BeforeClass;
@@ -125,9 +126,19 @@ public class BasicFunctionalityTest exte
// test merge factor picked up
SolrCore core = h.getCore();
- IndexWriter writer = ((DirectUpdateHandler2)core.getUpdateHandler()).getSolrCoreState().getIndexWriter(core);
- assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy)writer.getConfig().getMergePolicy()).getMergeFactor());
-
+ RefCounted<IndexWriter> iw = ((DirectUpdateHandler2) core
+ .getUpdateHandler()).getSolrCoreState().getIndexWriter(core);
+ try {
+ assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy) iw
+ .get().getConfig().getMergePolicy()).getMergeFactor());
+ } finally {
+ iw.decref();
+ }
+ // test stats call
+ NamedList stats = core.getStatistics();
+ assertEquals("collection1", stats.get("coreName"));
+ assertTrue(stats.get("refCount") != null);
+
lrf.args.put(CommonParams.VERSION,"2.2");
assertQ("test query on empty index",
req("qlkciyopsbgzyvkylsjhchghjrdf")
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java Thu Jul 19 15:58:54 2012
@@ -37,7 +37,7 @@ public class MinimalSchemaTest extends S
*/
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solr/conf/solrconfig.xml","solr/conf/schema-minimal.xml");
+ initCore("solr/collection1/conf/solrconfig.xml","solr/collection1/conf/schema-minimal.xml");
/* make sure some misguided soul doesn't inadvertently give us
a uniqueKey field and defeat the point of the tests
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java Thu Jul 19 15:58:54 2012
@@ -17,6 +17,7 @@ package org.apache.solr;
* limitations under the License.
*/
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
@@ -31,6 +32,7 @@ import org.apache.solr.common.util.Named
*
* @since solr 4.0
*/
+@Slow
public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
String t1="a_t";
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedSearch.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestDistributedSearch.java Thu Jul 19 15:58:54 2012
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -43,6 +44,7 @@ import org.apache.solr.common.util.Named
*
* @since solr 1.3
*/
+@Slow
public class TestDistributedSearch extends BaseDistributedSearchTestCase {
String t1="a_t";
@@ -348,7 +350,13 @@ public class TestDistributedSearch exten
downJettys.add(downJetty);
}
- queryPartialResults(upShards, upClients, "q","*:*",ShardParams.SHARDS_INFO,"true",ShardParams.SHARDS_TOLERANT,"true");
+ queryPartialResults(upShards, upClients,
+ "q","*:*",
+ "facet","true",
+ "facet.field",t1,
+ "facet.limit",5,
+ ShardParams.SHARDS_INFO,"true",
+ ShardParams.SHARDS_TOLERANT,"true");
// restart the jettys
for (JettySolrRunner downJetty : downJettys) {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestRandomFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestRandomFaceting.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestRandomFaceting.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestRandomFaceting.java Thu Jul 19 15:58:54 2012
@@ -19,19 +19,16 @@ package org.apache.solr;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util._TestUtil;
-import org.apache.noggit.JSONUtil;
-import org.apache.noggit.ObjectBuilder;
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.util.TestUtils;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.*;
+@Slow
public class TestRandomFaceting extends SolrTestCaseJ4 {
public static final String FOO_STRING_FIELD = "foo_s1";
@@ -80,7 +77,7 @@ public class TestRandomFaceting extends
model = indexDocs(types, model, ndocs);
}
- void deleteSomeDocs() throws Exception {
+ void deleteSomeDocs() {
Random rand = random();
int percent = rand.nextInt(100);
if (model == null) return;
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java Thu Jul 19 15:58:54 2012
@@ -16,6 +16,7 @@
*/
package org.apache.solr;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -24,7 +25,6 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.commons.io.IOUtils;
import java.io.*;
import java.util.Properties;
@@ -37,7 +37,7 @@ import java.util.Properties;
* @since solr 1.4
*/
public class TestSolrCoreProperties extends LuceneTestCase {
- private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "conf" + File.separator;
+ private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
JettySolrRunner solrJetty;
SolrServer client;
@@ -106,8 +106,8 @@ public class TestSolrCoreProperties exte
getClass().getName() + "-" + System.currentTimeMillis());
- dataDir = new File(homeDir, "data");
- confDir = new File(homeDir, "conf");
+ dataDir = new File(homeDir + "/collection1", "data");
+ confDir = new File(homeDir + "/collection1", "conf");
homeDir.mkdirs();
@@ -115,29 +115,15 @@ public class TestSolrCoreProperties exte
confDir.mkdirs();
File f = new File(confDir, "solrconfig.xml");
- copyFile(SolrTestCaseJ4.getFile(getSolrConfigFile()), f);
+ IOUtils.copy(SolrTestCaseJ4.getFile(getSolrConfigFile()), f);
f = new File(confDir, "schema.xml");
- copyFile(SolrTestCaseJ4.getFile(getSchemaFile()), f);
+ IOUtils.copy(SolrTestCaseJ4.getFile(getSchemaFile()), f);
Properties p = new Properties();
p.setProperty("foo.foo1", "f1");
p.setProperty("foo.foo2", "f2");
FileOutputStream fos = new FileOutputStream(confDir + File.separator + "solrcore.properties");
p.store(fos, null);
- fos.close();
- IOUtils.closeQuietly(fos);
-
- }
-
-
- private void copyFile(File src, File dst) throws IOException {
- BufferedReader in = new BufferedReader(new FileReader(src));
- Writer out = new FileWriter(dst);
-
- for (String line = in.readLine(); null != line; line = in.readLine()) {
- out.write(line);
- }
- in.close();
- out.close();
+ IOUtils.close(fos);
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestTrie.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestTrie.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/TestTrie.java Thu Jul 19 15:58:54 2012
@@ -169,11 +169,11 @@ public class TestTrie extends SolrTestCa
assertQ("Range filter tint:[* to *] must match 10 documents", req("q", "*:*", "fq", "tdate:[* TO *]"), "//*[@numFound='10']");
// Test date math syntax
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
format.setTimeZone(TimeZone.getTimeZone("UTC"));
assertU(delQ("*:*"));
- DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.US);
+ DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.ROOT);
String largestDate = "";
for (int i = 0; i < 10; i++) {
// index 10 days starting with today
@@ -220,9 +220,9 @@ public class TestTrie extends SolrTestCa
checkPrecisionSteps("tdate");
// For tdate tests
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
format.setTimeZone(TimeZone.getTimeZone("UTC"));
- DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.US);
+ DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.ROOT);
for (int i = 0; i < 10; i++) {
long l = Integer.MAX_VALUE + i*1L;
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java Thu Jul 19 15:58:54 2012
@@ -38,7 +38,7 @@ import java.util.HashMap;
public class CommonGramsFilterFactoryTest extends BaseTokenStreamTestCase {
public void testInform() throws Exception {
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
assertTrue("loader is null and it shouldn't be", loader != null);
CommonGramsFilterFactory factory = new CommonGramsFilterFactory();
Map<String, String> args = new HashMap<String, String>();
@@ -88,7 +88,7 @@ public class CommonGramsFilterFactoryTes
* If no words are provided, then a set of english default stopwords is used.
*/
public void testDefaults() throws Exception {
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
assertTrue("loader is null and it shouldn't be", loader != null);
CommonGramsFilterFactory factory = new CommonGramsFilterFactory();
factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java Thu Jul 19 15:58:54 2012
@@ -37,7 +37,7 @@ import java.util.HashMap;
public class CommonGramsQueryFilterFactoryTest extends BaseTokenStreamTestCase {
public void testInform() throws Exception {
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
assertTrue("loader is null and it shouldn't be", loader != null);
CommonGramsQueryFilterFactory factory = new CommonGramsQueryFilterFactory();
Map<String, String> args = new HashMap<String, String>();
@@ -87,7 +87,7 @@ public class CommonGramsQueryFilterFacto
* If no words are provided, then a set of english default stopwords is used.
*/
public void testDefaults() throws Exception {
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
assertTrue("loader is null and it shouldn't be", loader != null);
CommonGramsQueryFilterFactory factory = new CommonGramsQueryFilterFactory();
factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java Thu Jul 19 15:58:54 2012
@@ -28,7 +28,6 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.util._TestUtil;
@@ -45,7 +44,7 @@ public class LegacyHTMLStripCharFilterTe
String gold = " this is some text here is a link and " +
"another link . " +
"This is an entity: & plus a <. Here is an &. ";
- LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(html)));
+ LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(new StringReader(html));
StringBuilder builder = new StringBuilder();
int ch = -1;
char [] goldArray = gold.toCharArray();
@@ -63,7 +62,7 @@ public class LegacyHTMLStripCharFilterTe
//Some sanity checks, but not a full-fledged check
public void testHTML() throws Exception {
InputStream stream = getClass().getResourceAsStream("htmlStripReaderTest.html");
- LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(CharReader.get(new InputStreamReader(stream, "UTF-8")));
+ LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(new InputStreamReader(stream, "UTF-8"));
StringBuilder builder = new StringBuilder();
int ch = -1;
while ((ch = reader.read()) != -1){
@@ -83,7 +82,7 @@ public class LegacyHTMLStripCharFilterTe
String gold = "\u0393";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -100,7 +99,7 @@ public class LegacyHTMLStripCharFilterTe
String gold = " <foo> \u00DCbermensch = \u0393 bar \u0393";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -117,7 +116,7 @@ public class LegacyHTMLStripCharFilterTe
String gold = " <junk/> ! @ and â";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -133,7 +132,7 @@ public class LegacyHTMLStripCharFilterTe
String test = "aaa bbb <reserved ccc=\"ddddd\"> eeee </reserved> ffff <reserved ggg=\"hhhh\"/> <other/>";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -150,7 +149,7 @@ public class LegacyHTMLStripCharFilterTe
public void testMalformedHTML() throws Exception {
String test = "a <a hr<ef=aa<a>> </close</a>";
String gold = "a <a hr<ef=aa > </close ";
- Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new LegacyHTMLStripCharFilter(new StringReader(test));
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -199,7 +198,7 @@ public class LegacyHTMLStripCharFilterTe
private void processBuffer(String test, String assertMsg) throws IOException {
// System.out.println("-------------------processBuffer----------");
- Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader
+ Reader reader = new LegacyHTMLStripCharFilter(new BufferedReader(new StringReader(test)));//force the use of BufferedReader
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -216,7 +215,7 @@ public class LegacyHTMLStripCharFilterTe
String test = "<!--- three dashes, still a valid comment ---> ";
String gold = " ";
- Reader reader = new LegacyHTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader
+ Reader reader = new LegacyHTMLStripCharFilter(new BufferedReader(new StringReader(test)));//force the use of BufferedReader
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -231,7 +230,7 @@ public class LegacyHTMLStripCharFilterTe
public void doTestOffsets(String in) throws Exception {
- LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(in))));
+ LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(new BufferedReader(new StringReader(in)));
int ch = 0;
int off = 0; // offset in the reader
int strOff = -1; // offset in the original string
@@ -268,7 +267,7 @@ public class LegacyHTMLStripCharFilterTe
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new LegacyHTMLStripCharFilter(CharReader.get(new BufferedReader(reader)));
+ return new LegacyHTMLStripCharFilter(new BufferedReader(reader));
}
};
@@ -280,7 +279,7 @@ public class LegacyHTMLStripCharFilterTe
int maxNumElements = 10000;
String text = _TestUtil.randomHtmlishString(random(), maxNumElements);
Reader reader
- = new LegacyHTMLStripCharFilter(CharReader.get(new StringReader(text)));
+ = new LegacyHTMLStripCharFilter(new StringReader(text));
while (reader.read() != -1);
}
@@ -315,7 +314,7 @@ public class LegacyHTMLStripCharFilterTe
}
}
Reader reader = new LegacyHTMLStripCharFilter
- (CharReader.get(new StringReader(text.toString())));
+ (new StringReader(text.toString()));
while (reader.read() != -1);
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java Thu Jul 19 15:58:54 2012
@@ -85,7 +85,7 @@ public class SnowballPorterFilterFactory
*/
public void testProtected() throws Exception {
SnowballPorterFilterFactory factory = new SnowballPorterFilterFactory();
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
Map<String,String> args = new HashMap<String,String>();
args.put("protected", "protwords.txt");
args.put("language", "English");
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestArabicFilters.java Thu Jul 19 15:58:54 2012
@@ -23,7 +23,6 @@ import java.util.Collections;
import java.util.Map;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -78,7 +77,7 @@ public class TestArabicFilters extends B
tokenizerFactory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
Map<String, String> args = Collections.emptyMap();
tokenizerFactory.init(args);
- TokenStream stream = tokenizerFactory.create(charfilterFactory.create(CharReader.get(reader)));
+ TokenStream stream = tokenizerFactory.create(charfilterFactory.create(reader));
assertTokenStreamContents(stream, new String[] { "Ù
Û", "Ø®Ùرد" });
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -38,7 +38,7 @@ public class TestDelimitedPayloadTokenFi
args.put(DelimitedPayloadTokenFilterFactory.ENCODER_ATTR, "float");
DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory();
factory.init(args);
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
factory.inform(loader);
TokenStream input = new MockTokenizer(new StringReader("the|0.1 quick|0.1 red|0.1"), MockTokenizer.WHITESPACE, false);
@@ -61,7 +61,7 @@ public class TestDelimitedPayloadTokenFi
args.put(DelimitedPayloadTokenFilterFactory.DELIMITER_ATTR, "*");
DelimitedPayloadTokenFilterFactory factory = new DelimitedPayloadTokenFilterFactory();
factory.init(args);
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
factory.inform(loader);
TokenStream input = new MockTokenizer(new StringReader("the*0.1 quick*0.1 red*0.1"), MockTokenizer.WHITESPACE, false);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -40,7 +40,7 @@ public class TestDictionaryCompoundWordT
Reader reader = new StringReader("I like to play softball");
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
DictionaryCompoundWordTokenFilterFactory factory = new DictionaryCompoundWordTokenFilterFactory();
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
Map<String,String> args = new HashMap<String,String>();
args.put("dictionary", "compoundDictionary.txt");
factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -42,7 +42,7 @@ public class TestElisionFilterFactory ex
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ElisionFilterFactory factory = new ElisionFilterFactory();
factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
Map<String,String> args = new HashMap<String,String>();
args.put("articles", "frenchArticles.txt");
factory.init(args);
@@ -61,7 +61,7 @@ public class TestElisionFilterFactory ex
factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
Map<String, String> args = Collections.emptyMap();
factory.init(args);
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
factory.inform(loader);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "avion" });
@@ -75,7 +75,7 @@ public class TestElisionFilterFactory ex
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ElisionFilterFactory factory = new ElisionFilterFactory();
factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
- ResourceLoader loader = new SolrResourceLoader(null, null);
+ ResourceLoader loader = new SolrResourceLoader("solr/collection1");
Map<String,String> args = new HashMap<String,String>();
args.put("articles", "frenchArticles.txt");
args.put("ignoreCase", "true");