You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/12/21 12:40:14 UTC
svn commit: r1721134 [31/48] - in /lucene/dev/branches/lucene6835: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/lucene/analysis/morfologik/
dev-tools/scripts/ lucene/ lucene/analysis/ lu...
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java Mon Dec 21 11:39:57 2015
@@ -28,9 +28,6 @@ import org.apache.solr.search.SolrIndexS
import java.io.File;
import java.io.IOException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* <p>
@@ -45,7 +42,6 @@ import org.slf4j.LoggerFactory;
* @since solr 1.3
**/
public class IndexBasedSpellChecker extends AbstractLuceneSpellChecker {
- private static final Logger log = LoggerFactory.getLogger(IndexBasedSpellChecker.class);
public static final String THRESHOLD_TOKEN_FREQUENCY = "thresholdTokenFrequency";
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java Mon Dec 21 11:39:57 2015
@@ -16,6 +16,7 @@ package org.apache.solr.spelling;
* limitations under the License.
*/
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -41,7 +42,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SpellCheckCollator {
- private static final Logger LOG = LoggerFactory.getLogger(SpellCheckCollator.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private int maxCollations = 1;
private int maxCollationTries = 0;
private int maxCollationEvaluations = 10000;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java Mon Dec 21 11:39:57 2015
@@ -90,7 +90,7 @@ public class SpellingQueryConverter exte
NMTOKEN = "([" + sb.toString() + "]|" + SURROGATE_PAIR + ")+";
}
- final static String PATTERN = "(?:(?!(" + NMTOKEN + ":|\\d+)))[\\p{L}_\\-0-9]+";
+ final static String PATTERN = "(?:(?!(" + NMTOKEN + ":|[\\^.]\\d+)))[^^.\\s][\\p{L}_\\-0-9]+";
// previous version: Pattern.compile("(?:(?!(\\w+:|\\d+)))\\w+");
protected Pattern QUERY_REGEX = Pattern.compile(PATTERN);
@@ -104,6 +104,8 @@ public class SpellingQueryConverter exte
if (original == null) { // this can happen with q.alt = and no query
return Collections.emptyList();
}
+ boolean mightContainRangeQuery = (original.indexOf('[') != -1 || original.indexOf('{') != -1)
+ && (original.indexOf(']') != -1 || original.indexOf('}') != -1);
Collection<Token> result = new ArrayList<>();
Matcher matcher = QUERY_REGEX.matcher(original);
String nextWord = null;
@@ -123,7 +125,10 @@ public class SpellingQueryConverter exte
if(matcher.find()) {
nextWord = matcher.group(0);
nextStartIndex = matcher.start();
- }
+ }
+ if(mightContainRangeQuery && "TO".equals(word)) {
+ continue;
+ }
if("AND".equals(word) || "OR".equals(word) || "NOT".equals(word)) {
lastBooleanOp = word;
continue;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java Mon Dec 21 11:39:57 2015
@@ -199,8 +199,8 @@ public class WordBreakSolrSpellChecker e
StringBuilder sb = new StringBuilder();
Token[] tokenArr = options.tokens.toArray(new Token[options.tokens.size()]);
+ List<Token> tokenArrWithSeparators = new ArrayList<>(options.tokens.size() + 2);
List<Term> termArr = new ArrayList<>(options.tokens.size() + 2);
-
List<ResultEntry> breakSuggestionList = new ArrayList<>();
List<ResultEntry> noBreakSuggestionList = new ArrayList<>();
boolean lastOneProhibited = false;
@@ -219,6 +219,7 @@ public class WordBreakSolrSpellChecker e
if (i > 0
&& (prohibited != lastOneProhibited || required != lastOneRequired || lastOneprocedesNewBooleanOp)) {
termArr.add(WordBreakSpellChecker.SEPARATOR_TERM);
+ tokenArrWithSeparators.add(null);
}
lastOneProhibited = prohibited;
lastOneRequired = required;
@@ -226,6 +227,7 @@ public class WordBreakSolrSpellChecker e
Term thisTerm = new Term(field, tokenArr[i].toString());
termArr.add(thisTerm);
+ tokenArrWithSeparators.add(tokenArr[i]);
if (breakWords) {
SuggestWord[][] breakSuggestions = wbsp.suggestWordBreaks(thisTerm,
numSuggestions, ir, options.suggestMode, sortMethod);
@@ -269,10 +271,10 @@ public class WordBreakSolrSpellChecker e
if (i > firstTermIndex) {
sb.append(" ");
}
- sb.append(tokenArr[i].toString());
+ sb.append(tokenArrWithSeparators.get(i).toString());
}
- Token token = new Token(sb.toString(), tokenArr[firstTermIndex]
- .startOffset(), tokenArr[lastTermIndex].endOffset());
+ Token token = new Token(sb.toString(), tokenArrWithSeparators.get(firstTermIndex)
+ .startOffset(), tokenArrWithSeparators.get(lastTermIndex).endOffset());
combineSuggestionList.add(new ResultEntry(token, cs.suggestion.string,
cs.suggestion.freq));
}
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java Mon Dec 21 11:39:57 2015
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -56,7 +57,7 @@ import static org.apache.solr.spelling.s
* {@link Dictionary}
* */
public class SolrSuggester implements Accountable {
- private static final Logger LOG = LoggerFactory.getLogger(SolrSuggester.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/** Name used when an unnamed suggester config is passed */
public static final String DEFAULT_DICT_NAME = "default";
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java Mon Dec 21 11:39:57 2015
@@ -24,6 +24,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
+import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
@@ -54,7 +55,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Suggester extends SolrSpellChecker {
- private static final Logger LOG = LoggerFactory.getLogger(Suggester.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/** Location of the source data - either a path to a file, or null for the
* current IndexReader.
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookupFactory.java Mon Dec 21 11:39:57 2015
@@ -1,5 +1,7 @@
package org.apache.solr.spelling.suggest.jaspell;
+import java.lang.invoke.MethodHandles;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -30,7 +32,7 @@ import org.slf4j.LoggerFactory;
* <b>Note:</b> This Suggester is not very RAM efficient.
*/
public class JaspellLookupFactory extends LookupFactory {
- private static final Logger LOG = LoggerFactory.getLogger(JaspellLookup.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String FILENAME = "jaspell.dat";
@Override
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java Mon Dec 21 11:39:57 2015
@@ -20,6 +20,7 @@ package org.apache.solr.store.blockcache
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Set;
import org.apache.lucene.index.IndexFileNames;
@@ -37,7 +38,7 @@ import org.slf4j.LoggerFactory;
* @lucene.experimental
*/
public class BlockDirectory extends FilterDirectory {
- public static Logger LOG = LoggerFactory.getLogger(BlockDirectory.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
public static final long BLOCK_MOD = 0x1FFF;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java Mon Dec 21 11:39:57 2015
@@ -95,7 +95,7 @@ public abstract class CustomBufferedInde
if (bufferLength < len) {
// Throw an exception when refill() could not read len bytes:
System.arraycopy(buffer, 0, b, offset, bufferLength);
- throw new IOException("read past EOF");
+ throw new EOFException("read past EOF: " + this);
} else {
System.arraycopy(buffer, 0, b, offset, len);
bufferPosition = len;
@@ -109,7 +109,7 @@ public abstract class CustomBufferedInde
// here, because there's no need to reread what we
// had in the buffer.
long after = bufferStart + bufferPosition + len;
- if (after > length()) throw new IOException("read past EOF");
+ if (after > length()) throw new EOFException("read past EOF: " + this);
readInternal(b, offset, len);
bufferStart = after;
bufferPosition = 0;
@@ -183,7 +183,7 @@ public abstract class CustomBufferedInde
if (end > length()) // don't read past EOF
end = length();
int newLength = (int) (end - start);
- if (newLength <= 0) throw new EOFException("read past EOF");
+ if (newLength <= 0) throw new EOFException("read past EOF: " + this);
if (buffer == null) {
buffer = store.takeBuffer(bufferSize);
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@ package org.apache.solr.store.hdfs;
*/
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -41,7 +42,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HdfsDirectory extends BaseDirectory {
- public static Logger LOG = LoggerFactory.getLogger(HdfsDirectory.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final int BUFFER_SIZE = 8192;
@@ -194,8 +195,7 @@ public class HdfsDirectory extends BaseD
}
static class HdfsIndexInput extends CustomBufferedIndexInput {
- public static Logger LOG = LoggerFactory
- .getLogger(HdfsIndexInput.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Path path;
private final FSDataInputStream inputStream;
@@ -267,4 +267,9 @@ public class HdfsDirectory extends BaseD
}
return this.hdfsDirPath.equals(((HdfsDirectory) obj).hdfsDirPath);
}
+
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName() + "@" + hdfsDirPath + " lockFactory=" + lockFactory;
+ }
}
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java Mon Dec 21 11:39:57 2015
@@ -19,6 +19,7 @@ package org.apache.solr.store.hdfs;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory;
*/
public class HdfsFileReader extends DataInput {
- public static Logger LOG = LoggerFactory.getLogger(HdfsFileReader.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Path path;
private FSDataInputStream inputStream;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@ package org.apache.solr.store.hdfs;
*/
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.util.Arrays;
import java.util.Iterator;
@@ -44,7 +45,7 @@ public class HdfsLocalityReporter implem
public static final String LOCALITY_BLOCKS_LOCAL = "locality.blocks.local";
public static final String LOCALITY_BLOCKS_RATIO = "locality.blocks.ratio";
- public static final Logger logger = LoggerFactory.getLogger(HdfsLocalityReporter.class);
+ private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private String hostname;
private final ConcurrentMap<HdfsDirectory,ConcurrentMap<FileStatus,BlockLocation[]>> cache;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@ package org.apache.solr.store.hdfs;
*/
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -35,7 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HdfsLockFactory extends LockFactory {
- public static Logger log = LoggerFactory.getLogger(HdfsLockFactory.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final HdfsLockFactory INSTANCE = new HdfsLockFactory();
@@ -88,17 +89,17 @@ public class HdfsLockFactory extends Loc
}
}
- return new HdfsLock(fs, lockFile);
+ return new HdfsLock(conf, lockFile);
}
private static final class HdfsLock extends Lock {
-
- private final FileSystem fs;
+
+ private final Configuration conf;
private final Path lockFile;
private volatile boolean closed;
- HdfsLock(FileSystem fs, Path lockFile) {
- this.fs = fs;
+ HdfsLock(Configuration conf, Path lockFile) {
+ this.conf = conf;
this.lockFile = lockFile;
}
@@ -107,6 +108,7 @@ public class HdfsLockFactory extends Loc
if (closed) {
return;
}
+ final FileSystem fs = FileSystem.get(lockFile.toUri(), conf);
try {
if (fs.exists(lockFile) && !fs.delete(lockFile, false)) {
throw new LockReleaseFailedException("failed to delete: " + lockFile);
@@ -120,5 +122,10 @@ public class HdfsLockFactory extends Loc
public void ensureValid() throws IOException {
// no idea how to implement this on HDFS
}
+
+ @Override
+ public String toString() {
+ return "HdfsLock(lockFile=" + lockFile + ")";
+ }
}
}
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java Mon Dec 21 11:39:57 2015
@@ -20,6 +20,7 @@ package org.apache.solr.update;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.lang.invoke.MethodHandles;
import java.nio.channels.Channels;
import java.nio.file.Files;
import java.util.Collection;
@@ -28,6 +29,8 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.util.FastOutputStream;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Extends {@link org.apache.solr.update.TransactionLog} to:
@@ -44,10 +47,14 @@ public class CdcrTransactionLog extends
private boolean isReplaying;
long startVersion; // (absolute) version of the first element of this transaction log
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static boolean debug = log.isDebugEnabled();
+
CdcrTransactionLog(File tlogFile, Collection<String> globalStrings) {
super(tlogFile, globalStrings);
// The starting version number will be used to seek more efficiently tlogs
+ // and to filter out tlog files during replication (in ReplicationHandler#getTlogFileList)
String filename = tlogFile.getName();
startVersion = Math.abs(Long.parseLong(filename.substring(filename.lastIndexOf('.') + 1)));
@@ -187,7 +194,7 @@ public class CdcrTransactionLog extends
}
@Override
- protected void close() {
+ public void close() {
try {
if (debug) {
log.debug("Closing tlog" + this);
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java Mon Dec 21 11:39:57 2015
@@ -19,7 +19,9 @@ package org.apache.solr.update;
import java.io.File;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
@@ -27,8 +29,15 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.processor.DistributedUpdateProcessor;
+import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +63,8 @@ public class CdcrUpdateLog extends Updat
public static String LOG_FILENAME_PATTERN = "%s.%019d.%1d";
- protected static Logger log = LoggerFactory.getLogger(CdcrUpdateLog.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static boolean debug = log.isDebugEnabled();
@Override
public void init(UpdateHandler uhandler, SolrCore core) {
@@ -239,26 +249,223 @@ public class CdcrUpdateLog extends Updat
}
/**
- * expert: Reset the update log before initialisation. This is needed by the IndexFetcher during a
+ * expert: Reset the update log before initialisation. This is called by
+ * {@link org.apache.solr.handler.IndexFetcher#moveTlogFiles(File)} during a
* a Recovery operation in order to re-initialise the UpdateLog with a new set of tlog files.
+ * @see #initForRecovery(File, long)
*/
- public void reset() {
- synchronized (this) {
+ public BufferedUpdates resetForRecovery() {
+ synchronized (this) { // since we blocked updates in IndexFetcher, this synchronization shouldn't strictly be necessary.
+ // If we are buffering, we need to return the related information to the index fetcher
+ // for properly initialising the new update log - SOLR-8263
+ BufferedUpdates bufferedUpdates = new BufferedUpdates();
+ if (state == State.BUFFERING && tlog != null) {
+ bufferedUpdates.tlog = tlog.tlogFile; // file to keep
+ bufferedUpdates.offset = this.recoveryInfo.positionOfStart;
+ }
+
// Close readers
- for (CdcrLogReader reader : new ArrayList<>(logPointers.keySet())) {
+ for (CdcrLogReader reader : logPointers.keySet()) {
reader.close();
}
+ logPointers.clear();
// Close and clear logs
+ doClose(prevTlog);
+ doClose(tlog);
+
for (TransactionLog log : logs) {
- log.deleteOnClose = false;
- log.decref();
- log.forceClose();
+ if (log == prevTlog || log == tlog) continue;
+ doClose(log);
}
+
logs.clear();
+ newestLogsOnStartup.clear();
+ tlog = prevTlog = null;
+ prevMapLog = prevMapLog2 = null;
+
+ map.clear();
+ if (prevMap != null) prevMap.clear();
+ if (prevMap2 != null) prevMap2.clear();
+
+ tlogFiles = null;
+ numOldRecords = 0;
+
+ oldDeletes.clear();
+ deleteByQueries.clear();
+
+ return bufferedUpdates;
+ }
+ }
+
+ public static class BufferedUpdates {
+ public File tlog;
+ public long offset;
+ }
+
+ /**
+ * <p>
+ * expert: Initialise the update log with a tlog file containing buffered updates. This is called by
+ * {@link org.apache.solr.handler.IndexFetcher#moveTlogFiles(File)} during a Recovery operation.
+ * This is mainly a copy of the original {@link UpdateLog#init(UpdateHandler, SolrCore)} method, but modified
+ * to:
+ * <ul>
+ * <li>preserve the same {@link VersionInfo} instance in order to not "unblock" updates, since the
+ * {@link org.apache.solr.handler.IndexFetcher#moveTlogFiles(File)} acquired a write lock from this instance.</li>
+ * <li>copy the buffered updates.</li>
+ * </ul>
+ * @see #resetForRecovery()
+ */
+ public void initForRecovery(File bufferedTlog, long offset) {
+ tlogFiles = getLogList(tlogDir);
+ id = getLastLogId() + 1; // add 1 since we will create a new log for the next update
+
+ if (debug) {
+ log.debug("UpdateHandler init: tlogDir=" + tlogDir + ", existing tlogs=" + Arrays.asList(tlogFiles) + ", next id=" + id);
+ }
+
+ TransactionLog oldLog = null;
+ for (String oldLogName : tlogFiles) {
+ File f = new File(tlogDir, oldLogName);
+ try {
+ oldLog = newTransactionLog(f, null, true);
+ addOldLog(oldLog, false); // don't remove old logs on startup since more than one may be uncapped.
+ } catch (Exception e) {
+ SolrException.log(log, "Failure to open existing log file (non fatal) " + f, e);
+ deleteFile(f);
+ }
+ }
+
+ // Record first two logs (oldest first) at startup for potential tlog recovery.
+ // It's possible that at abnormal close both "tlog" and "prevTlog" were uncapped.
+ for (TransactionLog ll : logs) {
+ newestLogsOnStartup.addFirst(ll);
+ if (newestLogsOnStartup.size() >= 2) break;
+ }
+
+ // TODO: these startingVersions assume that we successfully recover from all non-complete tlogs.
+ UpdateLog.RecentUpdates startingUpdates = getRecentUpdates();
+ long latestVersion = startingUpdates.getMaxRecentVersion();
+ try {
+ startingVersions = startingUpdates.getVersions(numRecordsToKeep);
+ startingOperation = startingUpdates.getLatestOperation();
+
+ // populate recent deletes list (since we can't get that info from the index)
+ for (int i=startingUpdates.deleteList.size()-1; i>=0; i--) {
+ DeleteUpdate du = startingUpdates.deleteList.get(i);
+ oldDeletes.put(new BytesRef(du.id), new LogPtr(-1,du.version));
+ }
+
+ // populate recent deleteByQuery commands
+ for (int i=startingUpdates.deleteByQueryList.size()-1; i>=0; i--) {
+ Update update = startingUpdates.deleteByQueryList.get(i);
+ List<Object> dbq = (List<Object>) update.log.lookup(update.pointer);
+ long version = (Long) dbq.get(1);
+ String q = (String) dbq.get(2);
+ trackDeleteByQuery(q, version);
+ }
+
+ } finally {
+ startingUpdates.close();
+ }
+
+ // Copy buffered updates
+ if (bufferedTlog != null) {
+ this.copyBufferedUpdates(bufferedTlog, offset, latestVersion);
+ }
+ }
+
+ /**
+ * <p>
+ * Read the entries from the given tlog file and replay them as buffered updates.
+ * The buffered tlog that we are trying to copy might contain duplicate operations with the
+ * current update log. During the tlog replication process, the replica might buffer update operations
+ * that will be present also in the tlog files downloaded from the leader. In order to remove these
+ * duplicates, it will skip any operations with a version inferior to the latest know version.
+ */
+ private void copyBufferedUpdates(File tlogSrc, long offsetSrc, long latestVersion) {
+ recoveryInfo = new RecoveryInfo();
+ recoveryInfo.positionOfStart = tlog == null ? 0 : tlog.snapshot();
+ state = State.BUFFERING;
+ operationFlags |= FLAG_GAP;
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM, DistributedUpdateProcessor.DistribPhase.FROMLEADER.toString());
+ SolrQueryRequest req = new LocalSolrQueryRequest(uhandler.core, params);
+
+ CdcrTransactionLog src = new CdcrTransactionLog(tlogSrc, null, true);
+ TransactionLog.LogReader tlogReader = src.getReader(offsetSrc);
+ try {
+ int operationAndFlags = 0;
+ for (; ; ) {
+ Object o = tlogReader.next();
+ if (o == null) break; // we reached the end of the tlog
+ // should currently be a List<Oper,Ver,Doc/Id>
+ List entry = (List) o;
+ operationAndFlags = (Integer) entry.get(0);
+ int oper = operationAndFlags & OPERATION_MASK;
+ long version = (Long) entry.get(1);
+ if (version <= latestVersion) {
+ // probably a buffered update that is also present in a tlog file coming from the leader,
+ // skip it.
+ log.debug("Dropping buffered operation - version {} < {}", version, latestVersion);
+ continue;
+ }
+
+ switch (oper) {
+ case UpdateLog.ADD: {
+ SolrInputDocument sdoc = (SolrInputDocument) entry.get(entry.size() - 1);
+ AddUpdateCommand cmd = new AddUpdateCommand(req);
+ cmd.solrDoc = sdoc;
+ cmd.setVersion(version);
+ cmd.setFlags(UpdateCommand.BUFFERING);
+ this.add(cmd);
+ break;
+ }
+ case UpdateLog.DELETE: {
+ byte[] idBytes = (byte[]) entry.get(2);
+ DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
+ cmd.setIndexedId(new BytesRef(idBytes));
+ cmd.setVersion(version);
+ cmd.setFlags(UpdateCommand.BUFFERING);
+ this.delete(cmd);
+ break;
+ }
+
+ case UpdateLog.DELETE_BY_QUERY: {
+ String query = (String) entry.get(2);
+ DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
+ cmd.query = query;
+ cmd.setVersion(version);
+ cmd.setFlags(UpdateCommand.BUFFERING);
+ this.deleteByQuery(cmd);
+ break;
+ }
+
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid Operation! " + oper);
+ }
+
+ }
+ }
+ catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to copy buffered updates", e);
+ }
+ finally {
+ try {
+ tlogReader.close();
+ }
+ finally {
+ this.doClose(src);
+ }
+ }
+ }
- // reset lastDataDir for #init()
- lastDataDir = null;
+ private void doClose(TransactionLog theLog) {
+ if (theLog != null) {
+ theLog.deleteOnClose = false;
+ theLog.decref();
+ theLog.forceClose();
}
}
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CommitTracker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CommitTracker.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CommitTracker.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/CommitTracker.java Mon Dec 21 11:39:57 2015
@@ -1,5 +1,7 @@
package org.apache.solr.update;
+import java.lang.invoke.MethodHandles;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -44,7 +46,7 @@ import org.slf4j.LoggerFactory;
* Public for tests.
*/
public final class CommitTracker implements Runnable {
- protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// scheduler delay for maxDoc-triggered autocommits
public final int DOC_COMMIT_DELAY_MS = 1;
@@ -80,7 +82,7 @@ public final class CommitTracker impleme
this.softCommit = softCommit;
this.openSearcher = openSearcher;
- SolrCore.log.info(name + " AutoCommit: " + this);
+ log.info(name + " AutoCommit: " + this);
}
public boolean getOpenSearcher() {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Mon Dec 21 11:39:57 2015
@@ -18,14 +18,19 @@ package org.apache.solr.update;
*/
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.lucene.index.IndexWriter;
-import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.cloud.ActionThrottle;
+import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.core.CoreContainer;
@@ -38,15 +43,17 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class DefaultSolrCoreState extends SolrCoreState implements RecoveryStrategy.RecoveryListener {
- public static Logger log = LoggerFactory.getLogger(DefaultSolrCoreState.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final boolean SKIP_AUTO_RECOVERY = Boolean.getBoolean("solrcloud.skip.autorecovery");
- private final Object recoveryLock = new Object();
+ private final ReentrantLock recoveryLock = new ReentrantLock();
private final ActionThrottle recoveryThrottle = new ActionThrottle("recovery", 10000);
private final ActionThrottle leaderThrottle = new ActionThrottle("leader", 5000);
+
+ private final AtomicInteger recoveryWaiting = new AtomicInteger();
// Use the readLock to retrieve the current IndexWriter (may be lazily opened)
// Use the writeLock for changing index writers
@@ -55,8 +62,8 @@ public final class DefaultSolrCoreState
private SolrIndexWriter indexWriter = null;
private DirectoryFactory directoryFactory;
- private volatile boolean recoveryRunning;
- private RecoveryStrategy recoveryStrat;
+ private volatile RecoveryStrategy recoveryStrat;
+ private volatile Future future;
private volatile boolean lastReplicationSuccess = true;
// will we attempt recovery as if we just started up (i.e. use starting versions rather than recent versions for peersync
@@ -240,72 +247,97 @@ public final class DefaultSolrCoreState
@Override
public void doRecovery(CoreContainer cc, CoreDescriptor cd) {
- MDCLoggingContext.setCoreDescriptor(cd);
- try {
- if (SKIP_AUTO_RECOVERY) {
- log.warn("Skipping recovery according to sys prop solrcloud.skip.autorecovery");
- return;
- }
-
- // check before we grab the lock
- if (cc.isShutDown()) {
- log.warn("Skipping recovery because Solr is shutdown");
- return;
- }
-
- synchronized (recoveryLock) {
- // to be air tight we must also check after lock
- if (cc.isShutDown()) {
- log.warn("Skipping recovery because Solr is shutdown");
- return;
- }
- log.info("Running recovery - first canceling any ongoing recovery");
- cancelRecovery();
-
- while (recoveryRunning) {
- try {
- recoveryLock.wait(1000);
- } catch (InterruptedException e) {
-
+
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ MDCLoggingContext.setCoreDescriptor(cd);
+ try {
+ if (SKIP_AUTO_RECOVERY) {
+ log.warn("Skipping recovery according to sys prop solrcloud.skip.autorecovery");
+ return;
}
- // check again for those that were waiting
+
+ // check before we grab the lock
if (cc.isShutDown()) {
log.warn("Skipping recovery because Solr is shutdown");
return;
}
- if (closed) return;
+
+ // if we can't get the lock, another recovery is running
+ // we check to see if there is already one waiting to go
+ // after the current one, and if there is, bail
+ boolean locked = recoveryLock.tryLock();
+ try {
+ if (!locked) {
+ if (recoveryWaiting.get() > 0) {
+ return;
+ }
+ recoveryWaiting.incrementAndGet();
+ } else {
+ recoveryWaiting.incrementAndGet();
+ cancelRecovery();
+ }
+
+ recoveryLock.lock();
+ try {
+ recoveryWaiting.decrementAndGet();
+
+ // to be air tight we must also check after lock
+ if (cc.isShutDown()) {
+ log.warn("Skipping recovery because Solr is shutdown");
+ return;
+ }
+ log.info("Running recovery");
+
+ recoveryThrottle.minimumWaitBetweenActions();
+ recoveryThrottle.markAttemptingAction();
+
+ recoveryStrat = new RecoveryStrategy(cc, cd, DefaultSolrCoreState.this);
+ recoveryStrat.setRecoveringAfterStartup(recoveringAfterStartup);
+ future = cc.getUpdateShardHandler().getRecoveryExecutor().submit(recoveryStrat);
+ try {
+ future.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (ExecutionException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ } finally {
+ recoveryLock.unlock();
+ }
+ } finally {
+ if (locked) recoveryLock.unlock();
+ }
+ } finally {
+ MDCLoggingContext.clear();
}
-
- recoveryThrottle.minimumWaitBetweenActions();
- recoveryThrottle.markAttemptingAction();
-
- recoveryStrat = new RecoveryStrategy(cc, cd, this);
- recoveryStrat.setRecoveringAfterStartup(recoveringAfterStartup);
- recoveryStrat.start();
- recoveryRunning = true;
}
- } finally {
- MDCLoggingContext.clear();
+ };
+ try {
+ // we make recovery requests async - that async request may
+ // have to 'wait in line' a bit or bail if a recovery is
+ // already queued up - the recovery execution itself is run
+ // in another thread on another 'recovery' executor.
+ // The update executor is interrupted on shutdown and should
+ // not do disk IO.
+ // The recovery executor is not interrupted on shutdown.
+ //
+ // avoid deadlock: we can't use the recovery executor here
+ cc.getUpdateShardHandler().getUpdateExecutor().submit(thread);
+ } catch (RejectedExecutionException e) {
+ // fine, we are shutting down
}
}
@Override
public void cancelRecovery() {
- synchronized (recoveryLock) {
- if (recoveryStrat != null && recoveryRunning) {
+ if (recoveryStrat != null) {
+ try {
recoveryStrat.close();
- while (true) {
- try {
- recoveryStrat.join();
- } catch (InterruptedException e) {
- // not interruptible - keep waiting
- continue;
- }
- break;
- }
-
- recoveryRunning = false;
- recoveryLock.notifyAll();
+ } catch (NullPointerException e) {
+ // okay
}
}
}
@@ -314,14 +346,11 @@ public final class DefaultSolrCoreState
@Override
public void recovered() {
recoveringAfterStartup = false; // once we have successfully recovered, we no longer need to act as if we are recovering after startup
- recoveryRunning = false;
}
/** called from recoveryStrat on a failed recovery */
@Override
- public void failed() {
- recoveryRunning = false;
- }
+ public void failed() {}
@Override
public synchronized void close(IndexWriterCloser closer) {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,18 @@
package org.apache.solr.update;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
@@ -42,7 +54,6 @@ import org.apache.solr.request.LocalSolr
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.FunctionRangeQuery;
import org.apache.solr.search.QParser;
@@ -51,17 +62,9 @@ import org.apache.solr.search.SolrIndexS
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.function.ValueSourceRangeFilter;
import org.apache.solr.util.RefCounted;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicLong;
+import org.apache.solr.util.TestInjection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* <code>DirectUpdateHandler2</code> implements an UpdateHandler where documents are added
@@ -95,6 +98,8 @@ public class DirectUpdateHandler2 extend
protected boolean commitWithinSoftCommit;
protected boolean indexWriterCloseWaitsForMerges;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public DirectUpdateHandler2(SolrCore core) {
super(core);
@@ -143,7 +148,7 @@ public class DirectUpdateHandler2 extend
}
private void deleteAll() throws IOException {
- SolrCore.log.info(core.getLogId() + "REMOVING ALL DOCUMENTS FROM INDEX");
+ log.info(core.getLogId() + "REMOVING ALL DOCUMENTS FROM INDEX");
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
iw.get().deleteAll();
@@ -735,11 +740,14 @@ public class DirectUpdateHandler2 extend
}
- public static boolean commitOnClose = true; // TODO: make this a real config option?
+ public static boolean commitOnClose = true; // TODO: make this a real config option or move it to TestInjection
// IndexWriterCloser interface method - called from solrCoreState.decref(this)
@Override
public void closeWriter(IndexWriter writer) throws IOException {
+
+ assert TestInjection.injectNonGracefullClose(core.getCoreDescriptor().getCoreContainer());
+
boolean clearRequestInfo = false;
solrCoreState.getCommitLock().lock();
try {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@
package org.apache.solr.update;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -57,7 +58,9 @@ import org.slf4j.LoggerFactory;
*
*/
public class HdfsTransactionLog extends TransactionLog {
- public static Logger log = LoggerFactory.getLogger(HdfsTransactionLog.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static boolean debug = log.isDebugEnabled();
+ private static boolean trace = log.isTraceEnabled();
Path tlogFile;
@@ -301,7 +304,7 @@ public class HdfsTransactionLog extends
}
@Override
- protected void close() {
+ public void close() {
try {
if (debug) {
log.debug("Closing tlog" + this);
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java Mon Dec 21 11:39:57 2015
@@ -19,6 +19,7 @@ package org.apache.solr.update;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -38,6 +39,8 @@ import org.apache.solr.common.util.IOUti
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.util.HdfsUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** @lucene.experimental */
public class HdfsUpdateLog extends UpdateLog {
@@ -47,6 +50,9 @@ public class HdfsUpdateLog extends Updat
private volatile Path tlogDir;
private final String confDir;
private Integer tlogDfsReplication;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static boolean debug = log.isDebugEnabled();
// used internally by tests to track total count of failed tran log loads in init
public static AtomicLong INIT_FAILED_LOGS_COUNT = new AtomicLong();
@@ -101,7 +107,7 @@ public class HdfsUpdateLog extends Updat
maxNumLogsToKeep = objToInt(info.initArgs.get("maxNumLogsToKeep"), 10);
tlogDfsReplication = (Integer) info.initArgs.get( "tlogDfsReplication");
- if (tlogDfsReplication == null) tlogDfsReplication = 1;
+ if (tlogDfsReplication == null) tlogDfsReplication = 3;
log.info("Initializing HdfsUpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={} tlogDfsReplication={}",
dataDir, defaultSyncLevel, numRecordsToKeep, maxNumLogsToKeep, tlogDfsReplication);
@@ -235,18 +241,17 @@ public class HdfsUpdateLog extends Updat
// TODO: these startingVersions assume that we successfully recover from all
// non-complete tlogs.
- HdfsUpdateLog.RecentUpdates startingUpdates = getRecentUpdates();
- try {
+ try (RecentUpdates startingUpdates = getRecentUpdates()) {
startingVersions = startingUpdates.getVersions(getNumRecordsToKeep());
startingOperation = startingUpdates.getLatestOperation();
-
+
// populate recent deletes list (since we can't get that info from the
// index)
for (int i = startingUpdates.deleteList.size() - 1; i >= 0; i--) {
DeleteUpdate du = startingUpdates.deleteList.get(i);
oldDeletes.put(new BytesRef(du.id), new LogPtr(-1, du.version));
}
-
+
// populate recent deleteByQuery commands
for (int i = startingUpdates.deleteByQueryList.size() - 1; i >= 0; i--) {
Update update = startingUpdates.deleteByQueryList.get(i);
@@ -255,9 +260,7 @@ public class HdfsUpdateLog extends Updat
String q = (String) dbq.get(2);
trackDeleteByQuery(q, version);
}
-
- } finally {
- startingUpdates.close();
+
}
}
@@ -348,6 +351,12 @@ public class HdfsUpdateLog extends Updat
}
}
+ @Override
+ public void preSoftCommit(CommitUpdateCommand cmd) {
+ debug = log.isDebugEnabled();
+ super.preSoftCommit(cmd);
+ }
+
public String[] getLogList(Path tlogDir) throws FileNotFoundException, IOException {
final String prefix = TLOG_NAME+'.';
FileStatus[] files = fs.listStatus(tlogDir, new PathFilter() {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/LoggingInfoStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/LoggingInfoStream.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/LoggingInfoStream.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/LoggingInfoStream.java Mon Dec 21 11:39:57 2015
@@ -22,12 +22,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
/**
* An {@link InfoStream} implementation which passes messages on to Solr's logging.
*/
public class LoggingInfoStream extends InfoStream {
- public static final Logger log = LoggerFactory.getLogger(LoggingInfoStream.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Override
public void message(String component, String message) {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/PeerSync.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/PeerSync.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/PeerSync.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/PeerSync.java Mon Dec 21 11:39:57 2015
@@ -17,10 +17,8 @@
package org.apache.solr.update;
-import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.net.ConnectException;
import java.net.SocketException;
import java.util.ArrayList;
@@ -55,10 +53,13 @@ import org.apache.solr.update.processor.
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
/** @lucene.experimental */
public class PeerSync {
- public static Logger log = LoggerFactory.getLogger(PeerSync.class);
- public boolean debug = log.isDebugEnabled();
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static boolean debug = log.isDebugEnabled();
private List<String> replicas;
private int nUpdates;
@@ -69,7 +70,6 @@ public class PeerSync {
private HttpShardHandlerFactory shardHandlerFactory;
private ShardHandler shardHandler;
- private UpdateLog.RecentUpdates recentUpdates;
private List<Long> startingVersions;
private List<Long> ourUpdates;
@@ -182,13 +182,6 @@ public class PeerSync {
try {
log.info(msg() + "START replicas=" + replicas + " nUpdates=" + nUpdates);
- // TODO: does it ever make sense to allow sync when buffering or applying buffered? Someone might request that we
- // do it...
- if (!(ulog.getState() == UpdateLog.State.ACTIVE || ulog.getState() == UpdateLog.State.REPLAYING)) {
- log.error(msg() + "ERROR, update log not in ACTIVE or REPLAY state. " + ulog);
- // return false;
- }
-
if (debug) {
if (startingVersions != null) {
log.debug(msg() + "startingVersions=" + startingVersions.size() + " " + startingVersions);
@@ -202,12 +195,9 @@ public class PeerSync {
for (String replica : replicas) {
requestVersions(replica);
}
-
- recentUpdates = ulog.getRecentUpdates();
- try {
+
+ try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
ourUpdates = recentUpdates.getVersions(nUpdates);
- } finally {
- recentUpdates.close();
}
Collections.sort(ourUpdates, absComparator);
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Mon Dec 21 11:39:57 2015
@@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
import java.net.ConnectException;
import java.util.ArrayList;
import java.util.Collections;
@@ -52,7 +53,7 @@ import java.util.concurrent.Future;
public class SolrCmdDistributor {
private static final int MAX_RETRIES_ON_FORWARD = 25;
- public static Logger log = LoggerFactory.getLogger(SolrCmdDistributor.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private StreamingSolrClients clients;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCoreState.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCoreState.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrCoreState.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@ package org.apache.solr.update;
*/
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.concurrent.locks.Lock;
import org.apache.lucene.index.IndexWriter;
@@ -36,7 +37,7 @@ import org.slf4j.LoggerFactory;
*
*/
public abstract class SolrCoreState {
- public static Logger log = LoggerFactory.getLogger(SolrCoreState.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected boolean closed = false;
private final Object updateLock = new Object();
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Mon Dec 21 11:39:57 2015
@@ -17,6 +17,7 @@
package org.apache.solr.update;
+import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
@@ -44,7 +45,7 @@ import static org.apache.solr.core.Confi
* defined in the <indexConfig> section of solrconfig.xml
*/
public class SolrIndexConfig implements MapSerializable {
- public static final Logger log = LoggerFactory.getLogger(SolrIndexConfig.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
final String defaultMergePolicyClassName;
public static final String DEFAULT_MERGE_SCHEDULER_CLASSNAME = ConcurrentMergeScheduler.class.getName();
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@
package org.apache.solr.update;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.List;
@@ -49,7 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SolrIndexSplitter {
- public static Logger log = LoggerFactory.getLogger(SolrIndexSplitter.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
SolrIndexSearcher searcher;
SchemaField field;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@
package org.apache.solr.update;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.Codec;
@@ -41,7 +42,7 @@ import org.slf4j.LoggerFactory;
*/
public class SolrIndexWriter extends IndexWriter {
- private static Logger log = LoggerFactory.getLogger(SolrIndexWriter.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// These should *only* be used for debugging or monitoring purposes
public static final AtomicLong numOpens = new AtomicLong();
public static final AtomicLong numCloses = new AtomicLong();
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java Mon Dec 21 11:39:57 2015
@@ -30,6 +30,7 @@ import org.apache.solr.update.processor.
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -40,7 +41,7 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
public class StreamingSolrClients {
- public static Logger log = LoggerFactory.getLogger(StreamingSolrClients.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private HttpClient httpClient;
@@ -67,6 +68,9 @@ public class StreamingSolrClients {
String url = getFullUrl(req.node.getUrl());
ConcurrentUpdateSolrClient client = solrClients.get(url);
if (client == null) {
+ // NOTE: increasing to more than 1 threadCount for the client could cause updates to be reordered
+ // on a greater scale since the current behavior is to only increase the number of connections/Runners when
+ // the queue is more than half full.
client = new ConcurrentUpdateSolrClient(url, httpClient, 100, 1, updateExecutor, true) {
@Override
public void handleError(Throwable ex) {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/TransactionLog.java Mon Dec 21 11:39:57 2015
@@ -17,10 +17,12 @@
package org.apache.solr.update;
+import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
+import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
@@ -61,10 +63,10 @@ import org.slf4j.LoggerFactory;
* in them (since we know that if the request succeeds, all docs will be committed)
*
*/
-public class TransactionLog {
- public static Logger log = LoggerFactory.getLogger(TransactionLog.class);
- final boolean debug = log.isDebugEnabled();
- final boolean trace = log.isTraceEnabled();
+public class TransactionLog implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static boolean debug = log.isDebugEnabled();
+ private static boolean trace = log.isTraceEnabled();
public final static String END_MESSAGE="SOLR_TLOG_END";
@@ -536,7 +538,7 @@ public class TransactionLog {
}
}
- protected void close() {
+ public void close() {
try {
if (debug) {
log.debug("Closing tlog" + this);
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateHandler.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateHandler.java Mon Dec 21 11:39:57 2015
@@ -19,6 +19,7 @@ package org.apache.solr.update;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Vector;
import org.apache.solr.core.DirectoryFactory;
@@ -42,7 +43,7 @@ import org.slf4j.LoggerFactory;
*/
public abstract class UpdateHandler implements SolrInfoMBean {
- protected final static Logger log = LoggerFactory.getLogger(UpdateHandler.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected final SolrCore core;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateLog.java Mon Dec 21 11:39:57 2015
@@ -17,10 +17,12 @@
package org.apache.solr.update;
+import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
@@ -74,9 +76,9 @@ public class UpdateLog implements Plugin
public static String LOG_FILENAME_PATTERN = "%s.%019d";
public static String TLOG_NAME="tlog";
- public static Logger log = LoggerFactory.getLogger(UpdateLog.class);
- public boolean debug = log.isDebugEnabled();
- public boolean trace = log.isTraceEnabled();
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static boolean debug = log.isDebugEnabled();
+ private static boolean trace = log.isTraceEnabled();
// TODO: hack
public FileSystem getFs() {
@@ -311,19 +313,18 @@ public class UpdateLog implements Plugin
}
// TODO: these startingVersions assume that we successfully recover from all non-complete tlogs.
- UpdateLog.RecentUpdates startingUpdates = getRecentUpdates();
- try {
+ try (RecentUpdates startingUpdates = getRecentUpdates()) {
startingVersions = startingUpdates.getVersions(numRecordsToKeep);
startingOperation = startingUpdates.getLatestOperation();
// populate recent deletes list (since we can't get that info from the index)
- for (int i=startingUpdates.deleteList.size()-1; i>=0; i--) {
+ for (int i = startingUpdates.deleteList.size() - 1; i >= 0; i--) {
DeleteUpdate du = startingUpdates.deleteList.get(i);
- oldDeletes.put(new BytesRef(du.id), new LogPtr(-1,du.version));
+ oldDeletes.put(new BytesRef(du.id), new LogPtr(-1, du.version));
}
// populate recent deleteByQuery commands
- for (int i=startingUpdates.deleteByQueryList.size()-1; i>=0; i--) {
+ for (int i = startingUpdates.deleteByQueryList.size() - 1; i >= 0; i--) {
Update update = startingUpdates.deleteByQueryList.get(i);
List<Object> dbq = (List<Object>) update.log.lookup(update.pointer);
long version = (Long) dbq.get(1);
@@ -331,8 +332,6 @@ public class UpdateLog implements Plugin
trackDeleteByQuery(q, version);
}
- } finally {
- startingUpdates.close();
}
}
@@ -930,16 +929,32 @@ public class UpdateLog implements Plugin
}
}
- public class RecentUpdates {
- Deque<TransactionLog> logList; // newest first
+ public class RecentUpdates implements Closeable {
+
+ final Deque<TransactionLog> logList; // newest first
List<List<Update>> updateList;
HashMap<Long, Update> updates;
List<Update> deleteByQueryList;
List<DeleteUpdate> deleteList;
int latestOperation;
+ public RecentUpdates(Deque<TransactionLog> logList) {
+ this.logList = logList;
+ boolean success = false;
+ try {
+ update();
+ success = true;
+ } finally {
+ // defensive: if some unknown exception is thrown,
+ // make sure we close so that the tlogs are decref'd
+ if (!success) {
+ close();
+ }
+ }
+ }
+
public List<Long> getVersions(int n) {
- List<Long> ret = new ArrayList(n);
+ List<Long> ret = new ArrayList<>(n);
for (List<Update> singleList : updateList) {
for (Update ptr : singleList) {
@@ -1054,6 +1069,7 @@ public class UpdateLog implements Plugin
}
+ @Override
public void close() {
for (TransactionLog log : logList) {
log.decref();
@@ -1090,23 +1106,8 @@ public class UpdateLog implements Plugin
// TODO: what if I hand out a list of updates, then do an update, then hand out another list (and
// one of the updates I originally handed out fell off the list). Over-request?
+ return new RecentUpdates(logList);
- boolean success = false;
- RecentUpdates recentUpdates = null;
- try {
- recentUpdates = new RecentUpdates();
- recentUpdates.logList = logList;
- recentUpdates.update();
- success = true;
- } finally {
- // defensive: if some unknown exception is thrown,
- // make sure we close so that the tlogs are decref'd
- if (!success && recentUpdates != null) {
- recentUpdates.close();
- }
- }
-
- return recentUpdates;
}
public void bufferUpdates() {
@@ -1114,13 +1115,19 @@ public class UpdateLog implements Plugin
// it checks the state first
// assert state == State.ACTIVE;
- recoveryInfo = new RecoveryInfo();
-
// block all updates to eliminate race conditions
- // reading state and acting on it in the update processor
+ // reading state and acting on it in the distributed update processor
versionInfo.blockUpdates();
try {
- if (state != State.ACTIVE) return;
+ if (state == State.BUFFERING) {
+ log.info("Restarting buffering. previous=" + recoveryInfo);
+ } else if (state != State.ACTIVE) {
+ // we don't currently have support for handling other states
+ log.warn("Unexpected state for bufferUpdates: " + state + ", Ignoring request.");
+ return;
+ }
+
+ recoveryInfo = new RecoveryInfo();
if (log.isInfoEnabled()) {
log.info("Starting to buffer updates. " + this);
@@ -1574,9 +1581,7 @@ public class UpdateLog implements Plugin
Long highestVersion = null;
final RTimer timer = new RTimer();
- RecentUpdates recentUpdates = null;
- try {
- recentUpdates = getRecentUpdates();
+ try (RecentUpdates recentUpdates = getRecentUpdates()) {
long maxVersionFromRecent = recentUpdates.getMaxRecentVersion();
long maxVersionFromIndex = versions.getMaxVersionFromIndex(newSearcher);
@@ -1592,9 +1597,6 @@ public class UpdateLog implements Plugin
highestVersion = maxVersion;
} catch (IOException ioExc) {
log.warn("Failed to determine the max value of the version field due to: "+ioExc, ioExc);
- } finally {
- if (recentUpdates != null)
- recentUpdates.close();
}
log.info("Took {}ms to seed version buckets with highest version {}",
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java Mon Dec 21 11:39:57 2015
@@ -25,6 +25,7 @@ import org.apache.http.impl.conn.Pooling
import org.apache.http.impl.conn.SchemeRegistryFactory;
import org.apache.solr.client.solrj.impl.HttpClientConfigurer;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
@@ -34,16 +35,26 @@ import org.apache.solr.core.NodeConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.lang.invoke.MethodHandles;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
public class UpdateShardHandler {
- private static Logger log = LoggerFactory.getLogger(UpdateShardHandler.class);
-
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /*
+ * A downside to configuring an upper bound will be big update reorders (when that upper bound is hit)
+ * and then undetected shard inconsistency as a result.
+ * This update executor is used for different things too... both update streams (which may be very long lived)
+ * and control messages (peersync? LIR?) and could lead to starvation if limited.
+ * Therefore this thread pool is left unbounded. See SOLR-8205
+ */
private ExecutorService updateExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(
new SolrjNamedThreadFactory("updateExecutor"));
+ private ExecutorService recoveryExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(
+ new SolrjNamedThreadFactory("recoveryExecutor"));
+
private PoolingClientConnectionManager clientConnectionManager;
private final CloseableHttpClient client;
@@ -97,13 +108,32 @@ public class UpdateShardHandler {
return clientConnectionManager;
}
+ /**
+ * This method returns an executor that is not meant for disk IO and that will
+ * be interrupted on shutdown.
+ *
+ * @return an executor for update related activities that do not do disk IO.
+ */
public ExecutorService getUpdateExecutor() {
return updateExecutor;
}
+
+ /**
+ * In general, RecoveryStrategy threads do not do disk IO, but they open and close SolrCores
+ * in async threads, amoung other things, and can trigger disk IO, so we use this alternate
+ * executor rather than the 'updateExecutor', which is interrupted on shutdown.
+ *
+ * @return executor for {@link RecoveryStrategy} thread which will not be interrupted on close.
+ */
+ public ExecutorService getRecoveryExecutor() {
+ return recoveryExecutor;
+ }
public void close() {
try {
- ExecutorUtil.shutdownAndAwaitTermination(updateExecutor);
+ // we interrupt on purpose here, but this exectuor should not run threads that do disk IO!
+ ExecutorUtil.shutdownWithInterruptAndAwaitTermination(updateExecutor);
+ ExecutorUtil.shutdownAndAwaitTermination(recoveryExecutor);
} catch (Exception e) {
SolrException.log(log, e);
} finally {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/VersionInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/VersionInfo.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/VersionInfo.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/VersionInfo.java Mon Dec 21 11:39:57 2015
@@ -18,22 +18,21 @@
package org.apache.solr.update;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Map;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.LegacyNumericUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.schema.IndexSchema;
@@ -45,7 +44,7 @@ import org.slf4j.LoggerFactory;
public class VersionInfo {
- public static Logger log = LoggerFactory.getLogger(VersionInfo.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String VERSION_FIELD="_version_";
@@ -221,7 +220,7 @@ public class VersionInfo {
}
}
- public Long getMaxVersionFromIndex(SolrIndexSearcher searcher) throws IOException {
+ public Long getMaxVersionFromIndex(IndexSearcher searcher) throws IOException {
String versionFieldName = versionField.getName();
@@ -230,8 +229,9 @@ public class VersionInfo {
// if indexed, then we have terms to get the max from
if (versionField.indexed()) {
- Terms versionTerms = searcher.getLeafReader().terms(versionFieldName);
- Long max = (versionTerms != null) ? NumericUtils.getMaxLong(versionTerms) : null;
+ LeafReader leafReader = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader());
+ Terms versionTerms = leafReader.terms(versionFieldName);
+ Long max = (versionTerms != null) ? LegacyNumericUtils.getMaxLong(versionTerms) : null;
if (max != null) {
maxVersionInIndex = max.longValue();
log.info("Found MAX value {} from Terms for {} in index", maxVersionInIndex, versionFieldName);
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java Mon Dec 21 11:39:57 2015
@@ -36,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -121,7 +122,7 @@ import static org.apache.solr.core.Confi
* </processor></pre>
*/
public class AddSchemaFieldsUpdateProcessorFactory extends UpdateRequestProcessorFactory implements SolrCoreAware {
- public final static Logger log = LoggerFactory.getLogger(AddSchemaFieldsUpdateProcessorFactory.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String TYPE_MAPPING_PARAM = "typeMapping";
private static final String VALUE_CLASS_PARAM = "valueClass";
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AllValuesOrNoneFieldMutatingUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AllValuesOrNoneFieldMutatingUpdateProcessor.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AllValuesOrNoneFieldMutatingUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AllValuesOrNoneFieldMutatingUpdateProcessor.java Mon Dec 21 11:39:57 2015
@@ -20,7 +20,8 @@ package org.apache.solr.update.processor
import org.apache.solr.common.SolrInputField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
+
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -39,7 +40,7 @@ import java.util.Locale;
*/
public abstract class AllValuesOrNoneFieldMutatingUpdateProcessor extends FieldMutatingUpdateProcessor {
- private static final Logger log = LoggerFactory.getLogger(AllValuesOrNoneFieldMutatingUpdateProcessor.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final Object DELETE_VALUE_SINGLETON = new Object() {
@Override
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java Mon Dec 21 11:39:57 2015
@@ -1,5 +1,7 @@
package org.apache.solr.update.processor;
+import java.lang.invoke.MethodHandles;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -42,7 +44,7 @@ import org.slf4j.LoggerFactory;
*/
public class AtomicUpdateDocumentMerger {
- private final static Logger log = LoggerFactory.getLogger(AtomicUpdateDocumentMerger.class);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected final IndexSchema schema;
protected final SchemaField idField;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java Mon Dec 21 11:39:57 2015
@@ -18,6 +18,7 @@ package org.apache.solr.update.processor
*/
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -26,6 +27,8 @@ import org.apache.solr.response.SolrQuer
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.UpdateCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* <p>
@@ -38,6 +41,8 @@ public class CdcrUpdateProcessor extends
public static final String CDCR_UPDATE = "cdcr.update";
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
public CdcrUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
super(req, rsp, next);
}