You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/08/11 14:19:39 UTC
svn commit: r1512909 [29/38] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-too...
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java Sun Aug 11 12:19:13 2013
@@ -28,9 +28,11 @@ import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -68,6 +70,11 @@ public class SolrRequestParsers
public static final String SIMPLE = "simple";
public static final String STANDARD = "standard";
+ private static final Charset CHARSET_US_ASCII = Charset.forName("US-ASCII");
+
+ public static final String INPUT_ENCODING_KEY = "ie";
+ private static final byte[] INPUT_ENCODING_BYTES = INPUT_ENCODING_KEY.getBytes(CHARSET_US_ASCII);
+
private final HashMap<String, SolrRequestParser> parsers =
new HashMap<String, SolrRequestParser>();
private final boolean enableRemoteStreams;
@@ -242,7 +249,7 @@ public class SolrRequestParsers
}
}
};
- parseFormDataContent(in, Long.MAX_VALUE, IOUtils.CHARSET_UTF_8, map);
+ parseFormDataContent(in, Long.MAX_VALUE, IOUtils.CHARSET_UTF_8, map, true);
} catch (IOException ioe) {
throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
}
@@ -256,23 +263,53 @@ public class SolrRequestParsers
* @param charset to be used to decode resulting bytes after %-decoding
* @param map place all parameters in this map
*/
- @SuppressWarnings("fallthrough")
- static long parseFormDataContent(final InputStream postContent, final long maxLen, final Charset charset, final Map<String,String[]> map) throws IOException {
- final CharsetDecoder charsetDecoder = charset.newDecoder()
- .onMalformedInput(CodingErrorAction.REPORT)
- .onUnmappableCharacter(CodingErrorAction.REPORT);
+ @SuppressWarnings({"fallthrough", "resource"})
+ static long parseFormDataContent(final InputStream postContent, final long maxLen, Charset charset, final Map<String,String[]> map, boolean supportCharsetParam) throws IOException {
+ CharsetDecoder charsetDecoder = supportCharsetParam ? null : getCharsetDecoder(charset);
+ final LinkedList<Object> buffer = supportCharsetParam ? new LinkedList<Object>() : null;
long len = 0L, keyPos = 0L, valuePos = 0L;
- final ByteArrayOutputStream2 keyStream = new ByteArrayOutputStream2(),
- valueStream = new ByteArrayOutputStream2();
- ByteArrayOutputStream2 currentStream = keyStream;
+ final ByteArrayOutputStream keyStream = new ByteArrayOutputStream(),
+ valueStream = new ByteArrayOutputStream();
+ ByteArrayOutputStream currentStream = keyStream;
for(;;) {
int b = postContent.read();
switch (b) {
case -1: // end of stream
case '&': // separator
if (keyStream.size() > 0) {
- final String key = decodeChars(keyStream, keyPos, charsetDecoder), value = decodeChars(valueStream, valuePos, charsetDecoder);
- MultiMapSolrParams.addParam(key, value, map);
+ final byte[] keyBytes = keyStream.toByteArray(), valueBytes = valueStream.toByteArray();
+ if (Arrays.equals(keyBytes, INPUT_ENCODING_BYTES)) {
+ // we found a charset declaration in the raw bytes
+ if (charsetDecoder != null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ supportCharsetParam ? (
+ "Query string invalid: duplicate '"+
+ INPUT_ENCODING_KEY + "' (input encoding) key."
+ ) : (
+ "Key '" + INPUT_ENCODING_KEY + "' (input encoding) cannot "+
+ "be used in POSTed application/x-www-form-urlencoded form data. "+
+ "To set the input encoding of POSTed form data, use the "+
+ "'Content-Type' header and provide a charset!"
+ )
+ );
+ }
+ // decode the charset from raw bytes
+ charset = Charset.forName(decodeChars(valueBytes, keyPos, getCharsetDecoder(CHARSET_US_ASCII)));
+ charsetDecoder = getCharsetDecoder(charset);
+ // finally decode all buffered tokens
+ decodeBuffer(buffer, map, charsetDecoder);
+ } else if (charsetDecoder == null) {
+ // we have no charset decoder until now, buffer the keys / values for later processing:
+ buffer.add(keyBytes);
+ buffer.add(Long.valueOf(keyPos));
+ buffer.add(valueBytes);
+ buffer.add(Long.valueOf(valuePos));
+ } else {
+ // we already have a charsetDecoder, so we can directly decode without buffering:
+ final String key = decodeChars(keyBytes, keyPos, charsetDecoder),
+ value = decodeChars(valueBytes, valuePos, charsetDecoder);
+ MultiMapSolrParams.addParam(key, value, map);
+ }
} else if (valueStream.size() > 0) {
throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded invalid: missing key");
}
@@ -309,12 +346,23 @@ public class SolrRequestParsers
throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content exceeds upload limit of " + (maxLen/1024L) + " KB");
}
}
+ // if we have not seen a charset declaration, decode the buffer now using the default one (UTF-8 or given via Content-Type):
+ if (buffer != null && !buffer.isEmpty()) {
+ assert charsetDecoder == null;
+ decodeBuffer(buffer, map, getCharsetDecoder(charset));
+ }
return len;
}
- private static String decodeChars(ByteArrayOutputStream2 stream, long position, CharsetDecoder charsetDecoder) {
+ private static CharsetDecoder getCharsetDecoder(Charset charset) {
+ return charset.newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+
+ private static String decodeChars(byte[] bytes, long position, CharsetDecoder charsetDecoder) {
try {
- return charsetDecoder.decode(ByteBuffer.wrap(stream.buffer(), 0, stream.size())).toString();
+ return charsetDecoder.decode(ByteBuffer.wrap(bytes)).toString();
} catch (CharacterCodingException cce) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"URLDecoder: Invalid character encoding detected after position " + position +
@@ -323,10 +371,18 @@ public class SolrRequestParsers
}
}
- /** Makes the buffer of ByteArrayOutputStream available without copy. */
- static final class ByteArrayOutputStream2 extends ByteArrayOutputStream {
- byte[] buffer() {
- return buf;
+ private static void decodeBuffer(final LinkedList<Object> input, final Map<String,String[]> map, CharsetDecoder charsetDecoder) {
+ for (final Iterator<Object> it = input.iterator(); it.hasNext(); ) {
+ final byte[] keyBytes = (byte[]) it.next();
+ it.remove();
+ final Long keyPos = (Long) it.next();
+ it.remove();
+ final byte[] valueBytes = (byte[]) it.next();
+ it.remove();
+ final Long valuePos = (Long) it.next();
+ it.remove();
+ MultiMapSolrParams.addParam(decodeChars(keyBytes, keyPos.longValue(), charsetDecoder),
+ decodeChars(valueBytes, valuePos.longValue(), charsetDecoder), map);
}
}
@@ -361,281 +417,272 @@ public class SolrRequestParsers
public void setAddRequestHeadersToContext(boolean addRequestHeadersToContext) {
this.addHttpRequestToContext = addRequestHeadersToContext;
}
-}
-//-----------------------------------------------------------------
-//-----------------------------------------------------------------
+ //-----------------------------------------------------------------
+ //-----------------------------------------------------------------
-// I guess we don't really even need the interface, but i'll keep it here just for kicks
-interface SolrRequestParser
-{
- public SolrParams parseParamsAndFillStreams(
- final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception;
-}
+ // I guess we don't really even need the interface, but i'll keep it here just for kicks
+ interface SolrRequestParser
+ {
+ public SolrParams parseParamsAndFillStreams(
+ final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception;
+ }
-//-----------------------------------------------------------------
-//-----------------------------------------------------------------
-
-/**
- * The simple parser just uses the params directly, does not support POST URL-encoded forms
- */
-class SimpleRequestParser implements SolrRequestParser
-{
- @Override
- public SolrParams parseParamsAndFillStreams(
- final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ //-----------------------------------------------------------------
+ //-----------------------------------------------------------------
+
+ /**
+ * The simple parser just uses the params directly, does not support POST URL-encoded forms
+ */
+ static class SimpleRequestParser implements SolrRequestParser
{
- return SolrRequestParsers.parseQueryString(req.getQueryString());
+ @Override
+ public SolrParams parseParamsAndFillStreams(
+ final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ {
+ return parseQueryString(req.getQueryString());
+ }
}
-}
-/**
- * Wrap an HttpServletRequest as a ContentStream
- */
-class HttpRequestContentStream extends ContentStreamBase
-{
- private final HttpServletRequest req;
-
- public HttpRequestContentStream( HttpServletRequest req ) {
- this.req = req;
-
- contentType = req.getContentType();
- // name = ???
- // sourceInfo = ???
+ /**
+ * Wrap an HttpServletRequest as a ContentStream
+ */
+ static class HttpRequestContentStream extends ContentStreamBase
+ {
+ private final HttpServletRequest req;
- String v = req.getHeader( "Content-Length" );
- if( v != null ) {
- size = Long.valueOf( v );
+ public HttpRequestContentStream( HttpServletRequest req ) {
+ this.req = req;
+
+ contentType = req.getContentType();
+ // name = ???
+ // sourceInfo = ???
+
+ String v = req.getHeader( "Content-Length" );
+ if( v != null ) {
+ size = Long.valueOf( v );
+ }
}
- }
- @Override
- public InputStream getStream() throws IOException {
- return req.getInputStream();
+ @Override
+ public InputStream getStream() throws IOException {
+ return req.getInputStream();
+ }
}
-}
-/**
- * Wrap a FileItem as a ContentStream
- */
-class FileItemContentStream extends ContentStreamBase
-{
- private final FileItem item;
-
- public FileItemContentStream( FileItem f )
+ /**
+ * Wrap a FileItem as a ContentStream
+ */
+ static class FileItemContentStream extends ContentStreamBase
{
- item = f;
- contentType = item.getContentType();
- name = item.getName();
- sourceInfo = item.getFieldName();
- size = item.getSize();
- }
-
- @Override
- public InputStream getStream() throws IOException {
- return item.getInputStream();
- }
-}
-
-/**
- * The raw parser just uses the params directly
- */
-class RawRequestParser implements SolrRequestParser
-{
- @Override
- public SolrParams parseParamsAndFillStreams(
- final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ private final FileItem item;
+
+ public FileItemContentStream( FileItem f )
+ {
+ item = f;
+ contentType = item.getContentType();
+ name = item.getName();
+ sourceInfo = item.getFieldName();
+ size = item.getSize();
+ }
+
+ @Override
+ public InputStream getStream() throws IOException {
+ return item.getInputStream();
+ }
+ }
+
+ /**
+ * The raw parser just uses the params directly
+ */
+ static class RawRequestParser implements SolrRequestParser
{
- streams.add( new HttpRequestContentStream( req ) );
- return SolrRequestParsers.parseQueryString( req.getQueryString() );
+ @Override
+ public SolrParams parseParamsAndFillStreams(
+ final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ {
+ streams.add( new HttpRequestContentStream( req ) );
+ return parseQueryString( req.getQueryString() );
+ }
}
-}
-/**
- * Extract Multipart streams
- */
-class MultipartRequestParser implements SolrRequestParser
-{
- private final int uploadLimitKB;
-
- public MultipartRequestParser( int limit )
- {
- uploadLimitKB = limit;
- }
-
- @Override
- public SolrParams parseParamsAndFillStreams(
- final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ /**
+ * Extract Multipart streams
+ */
+ static class MultipartRequestParser implements SolrRequestParser
{
- if( !ServletFileUpload.isMultipartContent(req) ) {
- throw new SolrException( ErrorCode.BAD_REQUEST, "Not multipart content! "+req.getContentType() );
- }
+ private final int uploadLimitKB;
- MultiMapSolrParams params = SolrRequestParsers.parseQueryString( req.getQueryString() );
+ public MultipartRequestParser( int limit )
+ {
+ uploadLimitKB = limit;
+ }
- // Create a factory for disk-based file items
- DiskFileItemFactory factory = new DiskFileItemFactory();
+ @Override
+ public SolrParams parseParamsAndFillStreams(
+ final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ {
+ if( !ServletFileUpload.isMultipartContent(req) ) {
+ throw new SolrException( ErrorCode.BAD_REQUEST, "Not multipart content! "+req.getContentType() );
+ }
+
+ MultiMapSolrParams params = parseQueryString( req.getQueryString() );
+
+ // Create a factory for disk-based file items
+ DiskFileItemFactory factory = new DiskFileItemFactory();
- // Set factory constraints
- // TODO - configure factory.setSizeThreshold(yourMaxMemorySize);
- // TODO - configure factory.setRepository(yourTempDirectory);
-
- // Create a new file upload handler
- ServletFileUpload upload = new ServletFileUpload(factory);
- upload.setSizeMax( ((long) uploadLimitKB) * 1024L );
-
- // Parse the request
- List items = upload.parseRequest(req);
- Iterator iter = items.iterator();
- while (iter.hasNext()) {
- FileItem item = (FileItem) iter.next();
-
- // If its a form field, put it in our parameter map
- if (item.isFormField()) {
- MultiMapSolrParams.addParam(
- item.getFieldName(),
- item.getString(), params.getMap() );
- }
- // Add the stream
- else {
- streams.add( new FileItemContentStream( item ) );
- }
+ // Set factory constraints
+ // TODO - configure factory.setSizeThreshold(yourMaxMemorySize);
+ // TODO - configure factory.setRepository(yourTempDirectory);
+
+ // Create a new file upload handler
+ ServletFileUpload upload = new ServletFileUpload(factory);
+ upload.setSizeMax( ((long) uploadLimitKB) * 1024L );
+
+ // Parse the request
+ List items = upload.parseRequest(req);
+ Iterator iter = items.iterator();
+ while (iter.hasNext()) {
+ FileItem item = (FileItem) iter.next();
+
+ // If its a form field, put it in our parameter map
+ if (item.isFormField()) {
+ MultiMapSolrParams.addParam(
+ item.getFieldName(),
+ item.getString(), params.getMap() );
+ }
+ // Add the stream
+ else {
+ streams.add( new FileItemContentStream( item ) );
+ }
+ }
+ return params;
}
- return params;
}
-}
-/**
- * Extract application/x-www-form-urlencoded form data for POST requests
- */
-class FormDataRequestParser implements SolrRequestParser
-{
- private final int uploadLimitKB;
-
- public FormDataRequestParser( int limit )
- {
- uploadLimitKB = limit;
- }
-
- @Override
- public SolrParams parseParamsAndFillStreams(
- final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ /**
+ * Extract application/x-www-form-urlencoded form data for POST requests
+ */
+ static class FormDataRequestParser implements SolrRequestParser
{
- if (!isFormData(req)) {
- throw new SolrException( ErrorCode.BAD_REQUEST, "Not application/x-www-form-urlencoded content: "+req.getContentType() );
- }
-
- final Map<String,String[]> map = new HashMap<String, String[]>();
+ private final int uploadLimitKB;
- // also add possible URL parameters and include into the map (parsed using UTF-8):
- final String qs = req.getQueryString();
- if (qs != null) {
- SolrRequestParsers.parseQueryString(qs, map);
+ public FormDataRequestParser( int limit )
+ {
+ uploadLimitKB = limit;
}
- // may be -1, so we check again later. But if its already greater we can stop processing!
- final long totalLength = req.getContentLength();
- final long maxLength = ((long) uploadLimitKB) * 1024L;
- if (totalLength > maxLength) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
- totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
+ @Override
+ public SolrParams parseParamsAndFillStreams(
+ final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ {
+ if (!isFormData(req)) {
+ throw new SolrException( ErrorCode.BAD_REQUEST, "Not application/x-www-form-urlencoded content: "+req.getContentType() );
+ }
+
+ final Map<String,String[]> map = new HashMap<String, String[]>();
+
+ // also add possible URL parameters and include into the map (parsed using UTF-8):
+ final String qs = req.getQueryString();
+ if (qs != null) {
+ parseQueryString(qs, map);
+ }
+
+ // may be -1, so we check again later. But if its already greater we can stop processing!
+ final long totalLength = req.getContentLength();
+ final long maxLength = ((long) uploadLimitKB) * 1024L;
+ if (totalLength > maxLength) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
+ totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
+ }
+
+ // get query String from request body, using the charset given in content-type:
+ final String cs = ContentStreamBase.getCharsetFromContentType(req.getContentType());
+ final Charset charset = (cs == null) ? IOUtils.CHARSET_UTF_8 : Charset.forName(cs);
+ InputStream in = null;
+ try {
+ in = req.getInputStream();
+ final long bytesRead = parseFormDataContent(FastInputStream.wrap(in), maxLength, charset, map, false);
+ if (bytesRead == 0L && totalLength > 0L) {
+ throw getParameterIncompatibilityException();
+ }
+ } catch (IOException ioe) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
+ } catch (IllegalStateException ise) {
+ throw (SolrException) getParameterIncompatibilityException().initCause(ise);
+ } finally {
+ IOUtils.closeWhileHandlingException(in);
+ }
+
+ return new MultiMapSolrParams(map);
}
- // get query String from request body, using the charset given in content-type:
- final String cs = ContentStreamBase.getCharsetFromContentType(req.getContentType());
- final Charset charset = (cs == null) ? IOUtils.CHARSET_UTF_8 : Charset.forName(cs);
- InputStream in = null;
- try {
- in = req.getInputStream();
- final long bytesRead = SolrRequestParsers.parseFormDataContent(FastInputStream.wrap(in), maxLength, charset, map);
- if (bytesRead == 0L && totalLength > 0L) {
- throw getParameterIncompatibilityException();
- }
- } catch (IOException ioe) {
- throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
- } catch (IllegalStateException ise) {
- throw (SolrException) getParameterIncompatibilityException().initCause(ise);
- } finally {
- IOUtils.closeWhileHandlingException(in);
+ private SolrException getParameterIncompatibilityException() {
+ return new SolrException(ErrorCode.SERVER_ERROR,
+ "Solr requires that request parameters sent using application/x-www-form-urlencoded " +
+ "content-type can be read through the request input stream. Unfortunately, the " +
+ "stream was empty / not available. This may be caused by another servlet filter calling " +
+ "ServletRequest.getParameter*() before SolrDispatchFilter, please remove it."
+ );
}
- return new MultiMapSolrParams(map);
- }
-
- private SolrException getParameterIncompatibilityException() {
- return new SolrException(ErrorCode.SERVER_ERROR,
- "Solr requires that request parameters sent using application/x-www-form-urlencoded " +
- "content-type can be read through the request input stream. Unfortunately, the " +
- "stream was empty / not available. This may be caused by another servlet filter calling " +
- "ServletRequest.getParameter*() before SolrDispatchFilter, please remove it."
- );
- }
-
- public boolean isFormData(HttpServletRequest req) {
- String contentType = req.getContentType();
- if (contentType != null) {
- int idx = contentType.indexOf( ';' );
- if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
- contentType = contentType.substring( 0, idx );
- }
- contentType = contentType.trim();
- if( "application/x-www-form-urlencoded".equalsIgnoreCase(contentType)) {
- return true;
+ public boolean isFormData(HttpServletRequest req) {
+ String contentType = req.getContentType();
+ if (contentType != null) {
+ int idx = contentType.indexOf( ';' );
+ if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
+ contentType = contentType.substring( 0, idx );
+ }
+ contentType = contentType.trim();
+ if( "application/x-www-form-urlencoded".equalsIgnoreCase(contentType)) {
+ return true;
+ }
}
+ return false;
}
- return false;
}
-}
-/**
- * The default Logic
- */
-class StandardRequestParser implements SolrRequestParser
-{
- MultipartRequestParser multipart;
- RawRequestParser raw;
- FormDataRequestParser formdata;
-
- StandardRequestParser(MultipartRequestParser multi, RawRequestParser raw, FormDataRequestParser formdata)
- {
- this.multipart = multi;
- this.raw = raw;
- this.formdata = formdata;
- }
-
- @Override
- public SolrParams parseParamsAndFillStreams(
- final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ /**
+ * The default Logic
+ */
+ static class StandardRequestParser implements SolrRequestParser
{
- String method = req.getMethod().toUpperCase(Locale.ROOT);
- if ("GET".equals(method) || "HEAD".equals(method)
- || ("PUT".equals(method) && req.getRequestURI().contains("/schema"))) {
- return SolrRequestParsers.parseQueryString(req.getQueryString());
- }
- if ("POST".equals( method ) ) {
- if (formdata.isFormData(req)) {
- return formdata.parseParamsAndFillStreams(req, streams);
- }
- if (ServletFileUpload.isMultipartContent(req)) {
- return multipart.parseParamsAndFillStreams(req, streams);
+ MultipartRequestParser multipart;
+ RawRequestParser raw;
+ FormDataRequestParser formdata;
+
+ StandardRequestParser(MultipartRequestParser multi, RawRequestParser raw, FormDataRequestParser formdata)
+ {
+ this.multipart = multi;
+ this.raw = raw;
+ this.formdata = formdata;
+ }
+
+ @Override
+ public SolrParams parseParamsAndFillStreams(
+ final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+ {
+ String method = req.getMethod().toUpperCase(Locale.ROOT);
+ if ("GET".equals(method) || "HEAD".equals(method)
+ || ("PUT".equals(method) && req.getRequestURI().contains("/schema"))) {
+ return parseQueryString(req.getQueryString());
+ }
+ if ("POST".equals( method ) ) {
+ if (formdata.isFormData(req)) {
+ return formdata.parseParamsAndFillStreams(req, streams);
+ }
+ if (ServletFileUpload.isMultipartContent(req)) {
+ return multipart.parseParamsAndFillStreams(req, streams);
+ }
+ return raw.parseParamsAndFillStreams(req, streams);
}
- return raw.parseParamsAndFillStreams(req, streams);
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported method: " + method + " for request " + req);
}
- throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported method: " + method + " for request " + req);
}
-}
-
-
-
-
-
-
-
-
-
+}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java Sun Aug 11 12:19:13 2013
@@ -202,7 +202,7 @@ public class SpellCheckCollator {
//then be sure all of the new words have the same optional/required/prohibited status in the query.
while(indexOfSpace>-1 && indexOfSpace<corr.length()-1) {
addParenthesis = true;
- char previousChar = tok.startOffset()>0 ? collation.charAt(tok.startOffset()-1) : ' ';
+ char previousChar = tok.startOffset()>0 ? origQuery.charAt(tok.startOffset()-1) : ' ';
if(previousChar=='-' || previousChar=='+') {
corrSb.insert(indexOfSpace + bump, previousChar);
if(requiredOrProhibited==null) {
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java Sun Aug 11 12:19:13 2013
@@ -18,8 +18,6 @@
package org.apache.solr.spelling;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -28,11 +26,10 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
@@ -160,7 +157,7 @@ public class SpellingQueryConverter exte
flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;
}
try {
- analyze(result, new StringReader(word), startIndex, flagValue);
+ analyze(result, word, startIndex, flagValue);
} catch (IOException e) {
// TODO: shouldn't we log something?
}
@@ -174,7 +171,7 @@ public class SpellingQueryConverter exte
return result;
}
- protected void analyze(Collection<Token> result, Reader text, int offset, int flagsAttValue) throws IOException {
+ protected void analyze(Collection<Token> result, String text, int offset, int flagsAttValue) throws IOException {
TokenStream stream = analyzer.tokenStream("", text);
// TODO: support custom attributes
CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java Sun Aug 11 12:19:13 2013
@@ -18,7 +18,6 @@ package org.apache.solr.spelling;
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -38,7 +37,7 @@ public class SuggestQueryConverter exten
Collection<Token> result = new ArrayList<Token>();
try {
- analyze(result, new StringReader(original), 0, 0);
+ analyze(result, original, 0, 0);
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FuzzyLookupFactory.java Sun Aug 11 12:19:13 2013
@@ -33,23 +33,33 @@ import org.apache.solr.spelling.suggest.
public class FuzzyLookupFactory extends LookupFactory {
/**
+ * If <code>true</code>, maxEdits, minFuzzyLength, transpositions and nonFuzzyPrefix
+ * will be measured in Unicode code points (actual letters) instead of bytes.
+ */
+ public static final String UNICODE_AWARE = "unicodeAware";
+
+ /**
* Maximum number of edits allowed, used by {@link LevenshteinAutomata#toAutomaton(int)}
+ * in bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
*/
public static final String MAX_EDITS = "maxEdits";
/**
* If transpositions are allowed, Fuzzy suggestions will be computed based on a primitive
* edit operation. If it is false, it will be based on the classic Levenshtein algorithm.
+ * Transpositions of bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
*/
public static final String TRANSPOSITIONS = "transpositions";
/**
* Length of common (non-fuzzy) prefix for the suggestions
+ * in bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
*/
public static final String NON_FUZZY_PREFIX = "nonFuzzyPrefix";
/**
* Minimum length of lookup key before any edits are allowed for the suggestions
+ * in bytes or Unicode code points (if {@link #UNICODE_AWARE} option is set to true).
*/
public static final String MIN_FUZZY_LENGTH = "minFuzzyLength";
@@ -113,9 +123,13 @@ public class FuzzyLookupFactory extends
? Integer.parseInt(params.get(MIN_FUZZY_LENGTH).toString())
:FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
+ boolean unicodeAware = (params.get(UNICODE_AWARE) != null)
+ ? Boolean.valueOf(params.get(UNICODE_AWARE).toString())
+ : FuzzySuggester.DEFAULT_UNICODE_AWARE;
+
return new FuzzySuggester(indexAnalyzer, queryAnalyzer, options,
maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, maxEdits,
- transpositions, nonFuzzyPrefix, minFuzzyLength);
+ transpositions, nonFuzzyPrefix, minFuzzyLength, unicodeAware);
}
@Override
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Sun Aug 11 12:19:13 2013
@@ -187,6 +187,76 @@ public final class DefaultSolrCoreState
}
}
}
+
+ @Override
+ public synchronized void closeIndexWriter(SolrCore core, boolean rollback)
+ throws IOException {
+ log.info("Closing IndexWriter...");
+ String coreName = core.getName();
+ synchronized (writerPauseLock) {
+ if (closed) {
+ throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Already closed");
+ }
+
+ // we need to wait for the Writer to fall out of use
+ // first lets stop it from being lent out
+ pauseWriter = true;
+ // then lets wait until its out of use
+ log.info("Waiting until IndexWriter is unused... core=" + coreName);
+
+ while (!writerFree) {
+ try {
+ writerPauseLock.wait(100);
+ } catch (InterruptedException e) {}
+
+ if (closed) {
+ throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE,
+ "SolrCoreState already closed");
+ }
+ }
+
+ if (indexWriter != null) {
+ if (!rollback) {
+ try {
+ log.info("Closing old IndexWriter... core=" + coreName);
+ indexWriter.close();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error closing old IndexWriter. core="
+ + coreName, t);
+ }
+ } else {
+ try {
+ log.info("Rollback old IndexWriter... core=" + coreName);
+ indexWriter.rollback();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error rolling back old IndexWriter. core="
+ + coreName, t);
+ }
+ }
+ }
+
+ }
+ }
+
+ @Override
+ public synchronized void openIndexWriter(SolrCore core) throws IOException {
+ log.info("Creating new IndexWriter...");
+ synchronized (writerPauseLock) {
+ if (closed) {
+ throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Already closed");
+ }
+
+ try {
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2");
+ log.info("New IndexWriter is ready to be used.");
+ // we need to null this so it picks up the new writer next get call
+ refCntWriter = null;
+ } finally {
+ pauseWriter = false;
+ writerPauseLock.notifyAll();
+ }
+ }
+ }
@Override
public synchronized void rollbackIndexWriter(SolrCore core) throws IOException {
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Sun Aug 11 12:19:13 2013
@@ -31,6 +31,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
@@ -108,6 +109,8 @@ public class DirectUpdateHandler2 extend
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
+
+
}
public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
@@ -125,6 +128,13 @@ public class DirectUpdateHandler2 extend
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, updateHandlerInfo.openSearcher, true);
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
+
+ UpdateLog existingLog = updateHandler.getUpdateLog();
+ if (this.ulog != null && this.ulog == existingLog) {
+ // If we are reusing the existing update log, inform the log that it's update handler has changed.
+ // We do this as late as possible.
+ this.ulog.init(this, core);
+ }
}
private void deleteAll() throws IOException {
@@ -423,11 +433,11 @@ public class DirectUpdateHandler2 extend
log.info("start " + cmd);
- IndexReader[] readers = cmd.readers;
- if (readers != null && readers.length > 0) {
+ List<DirectoryReader> readers = cmd.readers;
+ if (readers != null && readers.size() > 0) {
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
- iw.get().addIndexes(readers);
+ iw.get().addIndexes(readers.toArray(new IndexReader[readers.size()]));
} finally {
iw.decref();
}
@@ -531,11 +541,17 @@ public class DirectUpdateHandler2 extend
}
// SolrCore.verbose("writer.commit() start writer=",writer);
- final Map<String,String> commitData = new HashMap<String,String>();
- commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY,
- String.valueOf(System.currentTimeMillis()));
- writer.setCommitData(commitData);
- writer.commit();
+
+ if (writer.hasUncommittedChanges()) {
+ final Map<String,String> commitData = new HashMap<String,String>();
+ commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY,
+ String.valueOf(System.currentTimeMillis()));
+ writer.setCommitData(commitData);
+ writer.commit();
+ } else {
+ log.info("No uncommitted changes. Skipping IW.commit.");
+ }
+
// SolrCore.verbose("writer.commit() end");
numDocsPending.set(0);
callPostCommitCallbacks();
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/MergeIndexesCommand.java Sun Aug 11 12:19:13 2013
@@ -17,9 +17,14 @@
package org.apache.solr.update;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
import org.apache.lucene.index.DirectoryReader;
import org.apache.solr.request.SolrQueryRequest;
+import java.util.List;
+
/**
* A merge indexes command encapsulated in an object.
*
@@ -27,9 +32,9 @@ import org.apache.solr.request.SolrQuery
*
*/
public class MergeIndexesCommand extends UpdateCommand {
- public DirectoryReader[] readers;
+ public List<DirectoryReader> readers;
- public MergeIndexesCommand(DirectoryReader[] readers, SolrQueryRequest req) {
+ public MergeIndexesCommand(List<DirectoryReader> readers, SolrQueryRequest req) {
super(req);
this.readers = readers;
}
@@ -42,12 +47,13 @@ public class MergeIndexesCommand extends
@Override
public String toString() {
StringBuilder sb = new StringBuilder(super.toString());
- if (readers != null && readers.length > 0) {
- sb.append(readers[0].directory());
- for (int i = 1; i < readers.length; i++) {
- sb.append(",").append(readers[i].directory());
+ Joiner joiner = Joiner.on(",");
+ Iterable<String> directories = Iterables.transform(readers, new Function<DirectoryReader, String>() {
+ public String apply(DirectoryReader reader) {
+ return reader.directory().toString();
}
- }
+ });
+ joiner.skipNulls().join(sb, directories);
sb.append('}');
return sb.toString();
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Sun Aug 11 12:19:13 2013
@@ -114,11 +114,11 @@ public class SolrCmdDistributor {
// make sure any pending deletes are flushed
flushDeletes(1);
-
+
// TODO: this is brittle
// need to make a clone since these commands may be reused
AddUpdateCommand clone = new AddUpdateCommand(null);
-
+
clone.solrDoc = cmd.solrDoc;
clone.commitWithin = cmd.commitWithin;
clone.overwrite = cmd.overwrite;
@@ -135,10 +135,79 @@ public class SolrCmdDistributor {
}
alist.add(addRequest);
}
-
+
flushAdds(maxBufferedAddsPerServer);
}
-
+
+ /**
+ * Synchronous (blocking) add to specified node. Any error returned from node is propagated.
+ */
+ public void syncAdd(AddUpdateCommand cmd, Node node, ModifiableSolrParams params) throws IOException {
+ log.info("SYNCADD on {} : {}", node, cmd.getPrintableId());
+ checkResponses(false);
+ // flush all pending deletes
+ flushDeletes(1);
+ // flush all pending adds
+ flushAdds(1);
+ // finish with the pending requests
+ checkResponses(false);
+
+ UpdateRequestExt ureq = new UpdateRequestExt();
+ ureq.add(cmd.solrDoc, cmd.commitWithin, cmd.overwrite);
+ ureq.setParams(params);
+ syncRequest(node, ureq);
+ }
+
+ public void syncDelete(DeleteUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {
+ log.info("SYNCDELETE on {} : ", nodes, cmd);
+ checkResponses(false);
+ // flush all pending adds
+ flushAdds(1);
+ // flush all pending deletes
+ flushDeletes(1);
+ // finish pending requests
+ checkResponses(false);
+
+ DeleteUpdateCommand clonedCmd = clone(cmd);
+ DeleteRequest deleteRequest = new DeleteRequest();
+ deleteRequest.cmd = clonedCmd;
+ deleteRequest.params = params;
+
+ UpdateRequestExt ureq = new UpdateRequestExt();
+ if (cmd.isDeleteById()) {
+ ureq.deleteById(cmd.getId(), cmd.getVersion());
+ } else {
+ ureq.deleteByQuery(cmd.query);
+ }
+ ureq.setParams(params);
+ for (Node node : nodes) {
+ syncRequest(node, ureq);
+ }
+ }
+
+ private void syncRequest(Node node, UpdateRequestExt ureq) {
+ Request sreq = new Request();
+ sreq.node = node;
+ sreq.ureq = ureq;
+
+ String url = node.getUrl();
+ String fullUrl;
+ if (!url.startsWith("http://") && !url.startsWith("https://")) {
+ fullUrl = "http://" + url;
+ } else {
+ fullUrl = url;
+ }
+
+ HttpSolrServer server = new HttpSolrServer(fullUrl,
+ updateShardHandler.getHttpClient());
+
+ try {
+ sreq.ursp = server.request(ureq);
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Failed synchronous update on shard " + sreq.node + " update: " + ureq , e);
+ }
+ }
+
public void distribCommit(CommitUpdateCommand cmd, List<Node> nodes,
ModifiableSolrParams params) throws IOException {
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrCoreState.java Sun Aug 11 12:19:13 2013
@@ -88,6 +88,26 @@ public abstract class SolrCoreState {
*/
public abstract void newIndexWriter(SolrCore core, boolean rollback) throws IOException;
+
+ /**
+ * Expert method that closes the IndexWriter - you must call {@link #openIndexWriter(SolrCore)}
+ * in a finally block after calling this method.
+ *
+ * @param core that the IW belongs to
+ * @param rollback true if IW should rollback rather than close
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public abstract void closeIndexWriter(SolrCore core, boolean rollback) throws IOException;
+
+ /**
+ * Expert method that opens the IndexWriter - you must call {@link #closeIndexWriter(SolrCore, boolean)}
+ * first, and then call this method in a finally block.
+ *
+ * @param core that the IW belongs to
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public abstract void openIndexWriter(SolrCore core) throws IOException;
+
/**
* Get the current IndexWriter. If a new IndexWriter must be created, use the
* settings from the given {@link SolrCore}.
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Sun Aug 11 12:19:13 2013
@@ -17,12 +17,15 @@
package org.apache.solr.update;
+import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.Version;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.schema.IndexSchema;
@@ -30,6 +33,10 @@ import org.apache.solr.util.SolrPluginUt
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
import java.util.List;
/**
@@ -43,7 +50,14 @@ public class SolrIndexConfig {
public static final String DEFAULT_MERGE_SCHEDULER_CLASSNAME = ConcurrentMergeScheduler.class.getName();
public final Version luceneVersion;
+ /**
+ * The explicit value of <useCompoundFile> specified on this index config
+ * @deprecated use {@link #getUseCompoundFile}
+ */
+ @Deprecated
public final boolean useCompoundFile;
+ private boolean effectiveUseCompountFileSetting;
+
public final int maxBufferedDocs;
public final int maxMergeDocs;
public final int maxIndexingThreads;
@@ -55,11 +69,10 @@ public class SolrIndexConfig {
public final String lockType;
public final PluginInfo mergePolicyInfo;
public final PluginInfo mergeSchedulerInfo;
- public final int termIndexInterval;
public final PluginInfo mergedSegmentWarmerInfo;
- public String infoStreamFile = null;
+ public InfoStream infoStream = InfoStream.NO_OUTPUT;
// Available lock types
public final static String LOCK_TYPE_SIMPLE = "simple";
@@ -73,7 +86,7 @@ public class SolrIndexConfig {
@SuppressWarnings("deprecation")
private SolrIndexConfig(SolrConfig solrConfig) {
luceneVersion = solrConfig.luceneMatchVersion;
- useCompoundFile = false;
+ useCompoundFile = effectiveUseCompountFileSetting = false;
maxBufferedDocs = -1;
maxMergeDocs = -1;
maxIndexingThreads = IndexWriterConfig.DEFAULT_MAX_THREAD_STATES;
@@ -81,7 +94,6 @@ public class SolrIndexConfig {
ramBufferSizeMB = 100;
writeLockTimeout = -1;
lockType = LOCK_TYPE_NATIVE;
- termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL;
mergePolicyInfo = null;
mergeSchedulerInfo = null;
defaultMergePolicyClassName = TieredMergePolicy.class.getName();
@@ -105,15 +117,19 @@ public class SolrIndexConfig {
def = new SolrIndexConfig(solrConfig);
}
+ // sanity check: this will throw an error for us if there is more then one
+ // config section
+ Object unused = solrConfig.getNode(prefix, false);
+
luceneVersion = solrConfig.luceneMatchVersion;
// Assert that end-of-life parameters or syntax is not in our config.
// Warn for luceneMatchVersion's before LUCENE_36, fail fast above
assertWarnOrFail("The <mergeScheduler>myclass</mergeScheduler> syntax is no longer supported in solrconfig.xml. Please use syntax <mergeScheduler class=\"myclass\"/> instead.",
- !((solrConfig.get(prefix+"/mergeScheduler/text()",null) != null) && (solrConfig.get(prefix+"/mergeScheduler/@class",null) == null)),
+ !((solrConfig.getNode(prefix+"/mergeScheduler",false) != null) && (solrConfig.get(prefix+"/mergeScheduler/@class",null) == null)),
true);
assertWarnOrFail("The <mergePolicy>myclass</mergePolicy> syntax is no longer supported in solrconfig.xml. Please use syntax <mergePolicy class=\"myclass\"/> instead.",
- !((solrConfig.get(prefix+"/mergePolicy/text()",null) != null) && (solrConfig.get(prefix+"/mergePolicy/@class",null) == null)),
+ !((solrConfig.getNode(prefix+"/mergePolicy",false) != null) && (solrConfig.get(prefix+"/mergePolicy/@class",null) == null)),
true);
assertWarnOrFail("The <luceneAutoCommit>true|false</luceneAutoCommit> parameter is no longer valid in solrconfig.xml.",
solrConfig.get(prefix+"/luceneAutoCommit", null) == null,
@@ -121,6 +137,7 @@ public class SolrIndexConfig {
defaultMergePolicyClassName = def.defaultMergePolicyClassName;
useCompoundFile=solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile);
+ effectiveUseCompountFileSetting = useCompoundFile;
maxBufferedDocs=solrConfig.getInt(prefix+"/maxBufferedDocs",def.maxBufferedDocs);
maxMergeDocs=solrConfig.getInt(prefix+"/maxMergeDocs",def.maxMergeDocs);
maxIndexingThreads=solrConfig.getInt(prefix+"/maxIndexingThreads",def.maxIndexingThreads);
@@ -133,14 +150,21 @@ public class SolrIndexConfig {
mergeSchedulerInfo = getPluginInfo(prefix + "/mergeScheduler", solrConfig, def.mergeSchedulerInfo);
mergePolicyInfo = getPluginInfo(prefix + "/mergePolicy", solrConfig, def.mergePolicyInfo);
- termIndexInterval = solrConfig.getInt(prefix + "/termIndexInterval", def.termIndexInterval);
-
+ String val = solrConfig.get(prefix + "/termIndexInterval", null);
+ if (val != null) {
+ throw new IllegalArgumentException("Illegal parameter 'termIndexInterval'");
+ }
+
boolean infoStreamEnabled = solrConfig.getBool(prefix + "/infoStream", false);
if(infoStreamEnabled) {
- infoStreamFile= solrConfig.get(prefix + "/infoStream/@file", null);
- log.info("IndexWriter infoStream debug log is enabled: " + infoStreamFile);
+ String infoStreamFile = solrConfig.get(prefix + "/infoStream/@file", null);
+ if (infoStreamFile == null) {
+ log.info("IndexWriter infoStream solr logging is enabled");
+ infoStream = new LoggingInfoStream();
+ } else {
+ throw new IllegalArgumentException("Remove @file from <infoStream> to output messages to solr's logfile");
+ }
}
-
mergedSegmentWarmerInfo = getPluginInfo(prefix + "/mergedSegmentWarmer", solrConfig, def.mergedSegmentWarmerInfo);
if (mergedSegmentWarmerInfo != null && solrConfig.reopenReaders == false) {
throw new IllegalArgumentException("Supplying a mergedSegmentWarmer will do nothing since reopenReaders is false");
@@ -179,15 +203,17 @@ public class SolrIndexConfig {
if (ramBufferSizeMB != -1)
iwc.setRAMBufferSizeMB(ramBufferSizeMB);
- if (termIndexInterval != -1)
- iwc.setTermIndexInterval(termIndexInterval);
-
if (writeLockTimeout != -1)
iwc.setWriteLockTimeout(writeLockTimeout);
iwc.setSimilarity(schema.getSimilarity());
iwc.setMergePolicy(buildMergePolicy(schema));
iwc.setMergeScheduler(buildMergeScheduler(schema));
+ iwc.setInfoStream(infoStream);
+
+ // do this after buildMergePolicy since the backcompat logic
+ // there may modify the effective useCompoundFile
+ iwc.setUseCompoundFile(getUseCompoundFile());
if (maxIndexingThreads != -1) {
iwc.setMaxThreadStates(maxIndexingThreads);
@@ -199,13 +225,22 @@ public class SolrIndexConfig {
IndexReaderWarmer.class,
null,
new Class[] { InfoStream.class },
- new Object[] { InfoStream.NO_OUTPUT });
+ new Object[] { iwc.getInfoStream() });
iwc.setMergedSegmentWarmer(warmer);
}
return iwc;
}
+ /**
+ * Builds a MergePolicy, may also modify the value returned by
+ * getUseCompoundFile() for use by the IndexWriterConfig if
+ * "useCompoundFile" is specified as an init arg for
+ * an out of the box MergePolicy that no longer supports it
+ *
+ * @see #fixUseCFMergePolicyInitArg
+ * @see #getUseCompoundFile
+ */
private MergePolicy buildMergePolicy(IndexSchema schema) {
String mpClassName = mergePolicyInfo == null ? defaultMergePolicyClassName : mergePolicyInfo.className;
@@ -213,25 +248,31 @@ public class SolrIndexConfig {
if (policy instanceof LogMergePolicy) {
LogMergePolicy logMergePolicy = (LogMergePolicy) policy;
+ fixUseCFMergePolicyInitArg(LogMergePolicy.class);
if (maxMergeDocs != -1)
logMergePolicy.setMaxMergeDocs(maxMergeDocs);
- logMergePolicy.setUseCompoundFile(useCompoundFile);
+ logMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
if (mergeFactor != -1)
logMergePolicy.setMergeFactor(mergeFactor);
+
+
} else if (policy instanceof TieredMergePolicy) {
TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) policy;
+ fixUseCFMergePolicyInitArg(TieredMergePolicy.class);
- tieredMergePolicy.setUseCompoundFile(useCompoundFile);
+ tieredMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
if (mergeFactor != -1) {
tieredMergePolicy.setMaxMergeAtOnce(mergeFactor);
tieredMergePolicy.setSegmentsPerTier(mergeFactor);
}
- } else {
- log.warn("Use of compound file format or mergefactor cannot be configured if merge policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured policy's defaults will be used.");
+
+
+ } else if (mergeFactor != -1) {
+ log.warn("Use of <mergeFactor> cannot be configured if merge policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured policy's defaults will be used.");
}
if (mergePolicyInfo != null)
@@ -244,9 +285,58 @@ public class SolrIndexConfig {
String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
MergeScheduler scheduler = schema.getResourceLoader().newInstance(msClassName, MergeScheduler.class);
- if (mergeSchedulerInfo != null)
- SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs);
+ if (mergeSchedulerInfo != null) {
+ // LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads
+ // if someone has them configured.
+ if (scheduler instanceof ConcurrentMergeScheduler) {
+ NamedList args = mergeSchedulerInfo.initArgs.clone();
+ Integer maxMergeCount = (Integer) args.remove("maxMergeCount");
+ if (maxMergeCount == null) {
+ maxMergeCount = ((ConcurrentMergeScheduler) scheduler).getMaxMergeCount();
+ }
+ Integer maxThreadCount = (Integer) args.remove("maxThreadCount");
+ if (maxThreadCount == null) {
+ maxThreadCount = ((ConcurrentMergeScheduler) scheduler).getMaxThreadCount();
+ }
+ ((ConcurrentMergeScheduler)scheduler).setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
+ SolrPluginUtils.invokeSetters(scheduler, args);
+ } else {
+ SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs);
+ }
+ }
return scheduler;
}
+
+ public boolean getUseCompoundFile() {
+ return effectiveUseCompountFileSetting;
+ }
+
+ /**
+ * Lucene 4.4 removed the setUseCompoundFile(boolean) method from the two
+ * conrete MergePolicies provided with Lucene/Solr and added it to the
+ * IndexWRiterConfig.
+ * In the event that users have a value explicitly configured for this
+ * setter in their MergePolicy init args, we remove it from the MergePolicy
+ * init args, update the 'effective' useCompoundFile setting used by the
+ * IndexWriterConfig, and warn about discontinuing to use this init arg.
+ *
+ * @see #getUseCompoundFile
+ */
+ private void fixUseCFMergePolicyInitArg(Class c) {
+
+ if (null == mergePolicyInfo || null == mergePolicyInfo.initArgs) return;
+
+ Object useCFSArg = mergePolicyInfo.initArgs.remove("useCompoundFile");
+ if (null != useCFSArg) {
+ log.warn("Ignoring 'useCompoundFile' specified as an init arg for the <mergePolicy> since it is no directly longer supported by " + c.getSimpleName());
+ if (useCFSArg instanceof Boolean) {
+ boolean cfs = ((Boolean)useCFSArg).booleanValue();
+ log.warn("Please update your config to specify <useCompoundFile>"+cfs+"</useCompoundFile> directly in your <indexConfig> settings.");
+ effectiveUseCompountFileSetting = cfs;
+ } else {
+ log.error("MergePolicy's 'useCompoundFile' init arg is not a boolean, can not apply back compat logic to apply to the IndexWriterConfig: " + useCFSArg.toString());
+ }
+ }
+ }
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Sun Aug 11 12:19:13 2013
@@ -77,7 +77,7 @@ public class SolrIndexWriter extends Ind
super(directory,
config.toIndexWriterConfig(schema).
setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND).
- setIndexDeletionPolicy(delPolicy).setCodec(codec).setInfoStream(toInfoStream(config))
+ setIndexDeletionPolicy(delPolicy).setCodec(codec)
);
log.debug("Opened Writer " + name);
this.name = name;
@@ -88,20 +88,6 @@ public class SolrIndexWriter extends Ind
this.directoryFactory = factory;
}
- private static InfoStream toInfoStream(SolrIndexConfig config) throws IOException {
- String infoStreamFile = config.infoStreamFile;
- if (infoStreamFile != null) {
- File f = new File(infoStreamFile);
- File parent = f.getParentFile();
- if (parent != null) parent.mkdirs();
- FileOutputStream fos = new FileOutputStream(f, true);
- return new PrintStreamInfoStream(new PrintStream(fos, true, "UTF-8"));
- } else {
- return InfoStream.NO_OUTPUT;
- }
- }
-
-
/**
* use DocumentBuilder now...
* private final void addField(Document doc, String name, String val) {
@@ -164,11 +150,8 @@ public class SolrIndexWriter extends Ind
if (infoStream != null) {
infoStream.close();
}
-
isClosed = true;
-
directoryFactory.release(directory);
-
numCloses.incrementAndGet();
}
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/TransactionLog.java Sun Aug 11 12:19:13 2013
@@ -17,15 +17,6 @@
package org.apache.solr.update;
-import org.apache.lucene.util.BytesRef;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.FastInputStream;
-import org.apache.solr.common.util.FastOutputStream;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
@@ -34,15 +25,23 @@ import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.DataInputInputStream;
+import org.apache.solr.common.util.FastInputStream;
+import org.apache.solr.common.util.FastOutputStream;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Log Format: List{Operation, Version, ...}
* ADD, VERSION, DOC
@@ -75,7 +74,7 @@ public class TransactionLog {
FastOutputStream fos; // all accesses to this stream should be synchronized on "this" (The TransactionLog)
int numRecords;
- volatile boolean deleteOnClose = true; // we can delete old tlogs since they are currently only used for real-time-get (and in the future, recovery)
+ protected volatile boolean deleteOnClose = true; // we can delete old tlogs since they are currently only used for real-time-get (and in the future, recovery)
AtomicInteger refcount = new AtomicInteger(1);
Map<String,Integer> globalStringMap = new HashMap<String, Integer>();
@@ -98,7 +97,7 @@ public class TransactionLog {
};
public class LogCodec extends JavaBinCodec {
- public LogCodec() {
+ public LogCodec(JavaBinCodec.ObjectResolver resolver) {
super(resolver);
}
@@ -121,7 +120,7 @@ public class TransactionLog {
}
@Override
- public String readExternString(FastInputStream fis) throws IOException {
+ public String readExternString(DataInputInputStream fis) throws IOException {
int idx = readSize(fis);
if (idx != 0) {// idx != 0 is the index of the extern string
// no need to synchronize globalStringList - it's only updated before the first record is written to the log
@@ -191,6 +190,9 @@ public class TransactionLog {
}
}
+ // for subclasses
+ protected TransactionLog() {}
+
/** Returns the number of records in the log (currently includes the header and an optional commit).
* Note: currently returns 0 for reopened existing log files.
*/
@@ -245,7 +247,7 @@ public class TransactionLog {
public long writeData(Object o) {
- LogCodec codec = new LogCodec();
+ LogCodec codec = new LogCodec(resolver);
try {
long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
codec.init(fos);
@@ -260,7 +262,7 @@ public class TransactionLog {
private void readHeader(FastInputStream fis) throws IOException {
// read existing header
fis = fis != null ? fis : new ChannelFastInputStream(channel, 0);
- LogCodec codec = new LogCodec();
+ LogCodec codec = new LogCodec(resolver);
Map header = (Map)codec.unmarshal(fis);
fis.readInt(); // skip size
@@ -276,7 +278,7 @@ public class TransactionLog {
}
}
- private void addGlobalStrings(Collection<String> strings) {
+ protected void addGlobalStrings(Collection<String> strings) {
if (strings == null) return;
int origSize = globalStringMap.size();
for (String s : strings) {
@@ -297,7 +299,7 @@ public class TransactionLog {
}
}
- private void writeLogHeader(LogCodec codec) throws IOException {
+ protected void writeLogHeader(LogCodec codec) throws IOException {
long pos = fos.size();
assert pos == 0;
@@ -309,7 +311,7 @@ public class TransactionLog {
endRecord(pos);
}
- private void endRecord(long startRecordPosition) throws IOException {
+ protected void endRecord(long startRecordPosition) throws IOException {
fos.writeInt((int)(fos.size() - startRecordPosition));
numRecords++;
}
@@ -333,7 +335,7 @@ public class TransactionLog {
int lastAddSize;
public long write(AddUpdateCommand cmd, int flags) {
- LogCodec codec = new LogCodec();
+ LogCodec codec = new LogCodec(resolver);
SolrInputDocument sdoc = cmd.getSolrInputDocument();
try {
@@ -375,7 +377,7 @@ public class TransactionLog {
}
public long writeDelete(DeleteUpdateCommand cmd, int flags) {
- LogCodec codec = new LogCodec();
+ LogCodec codec = new LogCodec(resolver);
try {
checkWriteHeader(codec, null);
@@ -405,7 +407,7 @@ public class TransactionLog {
}
public long writeDeleteByQuery(DeleteUpdateCommand cmd, int flags) {
- LogCodec codec = new LogCodec();
+ LogCodec codec = new LogCodec(resolver);
try {
checkWriteHeader(codec, null);
@@ -431,7 +433,7 @@ public class TransactionLog {
public long writeCommit(CommitUpdateCommand cmd, int flags) {
- LogCodec codec = new LogCodec();
+ LogCodec codec = new LogCodec(resolver);
synchronized (this) {
try {
long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
@@ -479,7 +481,7 @@ public class TransactionLog {
}
ChannelFastInputStream fis = new ChannelFastInputStream(channel, pos);
- LogCodec codec = new LogCodec();
+ LogCodec codec = new LogCodec(resolver);
return codec.readVal(fis);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@@ -529,7 +531,7 @@ public class TransactionLog {
}
}
- private void close() {
+ protected void close() {
try {
if (debug) {
log.debug("Closing tlog" + this);
@@ -570,19 +572,22 @@ public class TransactionLog {
/** Returns a single threaded reverse reader */
public ReverseReader getReverseReader() throws IOException {
- return new ReverseReader();
+ return new FSReverseReader();
}
public class LogReader {
- ChannelFastInputStream fis;
- private LogCodec codec = new LogCodec();
+ private ChannelFastInputStream fis;
+ private LogCodec codec = new LogCodec(resolver);
public LogReader(long startingPos) {
incref();
fis = new ChannelFastInputStream(channel, startingPos);
}
+ // for classes that extend
+ protected LogReader() {}
+
/** Returns the next object from the log, or null if none available.
*
* @return The log record, or null if EOF
@@ -638,11 +643,32 @@ public class TransactionLog {
}
- public class ReverseReader {
+ public abstract class ReverseReader {
+
+
+
+ /** Returns the next object from the log, or null if none available.
+ *
+ * @return The log record, or null if EOF
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public abstract Object next() throws IOException;
+
+ /* returns the position in the log file of the last record returned by next() */
+ public abstract long position();
+ public abstract void close();
+
+ @Override
+ public abstract String toString() ;
+
+
+ }
+
+ public class FSReverseReader extends ReverseReader {
ChannelFastInputStream fis;
- private LogCodec codec = new LogCodec() {
+ private LogCodec codec = new LogCodec(resolver) {
@Override
- public SolrInputDocument readSolrInputDocument(FastInputStream dis) {
+ public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
// Given that the SolrInputDocument is last in an add record, it's OK to just skip
// reading it completely.
return null;
@@ -652,7 +678,7 @@ public class TransactionLog {
int nextLength; // length of the next record (the next one closer to the start of the log file)
long prevPos; // where we started reading from last time (so prevPos - nextLength == start of next record)
- public ReverseReader() throws IOException {
+ public FSReverseReader() throws IOException {
incref();
long sz;
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateHandler.java Sun Aug 11 12:19:13 2013
@@ -18,10 +18,11 @@
package org.apache.solr.update;
-import java.io.File;
import java.io.IOException;
import java.util.Vector;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.HdfsDirectoryFactory;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrEventListener;
@@ -52,7 +53,7 @@ public abstract class UpdateHandler impl
protected Vector<SolrEventListener> softCommitCallbacks = new Vector<SolrEventListener>();
protected Vector<SolrEventListener> optimizeCallbacks = new Vector<SolrEventListener>();
- protected volatile UpdateLog ulog;
+ protected final UpdateLog ulog;
private void parseEventListeners() {
final Class<SolrEventListener> clazz = SolrEventListener.class;
@@ -71,34 +72,6 @@ public abstract class UpdateHandler impl
}
}
-
- private void initLog(PluginInfo ulogPluginInfo) {
- if (ulogPluginInfo != null && ulogPluginInfo.isEnabled()) {
- ulog = new UpdateLog();
- ulog.init(ulogPluginInfo);
- // ulog = core.createInitInstance(ulogPluginInfo, UpdateLog.class, "update log", "solr.NullUpdateLog");
- ulog.init(this, core);
- }
- }
-
- // not thread safe - for startup
- private void clearLog(PluginInfo ulogPluginInfo) {
- if (ulogPluginInfo == null) return;
- File tlogDir = UpdateLog.getTlogDir(core, ulogPluginInfo);
- log.info("Clearing tlog files, tlogDir=" + tlogDir);
- if (tlogDir.exists()) {
- String[] files = UpdateLog.getLogList(tlogDir);
- for (String file : files) {
- File f = new File(tlogDir, file);
- boolean s = f.delete();
- if (!s) {
- log.error("Could not remove tlog file:" + f.getAbsolutePath());
- //throw new SolrException(ErrorCode.SERVER_ERROR, "Could not remove tlog file:" + f.getAbsolutePath());
- }
- }
- }
- }
-
protected void callPostCommitCallbacks() {
for (SolrEventListener listener : commitCallbacks) {
listener.postCommit();
@@ -127,14 +100,43 @@ public abstract class UpdateHandler impl
idFieldType = idField!=null ? idField.getType() : null;
parseEventListeners();
PluginInfo ulogPluginInfo = core.getSolrConfig().getPluginInfo(UpdateLog.class.getName());
- if (!core.isReloaded() && !core.getDirectoryFactory().isPersistent()) {
- clearLog(ulogPluginInfo);
- }
- if (updateLog == null) {
- initLog(ulogPluginInfo);
+
+
+ if (updateLog == null && ulogPluginInfo != null && ulogPluginInfo.isEnabled()) {
+ String dataDir = (String)ulogPluginInfo.initArgs.get("dir");
+
+ String ulogDir = core.getCoreDescriptor().getUlogDir();
+ if (ulogDir != null) {
+ dataDir = ulogDir;
+ }
+ if (dataDir == null || dataDir.length()==0) {
+ dataDir = core.getDataDir();
+ }
+
+ if (dataDir != null && dataDir.startsWith("hdfs:/")) {
+ DirectoryFactory dirFactory = core.getDirectoryFactory();
+ if (dirFactory instanceof HdfsDirectoryFactory) {
+ ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir());
+ } else {
+ ulog = new HdfsUpdateLog();
+ }
+
+ } else {
+ ulog = new UpdateLog();
+ }
+
+ if (!core.isReloaded() && !core.getDirectoryFactory().isPersistent()) {
+ ulog.clearLog(core, ulogPluginInfo);
+ }
+
+ ulog.init(ulogPluginInfo);
+
+ ulog.init(this, core);
} else {
- this.ulog = updateLog;
+ ulog = updateLog;
}
+ // ulog.init() when reusing an existing log is deferred (currently at the end of the DUH2 constructor
+
}
/**
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/UpdateLog.java Sun Aug 11 12:19:13 2013
@@ -17,15 +17,38 @@
package org.apache.solr.update;
+import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.fs.FileSystem;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
@@ -34,9 +57,6 @@ import org.apache.solr.request.SolrReque
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
-import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
-import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
-import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.util.DefaultSolrThreadFactory;
@@ -45,15 +65,6 @@ import org.apache.solr.util.plugin.Plugi
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
-
/** @lucene.experimental */
public class UpdateLog implements PluginInfoInitialized {
@@ -64,6 +75,10 @@ public class UpdateLog implements Plugin
public boolean debug = log.isDebugEnabled();
public boolean trace = log.isTraceEnabled();
+ // TODO: hack
+ public FileSystem getFs() {
+ return null;
+ }
public enum SyncLevel { NONE, FLUSH, FSYNC;
public static SyncLevel getSyncLevel(String level){
@@ -108,27 +123,27 @@ public class UpdateLog implements Plugin
}
long id = -1;
- private State state = State.ACTIVE;
- private int operationFlags; // flags to write in the transaction log with operations (i.e. FLAG_GAP)
+ protected State state = State.ACTIVE;
+ protected int operationFlags; // flags to write in the transaction log with operations (i.e. FLAG_GAP)
- private TransactionLog tlog;
- private TransactionLog prevTlog;
- private Deque<TransactionLog> logs = new LinkedList<TransactionLog>(); // list of recent logs, newest first
- private LinkedList<TransactionLog> newestLogsOnStartup = new LinkedList<TransactionLog>();
- private int numOldRecords; // number of records in the recent logs
-
- private Map<BytesRef,LogPtr> map = new HashMap<BytesRef, LogPtr>();
- private Map<BytesRef,LogPtr> prevMap; // used while committing/reopening is happening
- private Map<BytesRef,LogPtr> prevMap2; // used while committing/reopening is happening
- private TransactionLog prevMapLog; // the transaction log used to look up entries found in prevMap
- private TransactionLog prevMapLog2; // the transaction log used to look up entries found in prevMap
+ protected TransactionLog tlog;
+ protected TransactionLog prevTlog;
+ protected Deque<TransactionLog> logs = new LinkedList<TransactionLog>(); // list of recent logs, newest first
+ protected LinkedList<TransactionLog> newestLogsOnStartup = new LinkedList<TransactionLog>();
+ protected int numOldRecords; // number of records in the recent logs
+
+ protected Map<BytesRef,LogPtr> map = new HashMap<BytesRef, LogPtr>();
+ protected Map<BytesRef,LogPtr> prevMap; // used while committing/reopening is happening
+ protected Map<BytesRef,LogPtr> prevMap2; // used while committing/reopening is happening
+ protected TransactionLog prevMapLog; // the transaction log used to look up entries found in prevMap
+ protected TransactionLog prevMapLog2; // the transaction log used to look up entries found in prevMap
- private final int numDeletesToKeep = 1000;
- private final int numDeletesByQueryToKeep = 100;
+ protected final int numDeletesToKeep = 1000;
+ protected final int numDeletesByQueryToKeep = 100;
public final int numRecordsToKeep = 100;
// keep track of deletes only... this is not updated on an add
- private LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
+ protected LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
@Override
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > numDeletesToKeep;
@@ -145,21 +160,21 @@ public class UpdateLog implements Plugin
}
}
- private LinkedList<DBQ> deleteByQueries = new LinkedList<DBQ>();
+ protected LinkedList<DBQ> deleteByQueries = new LinkedList<DBQ>();
- private String[] tlogFiles;
- private File tlogDir;
- private Collection<String> globalStrings;
+ protected String[] tlogFiles;
+ protected File tlogDir;
+ protected Collection<String> globalStrings;
- private String dataDir;
- private String lastDataDir;
+ protected String dataDir;
+ protected String lastDataDir;
- private VersionInfo versionInfo;
+ protected VersionInfo versionInfo;
- private SyncLevel defaultSyncLevel = SyncLevel.FLUSH;
+ protected SyncLevel defaultSyncLevel = SyncLevel.FLUSH;
volatile UpdateHandler uhandler; // a core reload can change this reference!
- private volatile boolean cancelApplyBufferUpdate;
+ protected volatile boolean cancelApplyBufferUpdate;
List<Long> startingVersions;
int startingOperation; // last operation in the logs on startup
@@ -189,13 +204,17 @@ public class UpdateLog implements Plugin
defaultSyncLevel = SyncLevel.getSyncLevel((String)info.initArgs.get("syncLevel"));
}
+ /* Note, when this is called, uhandler is not completely constructed.
+ * This must be called when a new log is created, or
+ * for an existing log whenever the core or update handler changes.
+ */
public void init(UpdateHandler uhandler, SolrCore core) {
// ulogDir from CoreDescriptor overrides
String ulogDir = core.getCoreDescriptor().getUlogDir();
if (ulogDir != null) {
dataDir = ulogDir;
}
-
+
if (dataDir == null || dataDir.length()==0) {
dataDir = core.getDataDir();
}
@@ -276,8 +295,8 @@ public class UpdateLog implements Plugin
}
- public File getLogDir() {
- return tlogDir;
+ public String getLogDir() {
+ return tlogDir.getAbsolutePath();
}
public List<Long> getStartingVersions() {
@@ -291,7 +310,7 @@ public class UpdateLog implements Plugin
/* Takes over ownership of the log, keeping it until no longer needed
and then decrementing it's reference and dropping it.
*/
- private void addOldLog(TransactionLog oldLog, boolean removeOld) {
+ protected void addOldLog(TransactionLog oldLog, boolean removeOld) {
if (oldLog == null) return;
numOldRecords += oldLog.numRecords();
@@ -322,7 +341,7 @@ public class UpdateLog implements Plugin
}
- public static String[] getLogList(File directory) {
+ public String[] getLogList(File directory) {
final String prefix = TLOG_NAME+'.';
String[] names = directory.list(new FilenameFilter() {
@Override
@@ -330,6 +349,9 @@ public class UpdateLog implements Plugin
return name.startsWith(prefix);
}
});
+ if (names == null) {
+ throw new RuntimeException(new FileNotFoundException(directory.getAbsolutePath()));
+ }
Arrays.sort(names);
return names;
}
@@ -540,7 +562,7 @@ public class UpdateLog implements Plugin
}
}
- private void newMap() {
+ protected void newMap() {
prevMap2 = prevMap;
prevMapLog2 = prevMapLog;
@@ -793,7 +815,7 @@ public class UpdateLog implements Plugin
}
- private void ensureLog() {
+ protected void ensureLog() {
if (tlog == null) {
String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN, TLOG_NAME, id);
tlog = new TransactionLog(new File(tlogDir, newLogName), globalStrings);
@@ -968,6 +990,8 @@ public class UpdateLog implements Plugin
log.warn("Exception reverse reading log", ex);
break;
}
+
+ numUpdates++;
}
} catch (IOException e) {
@@ -1139,7 +1163,7 @@ public class UpdateLog implements Plugin
- private RecoveryInfo recoveryInfo;
+ protected RecoveryInfo recoveryInfo;
class LogReplayer implements Runnable {
private Logger loglog = log; // set to something different?
@@ -1416,7 +1440,7 @@ public class UpdateLog implements Plugin
}
}
- public static File getTlogDir(SolrCore core, PluginInfo info) {
+ protected String getTlogDir(SolrCore core, PluginInfo info) {
String dataDir = (String) info.initArgs.get("dir");
String ulogDir = core.getCoreDescriptor().getUlogDir();
@@ -1427,11 +1451,30 @@ public class UpdateLog implements Plugin
if (dataDir == null || dataDir.length() == 0) {
dataDir = core.getDataDir();
}
-
- return new File(dataDir, TLOG_NAME);
+
+ return dataDir + "/" + TLOG_NAME;
+ }
+
+ /**
+ * Clears the logs on the file system. Only call before init.
+ *
+ * @param core the SolrCore
+ * @param ulogPluginInfo the init info for the UpdateHandler
+ */
+ public void clearLog(SolrCore core, PluginInfo ulogPluginInfo) {
+ if (ulogPluginInfo == null) return;
+ File tlogDir = new File(getTlogDir(core, ulogPluginInfo));
+ if (tlogDir.exists()) {
+ String[] files = getLogList(tlogDir);
+ for (String file : files) {
+ File f = new File(tlogDir, file);
+ boolean s = f.delete();
+ if (!s) {
+ log.error("Could not remove tlog file:" + f);
+ }
+ }
+ }
}
}
-
-
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java Sun Aug 11 12:19:13 2013
@@ -195,25 +195,13 @@ public class CloneFieldUpdateProcessorFa
srcSelector =
FieldMutatingUpdateProcessor.createFieldNameSelector
- (core.getResourceLoader(),
- core,
- srcInclusions.fieldName,
- srcInclusions.typeName,
- srcInclusions.typeClass,
- srcInclusions.fieldRegex,
- FieldMutatingUpdateProcessor.SELECT_NO_FIELDS);
+ (core.getResourceLoader(), core, srcInclusions, FieldMutatingUpdateProcessor.SELECT_NO_FIELDS);
for (SelectorParams exc : srcExclusions) {
srcSelector = FieldMutatingUpdateProcessor.wrap
(srcSelector,
FieldMutatingUpdateProcessor.createFieldNameSelector
- (core.getResourceLoader(),
- core,
- exc.fieldName,
- exc.typeName,
- exc.typeClass,
- exc.fieldRegex,
- FieldMutatingUpdateProcessor.SELECT_NO_FIELDS));
+ (core.getResourceLoader(), core, exc, FieldMutatingUpdateProcessor.SELECT_NO_FIELDS));
}
}