You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/05/08 21:15:34 UTC
[48/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12312:
Replication's IndexFetcher buf size should be initialized to an amount no
greater than the size of the file being transferred.
SOLR-12312: Replication's IndexFetcher buf size should be initialized
to an amount no greater than the size of the file being transferred.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/81f61120
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/81f61120
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/81f61120
Branch: refs/heads/jira/solr-11779
Commit: 81f611209c9b71b7538a46d047631ea164dc2a2c
Parents: b0b3293
Author: David Smiley <ds...@apache.org>
Authored: Mon May 7 14:54:11 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Mon May 7 14:54:11 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../org/apache/solr/handler/IndexFetcher.java | 40 ++++++--------------
2 files changed, 14 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/81f61120/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 48f9ad5..abc6d6c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -232,6 +232,9 @@ Optimizations
* SOLR-12066: Cleanup deleted core when node start (Cao Manh Dat)
+* SOLR-12312: Replication IndexFetcher should cap its internal buffer size when the file being transferred is small.
+ (Jeff Miller, David Smiley)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/81f61120/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 93f0edf..d26a0db 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -16,31 +16,6 @@
*/
package org.apache.solr.handler;
-import static org.apache.solr.common.params.CommonParams.JAVABIN;
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.handler.ReplicationHandler.ALIAS;
-import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
-import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
-import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE;
-import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST;
-import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION;
-import static org.apache.solr.handler.ReplicationHandler.COMMAND;
-import static org.apache.solr.handler.ReplicationHandler.COMPRESSION;
-import static org.apache.solr.handler.ReplicationHandler.CONF_FILES;
-import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT;
-import static org.apache.solr.handler.ReplicationHandler.EXTERNAL;
-import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER;
-import static org.apache.solr.handler.ReplicationHandler.FILE;
-import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
-import static org.apache.solr.handler.ReplicationHandler.GENERATION;
-import static org.apache.solr.handler.ReplicationHandler.INTERNAL;
-import static org.apache.solr.handler.ReplicationHandler.MASTER_URL;
-import static org.apache.solr.handler.ReplicationHandler.OFFSET;
-import static org.apache.solr.handler.ReplicationHandler.SIZE;
-import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO;
-import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE;
-import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES;
-
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -79,6 +54,7 @@ import java.util.zip.Adler32;
import java.util.zip.Checksum;
import java.util.zip.InflaterInputStream;
+import com.google.common.base.Strings;
import org.apache.http.client.HttpClient;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexCommit;
@@ -111,7 +87,7 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.ReplicationHandler.FileInfo;
+import org.apache.solr.handler.ReplicationHandler.*;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
@@ -128,7 +104,9 @@ import org.apache.solr.util.TestInjection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Strings;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.handler.ReplicationHandler.*;
/**
* <p> Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the
@@ -1539,7 +1517,7 @@ public class IndexFetcher {
private final long size;
private long bytesDownloaded = 0;
- private byte[] buf = new byte[1024 * 1024];
+ private byte[] buf;
private final Checksum checksum;
private int errorCount = 0;
private boolean aborted = false;
@@ -1549,6 +1527,7 @@ public class IndexFetcher {
this.file = file;
this.fileName = (String) fileDetails.get(NAME);
this.size = (Long) fileDetails.get(SIZE);
+ buf = new byte[(int)Math.min(this.size, ReplicationHandler.PACKET_SZ)];
this.solrParamOutput = solrParamOutput;
this.saveAs = saveAs;
indexGen = latestGen;
@@ -1629,8 +1608,11 @@ public class IndexFetcher {
LOG.warn("No content received for file: {}", fileName);
return NO_CONTENT;
}
- if (buf.length < packetSize)
+ //TODO consider recoding the remaining logic to not use/need buf[]; instead use the internal buffer of fis
+ if (buf.length < packetSize) {
+ //This shouldn't happen since sender should use PACKET_SZ and we init the buf based on that too
buf = new byte[packetSize];
+ }
if (checksum != null) {
//read the checksum
fis.readFully(longbytes);