svn commit: r1658078 - in /lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler: ReplicationHandler.java SnapPuller.java

[ma...@apache.org]

Author: markrmiller
Date: Sat Feb  7 17:00:08 2015
New Revision: 1658078

URL: http://svn.apache.org/r1658078
Log:
SOLR-6920, SOLR-6640: When we so not have a checksum for a file, always download files under 100kb and other small improvements.

Modified:
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SnapPuller.java

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1658078&r1=1658077&r2=1658078&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Sat Feb  7 17:00:08 2015
@@ -449,12 +449,17 @@ public class ReplicationHandler extends
       }
 
       // add the segments_N file
+      
       Map<String,Object> fileMeta = new HashMap<>();
       fileMeta.put(NAME, infos.getSegmentsFileName());
       fileMeta.put(SIZE, dir.fileLength(infos.getSegmentsFileName()));
       if (infos.getId() != null) {
         try (final IndexInput in = dir.openInput(infos.getSegmentsFileName(), IOContext.READONCE)) {
-          fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in));
+          try {
+            fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in));
+          } catch(Exception e) {
+             LOG.warn("Could not read checksum from index file.", e);
+          }
         }
       }
       result.add(fileMeta);

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1658078&r1=1658077&r2=1658078&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Sat Feb  7 17:00:08 2015
@@ -802,10 +802,10 @@ public class SnapPuller {
     }
     for (Map<String,Object> file : filesToDownload) {
       String filename = (String) file.get(NAME);
-      CompareResult compareResult = compareFile(indexDir, filename, (Long) file.get(SIZE), (Long) file.get(CHECKSUM));
+      long size = (Long) file.get(SIZE);
+      CompareResult compareResult = compareFile(indexDir, filename, size, (Long) file.get(CHECKSUM));
       if (!compareResult.equal || downloadCompleteIndex
-          || (!compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv")
-          || filename.startsWith("segments_")))) {
+          || filesToAlwaysDownloadIfChecksumFails(filename, size, compareResult)) {
         dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file,
             (String) file.get(NAME), false, latestGeneration);
         currentFile = file;
@@ -817,6 +817,14 @@ public class SnapPuller {
       }
     }
   }
+  
+  private boolean filesToAlwaysDownloadIfChecksumFails(String filename,
+      long size, CompareResult compareResult) {
+    // without checksums to compare, we always download .si, .liv, segments_N,
+    // and any file under 100kb
+    return !compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv")
+    || filename.startsWith("segments_") || size < 100000);
+  }
 
   static class CompareResult {
     boolean equal = false;
@@ -830,24 +838,33 @@ public class SnapPuller {
         long indexFileLen = indexInput.length();
         long indexFileChecksum = 0;
         
-        try {
-          indexFileChecksum = CodecUtil.retrieveChecksum(indexInput);
-          compareResult.checkSummed = true;
-        } catch (Exception e) {
-          LOG.warn("Could not retrieve checksum from file.", e);
+        if (backupIndexFileChecksum != null) {
+          try {
+            indexFileChecksum = CodecUtil.retrieveChecksum(indexInput);
+            compareResult.checkSummed = true;
+          } catch (Exception e) {
+            LOG.warn("Could not retrieve checksum from file.", e);
+          }
+        }
+        
+        if (!compareResult.checkSummed) {
+          // we don't have checksums to compare
           
           if (indexFileLen == backupIndexFileLen) {
             compareResult.equal = true;
             return compareResult;
           } else {
-            LOG.warn("File {} did not match. expected checksum is {} and actual is checksum {}. " +
-                "expected length is {} and actual length is {}", filename, backupIndexFileChecksum, indexFileChecksum,
+            LOG.warn(
+                "File {} did not match. "  + "expected length is {} and actual length is {}",
+                filename, backupIndexFileChecksum, indexFileChecksum,
                 backupIndexFileLen, indexFileLen);
             compareResult.equal = false;
             return compareResult;
           }
         }
         
+        // we have checksums to compare
+        
         if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) {
           compareResult.equal = true;
           return compareResult;