You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2017/01/03 19:25:25 UTC
lucene-solr:branch_6x: SOLR-9859: replication.properties cannot be
updated after being written and neither eplication.properties or
ndex.properties are durable in the face of a crash.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 812070a77 -> c6ea17186
SOLR-9859: replication.properties cannot be updated after being written and neither eplication.properties or ndex.properties are durable in the face of a crash.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c6ea1718
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c6ea1718
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c6ea1718
Branch: refs/heads/branch_6x
Commit: c6ea1718675beebdddd988b08b2b213155d8b20f
Parents: 812070a
Author: markrmiller <ma...@apache.org>
Authored: Tue Jan 3 13:47:59 2017 -0500
Committer: markrmiller <ma...@apache.org>
Committed: Tue Jan 3 14:25:15 2017 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../org/apache/solr/core/DirectoryFactory.java | 15 ++++++++++++
.../apache/solr/core/HdfsDirectoryFactory.java | 9 +++++++
.../solr/core/StandardDirectoryFactory.java | 22 +++++++++++++++++
.../org/apache/solr/handler/IndexFetcher.java | 25 +++++++++++---------
.../solr/handler/TestReplicationHandler.java | 7 +++++-
6 files changed, 69 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c6ea1718/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 09bc1aa..0b3a38a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -246,6 +246,9 @@ Bug Fixes
* SOLR-9154: Fix DirectSolrSpellChecker to work when added through the Config API. (Anshum Gupta)
+* SOLR-9859: replication.properties cannot be updated after being written and neither replication.properties or
+ index.properties are durable in the face of a crash. (Pushkar Raste, Chris de Kok, Cao Manh Dat, Mark Miller)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c6ea1718/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index 228260a..396a30d 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.Closeable;
import java.io.File;
import java.io.FileFilter;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
@@ -184,6 +185,20 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
fromDir.deleteFile(fileName);
}
+ // sub classes perform an atomic rename if possible, otherwise fall back to delete + rename
+ // this is important to support for index roll over durability after crashes
+ public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException {
+ try {
+ dir.deleteFile(toName);
+ } catch (FileNotFoundException e) {
+
+ } catch (Exception e) {
+ log.error("Exception deleting file", e);
+ }
+
+ dir.rename(fileName, toName);
+ }
+
/**
* Returns the Directory for a given path, using the specified rawLockType.
* Will return the same Directory instance for the same path.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c6ea1718/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index b003287..d481e03 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -29,8 +29,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.security.UserGroupInformation;
@@ -568,4 +570,11 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
}
}
+
+ // perform an atomic rename if possible
+ public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException {
+ String hdfsDirPath = getPath(dir);
+ FileContext fileContext = FileContext.getFileContext(getConf());
+ fileContext.rename(new Path(hdfsDirPath + "/" + fileName), new Path(hdfsDirPath + "/" + toName), Options.Rename.OVERWRITE);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c6ea1718/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
index d1e04ed..e3a7dd0 100644
--- a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
@@ -18,6 +18,11 @@ package org.apache.solr.core;
import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
import java.util.Locale;
import org.apache.commons.io.FileUtils;
@@ -153,5 +158,22 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory {
return baseDir;
}
+
+ // perform an atomic rename if possible
+ public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException {
+ Directory baseDir = getBaseDir(dir);
+ if (baseDir instanceof FSDirectory) {
+ Path path = ((FSDirectory) baseDir).getDirectory().toAbsolutePath();
+ try {
+ Files.move(FileSystems.getDefault().getPath(path.toString(), fileName),
+ FileSystems.getDefault().getPath(path.toString(), toName), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ } catch (AtomicMoveNotSupportedException e) {
+ Files.move(FileSystems.getDefault().getPath(path.toString(), fileName),
+ FileSystems.getDefault().getPath(path.toString(), toName), StandardCopyOption.REPLACE_EXISTING);
+ }
+ } else {
+ super.renameWithOverwrite(dir, fileName, toName);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c6ea1718/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 398553c..432bb8b 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -671,15 +671,19 @@ public class IndexFetcher {
sb = readToStringBuilder(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST));
props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString());
}
-
- final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
+
+
+ String tmpFileName = REPLICATION_PROPERTIES + "." + System.nanoTime();
+ final IndexOutput out = dir.createOutput(tmpFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8);
try {
props.store(outFile, "Replication details");
- dir.sync(Collections.singleton(REPLICATION_PROPERTIES));
+ dir.sync(Collections.singleton(tmpFileName));
} finally {
IOUtils.closeQuietly(outFile);
}
+
+ solrCore.getDirectoryFactory().renameWithOverwrite(dir, tmpFileName, REPLICATION_PROPERTIES);
} catch (Exception e) {
LOG.warn("Exception while updating statistics", e);
} finally {
@@ -1192,24 +1196,23 @@ public class IndexFetcher {
IOUtils.closeQuietly(is);
}
}
- try {
- dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
- } catch (IOException e) {
- // no problem
- }
- final IndexOutput out = dir.createOutput(IndexFetcher.INDEX_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
+
+ String tmpFileName = IndexFetcher.INDEX_PROPERTIES + "." + System.nanoTime();
+ final IndexOutput out = dir.createOutput(tmpFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
p.put("index", tmpIdxDirName);
Writer os = null;
try {
os = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8);
- p.store(os, IndexFetcher.INDEX_PROPERTIES);
- dir.sync(Collections.singleton(INDEX_PROPERTIES));
+ p.store(os, tmpFileName);
+ dir.sync(Collections.singleton(tmpFileName));
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unable to write " + IndexFetcher.INDEX_PROPERTIES, e);
} finally {
IOUtils.closeQuietly(os);
}
+
+ solrCore.getDirectoryFactory().renameWithOverwrite(dir, tmpFileName, IndexFetcher.INDEX_PROPERTIES);
return true;
} catch (IOException e1) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c6ea1718/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index 5f196bf..0c95baf 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -35,6 +35,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
+import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -308,7 +309,11 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
// check details on the slave a couple of times before & after fetching
for (int i = 0; i < 3; i++) {
NamedList<Object> details = getDetails(slaveClient);
-
+ List replicatedAtCount = (List) ((NamedList) details.get("slave")).get("indexReplicatedAtList");
+ if (i > 0) {
+ assertEquals(i, replicatedAtCount.size());
+ }
+
assertEquals("slave isMaster?",
"false", details.get("isMaster"));
assertEquals("slave isSlave?",