You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/03/27 11:36:54 UTC
svn commit: r1788904 - in /jackrabbit/oak/branches/1.4: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/
oak-lucene/src/test/java/org/apache/j...
Author: chetanm
Date: Mon Mar 27 11:36:54 2017
New Revision: 1788904
URL: http://svn.apache.org/viewvc?rev=1788904&view=rev
Log:
OAK-4114 - Cached lucene index gets corrupted in case of unclean shutdown and journal rollback in SegmentNodeStore
Merging 1769939, 1769940
Added:
jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/
jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java (with props)
jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java (with props)
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
jackrabbit/oak/branches/1.4/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 27 11:36:54 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735109,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740349,1740360,1740625-1740626,1740774,1740837,1740879,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742125,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746634,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287
,1750457,1750462,1750465,1750495,1750626,1750809,1750886-1750887,1751396,1751410,1751419,1751445-1751446,1751478,1751748,1751753,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447-1752448,1752508,1752596,1752616,1752659,1752672,1753262,1753331-1753332,1753335-1753336,1753355,1753444,1754117,1754239,1755157,1755191,1756520,1756580,1757119,1757166,1758213,1758713,1759433,1759795,1759826,1760326,1760340,1760373,1760387,1760486,1760492,1760494,1760661-1760662,1760677,1760701,1760709,1760946,1761412,1761444,1761571,1761762,1761787,1761866,1761876,1762453,1762612,1762632,1762635,1763347,1763355-1763356,1763378,1763465,1763735,1764678,1764705,1764814,1764898,1765817,1765983,1766071,1766390,1766423,1766496,1766519,1766554,1766644,1767025,1767265,1767502,1767704,1768446,1768637,1769078,1770694,1770982,1771022,1771093,1771098,1771739,1771852,1771870,1771902,1772155,1772162,1772228,1772593,1772768,1773190,1774497,1774787,1775474,1775622,1775628,1775757,1778
112,1778423,1778968,1779137,1779478,1780388,1780424,1780538,1780543,1781068,1781075,1781386,1781846,1781907,1782476,1783066,1783089,1783104-1783105,1783619,1783720,1783738,1783855,1784023,1784130,1784251,1784574,1784689,1785283,1787074,1787217
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735109,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740349,1740360,1740625-1740626,1740774,1740837,1740879,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742125,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746634,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287
,1750457,1750462,1750465,1750495,1750626,1750809,1750886-1750887,1751396,1751410,1751419,1751445-1751446,1751478,1751748,1751753,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447-1752448,1752508,1752596,1752616,1752659,1752672,1753262,1753331-1753332,1753335-1753336,1753355,1753444,1754117,1754239,1755157,1755191,1756520,1756580,1757119,1757166,1758213,1758713,1759433,1759795,1759826,1760326,1760340,1760373,1760387,1760486,1760492,1760494,1760661-1760662,1760677,1760701,1760709,1760946,1761412,1761444,1761571,1761762,1761787,1761866,1761876,1762453,1762612,1762632,1762635,1763347,1763355-1763356,1763378,1763465,1763735,1764678,1764705,1764814,1764898,1765817,1765983,1766071,1766390,1766423,1766496,1766519,1766554,1766644,1767025,1767265,1767502,1767704,1768446,1768637,1769078,1769939-1769940,1770694,1770982,1771022,1771093,1771098,1771739,1771852,1771870,1771902,1772155,1772162,1772228,1772593,1772768,1773190,1774497,1774787,1775474,1775622,1775
628,1775757,1778112,1778423,1778968,1779137,1779478,1780388,1780424,1780538,1780543,1781068,1781075,1781386,1781846,1781907,1782476,1783066,1783089,1783104-1783105,1783619,1783720,1783738,1783855,1784023,1784130,1784251,1784574,1784689,1785283,1787074,1787217
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java?rev=1788904&r1=1788903&r2=1788904&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java (original)
+++ jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java Mon Mar 27 11:36:54 2017
@@ -63,6 +63,7 @@ import com.google.common.hash.Hashing;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.commons.concurrent.NotifyingFutureTask;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexSanityChecker;
import org.apache.jackrabbit.oak.util.PerfLogger;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -122,6 +123,8 @@ public class IndexCopier implements Copy
private final Set<LocalIndexFile> copyInProgressFiles = Collections.newSetFromMap(new ConcurrentHashMap<LocalIndexFile, Boolean>());
private final boolean prefetchEnabled;
private volatile boolean closed;
+ private final Set<String> validatedIndexPaths = Sets.newConcurrentHashSet();
+
public IndexCopier(Executor executor, File indexRootDir) throws IOException {
this(executor, indexRootDir, false);
@@ -137,11 +140,14 @@ public class IndexCopier implements Copy
public Directory wrapForRead(String indexPath, IndexDefinition definition,
Directory remote) throws IOException {
Directory local = createLocalDirForIndexReader(indexPath, definition);
+ checkIntegrity(indexPath, local, remote);
return new CopyOnReadDirectory(remote, local, prefetchEnabled, indexPath, getSharedWorkingSet(indexPath));
}
public Directory wrapForWrite(IndexDefinition definition, Directory remote, boolean reindexMode) throws IOException {
Directory local = createLocalDirForIndexWriter(definition);
+ String indexPath = definition.getIndexPathFromConfig();
+ checkIntegrity(indexPath, local, remote);
return new CopyOnWriteDirectory(remote, local, reindexMode,
getIndexPathForLogging(definition), getSharedWorkingSet(definition.getIndexPathFromConfig()));
}
@@ -250,6 +256,26 @@ public class IndexCopier implements Copy
return sharedSet;
}
+ private void checkIntegrity(String indexPath, Directory local, Directory remote) throws IOException {
+ //For pre 1.6 setups indexPath can be null in some cases
+ if (indexPath == null) {
+ return;
+ }
+ if (validatedIndexPaths.contains(indexPath)){
+ return;
+ }
+
+ //The integrity check needs to be done for the very first time at startup when
+ //a directory gets created as at that time it can be ensured that there is no
+ //work in progress files, no memory mapping issue etc
+ //Also at this time its required that state in local dir should exactly same as
+ //one in remote dir
+ synchronized (validatedIndexPaths){
+ new IndexSanityChecker(indexPath, local, remote).check();
+ validatedIndexPaths.add(indexPath);
+ }
+ }
+
/**
* Creates the workDir. If it exists then it is cleaned
*
@@ -1115,7 +1141,7 @@ public class IndexCopier implements Copy
return "DeleteOldDirOnClose wrapper for " + getDelegate();
}
}
-
+
static final class LocalIndexFile {
final File dir;
final String name;
@@ -1123,7 +1149,7 @@ public class IndexCopier implements Copy
final boolean copyFromRemote;
private volatile int deleteAttemptCount;
final long creationTime = System.currentTimeMillis();
-
+
public LocalIndexFile(Directory dir, String fileName,
long size, boolean copyFromRemote){
this.copyFromRemote = copyFromRemote;
Added: jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java?rev=1788904&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java (added)
+++ jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java Mon Mar 27 11:36:54 2017
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.Directory;
+
+public class DirectoryUtils {
+ /**
+ * Get the file length in best effort basis.
+ * @return actual fileLength. -1 if cannot determine
+ */
+ public static long getFileLength(Directory dir, String fileName){
+ try{
+ //Check for file presence otherwise internally it results in
+ //an exception to be created
+ if (dir.fileExists(fileName)) {
+ return dir.fileLength(fileName);
+ }
+ } catch (Exception ignore){
+
+ }
+ return -1;
+ }
+
+ public static long dirSize(Directory directory) throws IOException {
+ long totalFileSize = 0L;
+ if (directory == null) {
+ return -1;
+ }
+ String[] files = directory.listAll();
+ if (files == null) {
+ return totalFileSize;
+ }
+ for (String file : files) {
+ totalFileSize += directory.fileLength(file);
+ }
+ return totalFileSize;
+ }
+}
Propchange: jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java?rev=1788904&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java (added)
+++ jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java Mon Mar 27 11:36:54 2017
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.lucene.store.Directory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Checks that all files in local which are present in remote have same file length.
+ * If there is a size mismatch in any one of the file then whole of local index content
+ * would be purged
+ */
+public class IndexSanityChecker {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final Directory local;
+ private final Directory remote;
+ private final String indexPath;
+ private int localFileCount;
+ private int remoteFileCount;
+ private long localDirSize;
+ private long remoteDirSize;
+
+ public IndexSanityChecker(String indexPath, Directory local, Directory remote) {
+ this.local = local;
+ this.remote = remote;
+ this.indexPath = indexPath;
+ }
+
+ public boolean check() throws IOException {
+ boolean allFine = true;
+ //TODO Add support for checksum based checks
+ if (isThereASizeMismatch()){
+ //In case of any mismatch just purge all local files
+ deleteAllFiles(local);
+ allFine = false;
+ } else {
+ //Remove local files which are not found in remote
+ for (String fileName : local.listAll()) {
+ if (!remote.fileExists(fileName)) {
+ local.deleteFile(fileName);
+ }
+ }
+ }
+
+ if (allFine) {
+ log.info("Local index directory content found to be valid for index [{}]. " +
+ "Stats Local: {} files ({}), Remote: {} files ({})", indexPath,
+ localFileCount, IOUtils.humanReadableByteCount(localDirSize),
+ remoteFileCount, IOUtils.humanReadableByteCount(remoteDirSize));
+ } else {
+ log.warn("Local index directory content were not found to be in sync with remote for index [{}]. " +
+ "Local directory content has been purged and would be synced again from remote", indexPath);
+ }
+ return allFine;
+ }
+
+ private boolean isThereASizeMismatch() throws IOException {
+ for (String fileName : remote.listAll()){
+ long localLength = DirectoryUtils.getFileLength(local, fileName);
+ long remoteLength = remote.fileLength(fileName);
+
+ //This is a weak check based on length.
+ if (localLength > 0 && localLength != remoteLength){
+ log.warn("[{}] Found local copy for {} in {} but size of local {} differs from remote {}. ",
+ indexPath, fileName, local, localLength, remoteLength);
+ return true;
+ }
+
+ if (localLength > 0) {
+ localDirSize += localLength;
+ localFileCount++;
+ }
+
+ remoteDirSize += remoteLength;
+ remoteFileCount++;
+ }
+ return false;
+ }
+
+ private static void deleteAllFiles(Directory dir) throws IOException {
+ for (String fileName : dir.listAll()){
+ dir.deleteFile(fileName);
+ }
+ }
+}
Propchange: jackrabbit/oak/branches/1.4/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/branches/1.4/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java?rev=1788904&r1=1788903&r2=1788904&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java Mon Mar 27 11:36:54 2017
@@ -993,6 +993,41 @@ public class IndexCopierTest {
executorService.shutdown();
}
+ @Test
+ public void directoryContentMismatch_COR() throws Exception{
+ Directory baseDir = new CloseSafeDir();
+ IndexDefinition defn = new IndexDefinition(root, builder.getNodeState());
+ IndexCopier copier = new RAMIndexCopier(baseDir, sameThreadExecutor(), getWorkDir(), true);
+
+ Directory remote = new RAMDirectory();
+ byte[] t1 = writeFile(remote, "t1");
+ byte[] t2 = writeFile(remote, "t2");
+
+ //State of remote directory should set before wrapping as later
+ //additions would not be picked up given COW assume remote directory
+ //to be read only
+ Directory local = copier.wrapForRead("/foo", defn, remote);
+
+ readAndAssert(local, "t1", t1);
+ readAndAssert(local, "t2", t2);
+
+ copier.close();
+
+ //2. Modify the same file in remote directory simulating rollback scenario
+ Directory remoteModified = new RAMDirectory();
+ t1 = writeFile(remoteModified, "t1");
+
+ //3. Reopen the copier
+ copier = new RAMIndexCopier(baseDir, sameThreadExecutor(), getWorkDir(), true);
+
+ //4. Post opening local the content should be in sync with remote
+ //So t1 should be recreated matching remote
+ //t2 should be removed
+ local = copier.wrapForRead("/foo", defn, remoteModified);
+ readAndAssert(baseDir, "t1", t1);
+ assertFalse(baseDir.fileExists("t2"));
+ }
+
private byte[] writeFile(Directory dir, String name) throws IOException {
byte[] data = randomBytes(rnd.nextInt(maxFileSize) + 1);
IndexOutput o = dir.createOutput(name, IOContext.DEFAULT);