You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2016/03/22 21:36:40 UTC
lucene-solr:master: SOLR-8855: The HDFS BlockDirectory should not
clean up it's cache on shutdown.
Repository: lucene-solr
Updated Branches:
refs/heads/master 275a259b1 -> 9aeb745a7
SOLR-8855: The HDFS BlockDirectory should not clean up it's cache on shutdown.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9aeb745a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9aeb745a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9aeb745a
Branch: refs/heads/master
Commit: 9aeb745a7daf84a8365e3d823ea314d9d371ae9b
Parents: 275a259
Author: markrmiller <ma...@apache.org>
Authored: Tue Mar 22 16:09:56 2016 -0400
Committer: markrmiller <ma...@apache.org>
Committed: Tue Mar 22 16:09:56 2016 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../solr/core/CachingDirectoryFactory.java | 20 ++++++-------
.../org/apache/solr/core/DirectoryFactory.java | 6 ++++
.../apache/solr/core/HdfsDirectoryFactory.java | 1 +
.../solr/core/ShutdownAwareDirectory.java | 30 ++++++++++++++++++++
.../src/java/org/apache/solr/core/SolrCore.java | 4 +++
.../solr/store/blockcache/BlockDirectory.java | 10 ++++++-
7 files changed, 60 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9aeb745a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 81e079e..3647bf7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -58,6 +58,8 @@ Bug Fixes
* SOLR-8857: HdfsUpdateLog does not use configured or new default number of version buckets and is
hard coded to 256. (Mark Miller, yonik, Gregory Chanan)
+* SOLR-8855: The HDFS BlockDirectory should not clean up it's cache on shutdown. (Mark Miller)
+
Optimizations
----------------------
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9aeb745a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
index f7877c4..11cf479 100644
--- a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
@@ -25,27 +25,17 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext.Context;
import org.apache.lucene.store.LockFactory;
-import org.apache.lucene.store.NRTCachingDirectory;
-import org.apache.lucene.store.NativeFSLockFactory;
-import org.apache.lucene.store.NoLockFactory;
-import org.apache.lucene.store.SimpleFSLockFactory;
-import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.ObjectReleaseTracker;
-import org.apache.solr.store.blockcache.BlockDirectory;
-import org.apache.solr.store.hdfs.HdfsDirectory;
-import org.apache.solr.store.hdfs.HdfsLockFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -302,9 +292,15 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory {
}
private void close(CacheValue val) {
+ log.info("Closing directory, CoreContainer#isShutdown={}", coreContainer != null ? coreContainer.isShutDown() : "null");
try {
- log.info("Closing directory: " + val.path);
- val.directory.close();
+ if (coreContainer != null && coreContainer.isShutDown() && val.directory instanceof ShutdownAwareDirectory) {
+ log.info("Closing directory on shutdown: " + val.path);
+ ((ShutdownAwareDirectory) val.directory).closeOnShutdown();
+ } else {
+ log.info("Closing directory: " + val.path);
+ val.directory.close();
+ }
assert ObjectReleaseTracker.release(val.directory);
} catch (Exception e) {
SolrException.log(log, "Error closing directory", e);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9aeb745a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index 99c3583..8cc9e7d 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -60,6 +60,8 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
public final static String LOCK_TYPE_SINGLE = "single";
public final static String LOCK_TYPE_NONE = "none";
public final static String LOCK_TYPE_HDFS = "hdfs";
+
+ protected volatile CoreContainer coreContainer;
/**
* Indicates a Directory will no longer be used, and when its ref count
@@ -325,4 +327,8 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
FileUtils.deleteDirectory(dirToRm);
return !dirToRm.isDirectory();
}
+
+ public void initCoreContainer(CoreContainer cc) {
+ this.coreContainer = cc;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9aeb745a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index 0d53667..6a37a6a 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -136,6 +136,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
@Override
public void init(NamedList args) {
+ super.init(args);
params = SolrParams.toSolrParams(args);
this.hdfsDataDir = getConfig(HDFS_HOME, null);
if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9aeb745a/solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java b/solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java
new file mode 100644
index 0000000..effe677
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.Directory;
+
+/**
+ * {@link CachingDirectoryFactory} will call this method
+ * rather than {@link Directory#close()} on shutdown if
+ * a Directory implements this interface.
+ */
+public interface ShutdownAwareDirectory {
+ public void closeOnShutdown() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9aeb745a/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index c5e54d2..e511de0 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -125,6 +125,7 @@ import static org.apache.solr.common.params.CommonParams.PATH;
*
*/
public final class SolrCore implements SolrInfoMBean, Closeable {
+
public static final String version="1.0";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -480,10 +481,13 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
if (info != null) {
log.info(info.className);
dirFactory = getResourceLoader().newInstance(info.className, DirectoryFactory.class);
+ // allow DirectoryFactory instances to access the CoreContainer
+ dirFactory.initCoreContainer(getCoreDescriptor().getCoreContainer());
dirFactory.init(info.initArgs);
} else {
log.info("solr.NRTCachingDirectoryFactory");
dirFactory = new NRTCachingDirectoryFactory();
+ dirFactory.initCoreContainer(getCoreDescriptor().getCoreContainer());
}
return dirFactory;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9aeb745a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
index d890ee8..c1b4684 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
@@ -29,6 +29,7 @@ import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.solr.core.ShutdownAwareDirectory;
import org.apache.solr.store.hdfs.HdfsDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,7 +37,7 @@ import org.slf4j.LoggerFactory;
/**
* @lucene.experimental
*/
-public class BlockDirectory extends FilterDirectory {
+public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
@@ -224,6 +225,13 @@ public class BlockDirectory extends FilterDirectory {
}
@Override
+ public void closeOnShutdown() throws IOException {
+ LOG.info("BlockDirectory closing on shutdown");
+ // we are shutting down, no need to clean up cache
+ super.close();
+ }
+
+ @Override
public void close() throws IOException {
try {
String[] files = listAll();