You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2015/04/08 17:47:10 UTC
svn commit: r1672109 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/core/
solr/core/src/java/org/apache/solr/store/hdfs/
solr/core/src/java/org/apache/solr/update/
solr/core/src/test/org/apache/solr/cloud/hd...
Author: markrmiller
Date: Wed Apr 8 15:47:10 2015
New Revision: 1672109
URL: http://svn.apache.org/r1672109
Log:
SOLR-7284: HdfsUpdateLog is using hdfs FileSystem.get without turning off the cache.
SOLR-7286: Using HDFS's FileSystem.newInstance does not guarantee a new instance.
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Wed Apr 8 15:47:10 2015
@@ -349,6 +349,12 @@ Bug Fixes
* SOLR-7338, SOLR-6583: A reloaded core will never register itself as active after a ZK session expiration
(Mark Miller, Timothy Potter)
+* SOLR-7284: HdfsUpdateLog is using hdfs FileSystem.get without turning off the cache.
+ (Mark Miller)
+
+* SOLR-7286: Using HDFS's FileSystem.newInstance does not guarantee a new instance.
+ (Mark Miller)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Wed Apr 8 15:47:10 2015
@@ -261,7 +261,7 @@ public class HdfsDirectoryFactory extend
Configuration conf = getConf();
FileSystem fileSystem = null;
try {
- fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), conf);
+ fileSystem = FileSystem.get(hdfsDirPath.toUri(), conf);
return fileSystem.exists(hdfsDirPath);
} catch (IOException e) {
LOG.error("Error checking if hdfs path exists", e);
@@ -275,6 +275,7 @@ public class HdfsDirectoryFactory extend
Configuration conf = new Configuration();
confDir = getConfig(CONFIG_DIRECTORY, null);
HdfsUtil.addHdfsResources(conf, confDir);
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
return conf;
}
@@ -283,7 +284,7 @@ public class HdfsDirectoryFactory extend
Configuration conf = getConf();
FileSystem fileSystem = null;
try {
- fileSystem = FileSystem.newInstance(new URI(cacheValue.path), conf);
+ fileSystem = FileSystem.get(new URI(cacheValue.path), conf);
boolean success = fileSystem.delete(new Path(cacheValue.path), true);
if (!success) {
throw new RuntimeException("Could not remove directory");
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java Wed Apr 8 15:47:10 2015
@@ -60,7 +60,7 @@ public class HdfsDirectory extends BaseD
super(lockFactory);
this.hdfsDirPath = hdfsDirPath;
this.configuration = configuration;
- fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), configuration);
+ fileSystem = FileSystem.get(hdfsDirPath.toUri(), configuration);
fileContext = FileContext.getFileContext(hdfsDirPath.toUri(), configuration);
while (true) {
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java Wed Apr 8 15:47:10 2015
@@ -64,7 +64,7 @@ public class HdfsLockFactory extends Loc
@Override
public boolean obtain() throws IOException {
FSDataOutputStream file = null;
- FileSystem fs = FileSystem.newInstance(lockPath.toUri(), conf);
+ FileSystem fs = FileSystem.get(lockPath.toUri(), conf);
try {
while (true) {
try {
@@ -111,7 +111,7 @@ public class HdfsLockFactory extends Loc
@Override
public void close() throws IOException {
- FileSystem fs = FileSystem.newInstance(lockPath.toUri(), conf);
+ FileSystem fs = FileSystem.get(lockPath.toUri(), conf);
try {
if (fs.exists(new Path(lockPath, lockName))
&& !fs.delete(new Path(lockPath, lockName), false)) throw new LockReleaseFailedException(
@@ -124,7 +124,7 @@ public class HdfsLockFactory extends Loc
@Override
public boolean isLocked() throws IOException {
boolean isLocked = false;
- FileSystem fs = FileSystem.newInstance(lockPath.toUri(), conf);
+ FileSystem fs = FileSystem.get(lockPath.toUri(), conf);
try {
isLocked = fs.exists(new Path(lockPath, lockName));
} finally {
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java Wed Apr 8 15:47:10 2015
@@ -97,7 +97,7 @@ public class HdfsUpdateLog extends Updat
if (confDir != null) {
HdfsUtil.addHdfsResources(conf, confDir);
}
-
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
return conf;
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java Wed Apr 8 15:47:10 2015
@@ -69,7 +69,7 @@ public class HdfsTestUtil {
conf.set("hadoop.security.authentication", "simple");
conf.set("hdfs.minidfs.basedir", dir + File.separator + "hdfsBaseDir");
conf.set("dfs.namenode.name.dir", dir + File.separator + "nameNodeNameDir");
-
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
System.setProperty("test.build.data", dir + File.separator + "hdfs" + File.separator + "build");
System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache");
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java Wed Apr 8 15:47:10 2015
@@ -215,7 +215,8 @@ public class StressHdfsTest extends Basi
// check that all dirs are gone
for (String dataDir : dataDirs) {
Configuration conf = new Configuration();
- FileSystem fs = FileSystem.newInstance(new URI(dataDir), conf);
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ FileSystem fs = FileSystem.get(new URI(dataDir), conf);
assertFalse(
"Data directory exists after collection removal : " + dataDir,
fs.exists(new Path(dataDir)));
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java?rev=1672109&r1=1672108&r2=1672109&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java Wed Apr 8 15:47:10 2015
@@ -81,7 +81,9 @@ public class TestRecoveryHdfs extends So
try {
URI uri = new URI(hdfsUri);
- fs = FileSystem.newInstance(uri, new Configuration());
+ Configuration conf = new Configuration();
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ fs = FileSystem.get(uri, conf);
} catch (IOException | URISyntaxException e) {
throw new RuntimeException(e);
}