You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by kr...@apache.org on 2019/03/30 18:27:52 UTC
[lucene-solr] branch branch_8x updated: SOLR-11473: Make
HDFSDirectoryFactory support other prefixes (besides hdfs:/)
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new ae95487d SOLR-11473: Make HDFSDirectoryFactory support other prefixes (besides hdfs:/)
ae95487d is described below
commit ae95487db69a730ebec7ee5c8f562dccdeb6611f
Author: Kevin Risden <kr...@apache.org>
AuthorDate: Fri Mar 29 17:56:31 2019 -0400
SOLR-11473: Make HDFSDirectoryFactory support other prefixes (besides hdfs:/)
Signed-off-by: Kevin Risden <kr...@apache.org>
---
solr/CHANGES.txt | 5 ++--
.../org/apache/solr/core/HdfsDirectoryFactory.java | 35 ++++++++++++++--------
.../backup/repository/HdfsBackupRepository.java | 18 +++++------
.../org/apache/solr/index/hdfs/CheckHdfsIndex.java | 15 ++++++----
.../java/org/apache/solr/update/HdfsUpdateLog.java | 11 +++++--
.../apache/solr/core/HdfsDirectoryFactoryTest.java | 18 +++++++++++
6 files changed, 70 insertions(+), 32 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index db15d72..8a35251 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -72,6 +72,8 @@ New Features
a back-compat check of the .system collection to notify users of potential compatibility issues after
upgrades or schema changes. (ab)
+* SOLR-11473: Make HDFSDirectoryFactory support other prefixes (besides hdfs:/) (Kevin Risden)
+
Bug Fixes
----------------------
@@ -116,8 +118,7 @@ Bug Fixes
* SOLR-13351: Workaround for VELOCITY-908 (Kevin Risden)
- * SOLR-13349: High CPU usage in Solr due to Java 8 bug (Erick Erickson)
-
+* SOLR-13349: High CPU usage in Solr due to Java 8 bug (Erick Erickson)
Improvements
----------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index a0806ed..464b030 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -192,7 +192,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
assert params != null : "init must be called before create";
log.info("creating directory factory for path {}", path);
- Configuration conf = getConf();
+ Configuration conf = getConf(new Path(path));
if (metrics == null) {
metrics = MetricsHolder.metrics;
@@ -333,16 +333,21 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
}
- public Configuration getConf() {
+ public Configuration getConf(Path path) {
Configuration conf = new Configuration();
confDir = getConfig(CONFIG_DIRECTORY, null);
HdfsUtil.addHdfsResources(conf, confDir);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+
+ if(path != null) {
+ String fsScheme = path.toUri().getScheme();
+ if(fsScheme != null) {
+ conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
+ }
+ }
return conf;
}
- protected synchronized void removeDirectory(final CacheValue cacheValue)
- throws IOException {
+ protected synchronized void removeDirectory(final CacheValue cacheValue) {
FileSystem fileSystem = getCachedFileSystem(cacheValue.path);
try {
@@ -359,7 +364,10 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
@Override
public boolean isAbsolute(String path) {
- return path.startsWith("hdfs:/");
+ if(path.startsWith("/")) {
+ return false;
+ }
+ return new Path(path).isAbsolute();
}
@Override
@@ -435,10 +443,11 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
}
- private FileSystem getCachedFileSystem(String path) {
+ private FileSystem getCachedFileSystem(String pathStr) {
try {
// no need to close the fs, the cache will do it
- return tmpFsCache.get(path, () -> FileSystem.get(new Path(path).toUri(), getConf()));
+ Path path = new Path(pathStr);
+ return tmpFsCache.get(pathStr, () -> FileSystem.get(path.toUri(), getConf(path)));
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
@@ -462,7 +471,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
synchronized (HdfsDirectoryFactory.class) {
if (kerberosInit == null) {
kerberosInit = Boolean.TRUE;
- final Configuration conf = getConf();
+ final Configuration conf = getConf(null);
final String authVal = conf.get(HADOOP_SECURITY_AUTHENTICATION);
final String kerberos = "kerberos";
if (authVal != null && !authVal.equals(kerberos)) {
@@ -471,7 +480,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
+ " connect to HDFS via kerberos");
}
// let's avoid modifying the supplied configuration, just to be conservative
- final Configuration ugiConf = new Configuration(getConf());
+ final Configuration ugiConf = new Configuration(getConf(null));
ugiConf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos);
UserGroupInformation.setConfiguration(ugiConf);
log.info(
@@ -585,8 +594,8 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
// perform an atomic rename if possible
public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException {
String hdfsDirPath = getPath(dir);
- FileContext fileContext = FileContext.getFileContext(getConf());
- fileContext.rename(new Path(hdfsDirPath + "/" + fileName), new Path(hdfsDirPath + "/" + toName), Options.Rename.OVERWRITE);
+ FileContext fileContext = FileContext.getFileContext(getConf(new Path(hdfsDirPath)));
+ fileContext.rename(new Path(hdfsDirPath, fileName), new Path(hdfsDirPath, toName), Options.Rename.OVERWRITE);
}
@Override
@@ -600,7 +609,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
Path dir2 = ((HdfsDirectory) baseToDir).getHdfsDirPath();
Path file1 = new Path(dir1, fileName);
Path file2 = new Path(dir2, fileName);
- FileContext fileContext = FileContext.getFileContext(getConf());
+ FileContext fileContext = FileContext.getFileContext(getConf(dir1));
fileContext.rename(file1, file2);
return;
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
index 64bb14cd..6c0b04c 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
@@ -65,11 +65,19 @@ public class HdfsBackupRepository implements BackupRepository {
}
}
+ String hdfsSolrHome = (String) Objects.requireNonNull(args.get(HdfsDirectoryFactory.HDFS_HOME),
+ "Please specify " + HdfsDirectoryFactory.HDFS_HOME + " property.");
+ Path path = new Path(hdfsSolrHome);
+ while (path != null) { // Compute the path of root file-system (without requiring an additional system property).
+ baseHdfsPath = path;
+ path = path.getParent();
+ }
+
// We don't really need this factory instance. But we want to initialize it here to
// make sure that all HDFS related initialization is at one place (and not duplicated here).
factory = new HdfsDirectoryFactory();
factory.init(args);
- this.hdfsConfig = factory.getConf();
+ this.hdfsConfig = factory.getConf(new Path(hdfsSolrHome));
// Configure the umask mode if specified.
if (args.get(HDFS_UMASK_MODE_PARAM) != null) {
@@ -77,14 +85,6 @@ public class HdfsBackupRepository implements BackupRepository {
this.hdfsConfig.set(FsPermission.UMASK_LABEL, umaskVal);
}
- String hdfsSolrHome = (String) Objects.requireNonNull(args.get(HdfsDirectoryFactory.HDFS_HOME),
- "Please specify " + HdfsDirectoryFactory.HDFS_HOME + " property.");
- Path path = new Path(hdfsSolrHome);
- while (path != null) { // Compute the path of root file-system (without requiring an additional system property).
- baseHdfsPath = path;
- path = path.getParent();
- }
-
try {
this.fileSystem = FileSystem.get(this.baseHdfsPath.toUri(), this.hdfsConfig);
} catch (IOException e) {
diff --git a/solr/core/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java b/solr/core/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java
index ce1a9e0..8f8b12d 100644
--- a/solr/core/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java
+++ b/solr/core/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java
@@ -52,13 +52,14 @@ public class CheckHdfsIndex {
System.out.println("\nIgnoring specified -dir-impl, instead using " + HdfsDirectory.class.getSimpleName());
}
- System.out.println("\nOpening index @ " + opts.getIndexPath() + "\n");
+ Path indexPath = new Path(opts.getIndexPath());
+ System.out.println("\nOpening index @ " + indexPath + "\n");
Directory directory;
try {
- directory = new HdfsDirectory(new Path(opts.getIndexPath()), getConf());
+ directory = new HdfsDirectory(indexPath, getConf(indexPath));
} catch (IOException e) {
- System.out.println("ERROR: could not open hdfs directory \"" + opts.getIndexPath() + "\"; exiting");
+ System.out.println("ERROR: could not open hdfs directory \"" + indexPath + "\"; exiting");
e.printStackTrace(System.out);
return 1;
}
@@ -69,11 +70,15 @@ public class CheckHdfsIndex {
}
}
- private static Configuration getConf() {
+ private static Configuration getConf(Path path) {
Configuration conf = new Configuration();
String confDir = System.getProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY);
HdfsUtil.addHdfsResources(conf, confDir);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+
+ String fsScheme = path.toUri().getScheme();
+ if(fsScheme != null) {
+ conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
+ }
return conf;
}
}
diff --git a/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java b/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
index c0bbeca..50e7a0c 100644
--- a/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java
@@ -75,12 +75,16 @@ public class HdfsUpdateLog extends UpdateLog {
log.info("Initializing HdfsUpdateLog: tlogDfsReplication={}", tlogDfsReplication);
}
- private Configuration getConf() {
+ private Configuration getConf(Path path) {
Configuration conf = new Configuration();
if (confDir != null) {
HdfsUtil.addHdfsResources(conf, confDir);
}
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+
+ String fsScheme = path.toUri().getScheme();
+ if(fsScheme != null) {
+ conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
+ }
return conf;
}
@@ -112,7 +116,8 @@ public class HdfsUpdateLog extends UpdateLog {
}
try {
- fs = FileSystem.get(new Path(dataDir).toUri(), getConf());
+ Path dataDirPath = new Path(dataDir);
+ fs = FileSystem.get(dataDirPath.toUri(), getConf(dataDirPath));
} catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
diff --git a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
index e8c16a3..b2794df 100644
--- a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
@@ -20,12 +20,15 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
+import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import com.google.common.base.Strings;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.store.Directory;
@@ -232,4 +235,19 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
}
}
}
+
+ @Test
+ public void testIsAbsolute() throws Exception {
+ try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
+ String relativePath = Strings.repeat(
+ RandomStrings.randomAsciiAlphanumOfLength(random(), random().nextInt(10) + 1) + '/',
+ random().nextInt(5) + 1);
+ assertFalse(hdfsFactory.isAbsolute(relativePath));
+ assertFalse(hdfsFactory.isAbsolute("/" + relativePath));
+
+ for(String rootPrefix : Arrays.asList("file://", "hdfs://", "s3a://", "foo://")) {
+ assertTrue(hdfsFactory.isAbsolute(rootPrefix + relativePath));
+ }
+ }
+ }
}