You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by kr...@apache.org on 2022/02/22 14:04:47 UTC
[solr] 07/10: SOLR-16025: Enable spotless on hdfs module
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch branch_9_0
in repository https://gitbox.apache.org/repos/asf/solr.git
commit c95caa1693afd05e53bfd8c73810bde05e61f5ec
Author: Kevin Risden <kr...@apache.org>
AuthorDate: Mon Feb 21 10:15:47 2022 -0500
SOLR-16025: Enable spotless on hdfs module
---
gradle/validation/spotless.gradle | 8 +-
.../org/apache/solr/core/HdfsDirectoryFactory.java | 358 ++++++-----
.../backup/repository/HdfsBackupRepository.java | 47 +-
.../solr/core/snapshots/SolrSnapshotsTool.java | 331 +++++++---
.../org/apache/solr/index/hdfs/CheckHdfsIndex.java | 12 +-
.../org/apache/solr/index/hdfs/package-info.java | 7 +-
.../apache/solr/store/blockcache/BlockCache.java | 113 ++--
.../solr/store/blockcache/BlockCacheKey.java | 16 +-
.../solr/store/blockcache/BlockCacheLocation.java | 24 +-
.../solr/store/blockcache/BlockDirectory.java | 259 ++++----
.../solr/store/blockcache/BlockDirectoryCache.java | 67 +-
.../apache/solr/store/blockcache/BlockLocks.java | 30 +-
.../apache/solr/store/blockcache/BufferStore.java | 56 +-
.../org/apache/solr/store/blockcache/Cache.java | 55 +-
.../solr/store/blockcache/CachedIndexOutput.java | 36 +-
.../store/blockcache/CustomBufferedIndexInput.java | 103 ++-
.../org/apache/solr/store/blockcache/Metrics.java | 106 ++--
.../blockcache/ReusedBufferedIndexOutput.java | 67 +-
.../org/apache/solr/store/blockcache/Store.java | 5 +-
.../apache/solr/store/blockcache/package-info.java | 8 +-
.../org/apache/solr/store/hdfs/HdfsDirectory.java | 87 ++-
.../org/apache/solr/store/hdfs/HdfsFileWriter.java | 32 +-
.../solr/store/hdfs/HdfsLocalityReporter.java | 133 ++--
.../apache/solr/store/hdfs/HdfsLockFactory.java | 23 +-
.../org/apache/solr/store/hdfs/package-info.java | 7 +-
.../org/apache/solr/update/HdfsTransactionLog.java | 249 ++++----
.../java/org/apache/solr/update/HdfsUpdateLog.java | 150 ++---
.../src/java/org/apache/solr/util/FSHDFSUtils.java | 83 ++-
.../src/java/org/apache/solr/util/HdfsUtil.java | 27 +-
.../solr/cloud/MoveReplicaHDFSFailoverTest.java | 120 ++--
.../org/apache/solr/cloud/MoveReplicaHDFSTest.java | 37 +-
.../cloud/SharedFSAutoReplicaFailoverTest.java | 184 ++++--
.../HdfsCloudIncrementalBackupTest.java | 207 +++---
.../HdfsCollectionsAPIDistributedZkTest.java | 15 +-
.../collections/TestHdfsCloudBackupRestore.java | 152 +++--
.../solr/cloud/hdfs/DisableKeyStoresFactory.java | 21 +-
.../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 34 +-
.../cloud/hdfs/HdfsBasicDistributedZk2Test.java | 31 +-
.../cloud/hdfs/HdfsBasicDistributedZkTest.java | 23 +-
.../hdfs/HdfsChaosMonkeyNothingIsSafeTest.java | 29 +-
.../cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java | 38 +-
.../solr/cloud/hdfs/HdfsFakeGroupMapping.java | 13 +-
.../apache/solr/cloud/hdfs/HdfsNNFailoverTest.java | 38 +-
.../solr/cloud/hdfs/HdfsRecoverLeaseTest.java | 138 ++--
.../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java | 20 +-
.../cloud/hdfs/HdfsRestartWhileUpdatingTest.java | 26 +-
.../apache/solr/cloud/hdfs/HdfsSyncSliceTest.java | 27 +-
.../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 169 ++---
.../apache/solr/cloud/hdfs/HdfsThreadLeakTest.java | 22 +-
.../HdfsTlogReplayBufferedWhileIndexingTest.java | 25 +-
.../cloud/hdfs/HdfsUnloadDistributedZkTest.java | 28 +-
.../hdfs/HdfsWriteToMultipleCollectionsTest.java | 71 ++-
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 120 ++--
.../apache/solr/core/HdfsDirectoryFactoryTest.java | 127 ++--
.../HdfsBackupRepositoryIntegrationTest.java | 135 ++--
.../repository/HdfsBackupRepositoryTest.java | 7 +-
.../solr/handler/TestHdfsBackupRestoreCore.java | 162 ++---
.../solr/hdfs/util/BadHdfsThreadsFilter.java | 5 +-
.../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 23 +-
.../org/apache/solr/search/TestRecoveryHdfs.java | 701 ++++++++++++---------
.../solr/store/blockcache/BlockCacheTest.java | 365 ++++++-----
.../solr/store/blockcache/BlockDirectoryTest.java | 45 +-
.../solr/store/blockcache/BufferStoreTest.java | 48 +-
.../apache/solr/store/hdfs/HdfsDirectoryTest.java | 79 +--
.../solr/store/hdfs/HdfsLockFactoryTest.java | 24 +-
.../org/apache/solr/update/TestHdfsUpdateLog.java | 85 +--
66 files changed, 3289 insertions(+), 2604 deletions(-)
diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradle
index 374db74..2793743 100644
--- a/gradle/validation/spotless.gradle
+++ b/gradle/validation/spotless.gradle
@@ -44,14 +44,20 @@ configure(project(":solr").subprojects) { prj ->
// Exclude certain files (generated ones, mostly).
switch (project.path) {
- case ":solr:modules:hdfs":
+ case ":solr:modules:sql":
case ":solr:core":
case ":solr:solrj":
case ":solr:test-framework":
targetExclude "src/**/*.java"
break
+<<<<<<< HEAD
case ":solr:modules:ltr":
targetExclude "src/test/**/*.java"
+=======
+ case ":solr:modules:hdfs":
+ // Exclude Hadoop copied files to make upgrades easier
+ targetExclude "src/**/org/apache/hadoop/**"
+>>>>>>> 2b6e490cafa (SOLR-16025: Enable spotless on hdfs module)
break
}
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/modules/hdfs/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index b88eb3a..c4cef6d 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -16,6 +16,12 @@
*/
package org.apache.solr.core;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@@ -28,11 +34,6 @@ import java.util.Locale;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -71,23 +72,26 @@ import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-
-public class HdfsDirectoryFactory extends CachingDirectoryFactory implements SolrCoreAware, SolrMetricProducer {
+public class HdfsDirectoryFactory extends CachingDirectoryFactory
+ implements SolrCoreAware, SolrMetricProducer {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public final static String LOCK_TYPE_HDFS = "hdfs";
+ public static final String LOCK_TYPE_HDFS = "hdfs";
public static final String BLOCKCACHE_SLAB_COUNT = "solr.hdfs.blockcache.slab.count";
- public static final String BLOCKCACHE_DIRECT_MEMORY_ALLOCATION = "solr.hdfs.blockcache.direct.memory.allocation";
+ public static final String BLOCKCACHE_DIRECT_MEMORY_ALLOCATION =
+ "solr.hdfs.blockcache.direct.memory.allocation";
public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
public static final String BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
- public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled"; // currently buggy and disabled
-
+ public static final String BLOCKCACHE_WRITE_ENABLED =
+ "solr.hdfs.blockcache.write.enabled"; // currently buggy and disabled
+
public static final String NRTCACHINGDIRECTORY_ENABLE = "solr.hdfs.nrtcachingdirectory.enable";
- public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB = "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
- public static final String NRTCACHINGDIRECTORY_MAXCACHEMB = "solr.hdfs.nrtcachingdirectory.maxcachedmb";
+ public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB =
+ "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
+ public static final String NRTCACHINGDIRECTORY_MAXCACHEMB =
+ "solr.hdfs.nrtcachingdirectory.maxcachedmb";
public static final String NUMBEROFBLOCKSPERBANK = "solr.hdfs.blockcache.blocksperbank";
public static final String LOCALITYMETRICS_ENABLED = "solr.hdfs.locality.metrics.enabled";
@@ -95,20 +99,20 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
public static final String KERBEROS_ENABLED = "solr.hdfs.security.kerberos.enabled";
public static final String KERBEROS_KEYTAB = "solr.hdfs.security.kerberos.keytabfile";
public static final String KERBEROS_PRINCIPAL = "solr.hdfs.security.kerberos.principal";
-
+
public static final String HDFS_HOME = "solr.hdfs.home";
-
+
public static final String CONFIG_DIRECTORY = "solr.hdfs.confdir";
-
+
public static final String CACHE_MERGES = "solr.hdfs.blockcache.cachemerges";
public static final String CACHE_READONCE = "solr.hdfs.blockcache.cachereadonce";
public static final String HDFS_UPDATE_LOG_CLASS_NAME = "solr.HdfsUpdateLog";
-
+
private SolrParams params;
-
+
private String hdfsDataDir;
-
+
private String confDir;
private boolean cacheReadOnce;
@@ -116,28 +120,31 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
private boolean cacheMerges;
private static BlockCache globalBlockCache;
-
+
public static Metrics metrics;
private static Boolean kerberosInit;
// we use this cache for FileSystem instances when we don't have access to a long lived instance
- private com.google.common.cache.Cache<String,FileSystem> tmpFsCache = CacheBuilder.newBuilder()
- .concurrencyLevel(10)
- .maximumSize(1000)
- .expireAfterAccess(5, TimeUnit.MINUTES).removalListener(new RemovalListener<String,FileSystem>() {
- @Override
- public void onRemoval(RemovalNotification<String,FileSystem> rn) {
- IOUtils.closeQuietly(rn.getValue());
- }
- })
- .build();
-
- private final static class MetricsHolder {
+ private com.google.common.cache.Cache<String, FileSystem> tmpFsCache =
+ CacheBuilder.newBuilder()
+ .concurrencyLevel(10)
+ .maximumSize(1000)
+ .expireAfterAccess(5, TimeUnit.MINUTES)
+ .removalListener(
+ new RemovalListener<String, FileSystem>() {
+ @Override
+ public void onRemoval(RemovalNotification<String, FileSystem> rn) {
+ IOUtils.closeQuietly(rn.getValue());
+ }
+ })
+ .build();
+
+ private static final class MetricsHolder {
// [JCIP SE, Goetz, 16.6] Lazy initialization
// Won't load until MetricsHolder is referenced
public static final Metrics metrics = new Metrics();
}
-
+
@Override
public void close() throws IOException {
super.close();
@@ -156,7 +163,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
}
- private final static class LocalityHolder {
+ private static final class LocalityHolder {
public static final HdfsLocalityReporter reporter = new HdfsLocalityReporter();
}
@@ -180,7 +187,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
initKerberos();
}
}
-
+
@Override
protected LockFactory createLockFactory(String rawLockType) throws IOException {
if (null == rawLockType) {
@@ -196,60 +203,72 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
case DirectoryFactory.LOCK_TYPE_NONE:
return NoLockFactory.INSTANCE;
default:
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Unrecognized lockType: " + rawLockType);
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Unrecognized lockType: " + rawLockType);
}
}
@Override
@SuppressWarnings("resource")
- protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
+ protected Directory create(String path, LockFactory lockFactory, DirContext dirContext)
+ throws IOException {
assert params != null : "init must be called before create";
log.info("creating directory factory for path {}", path);
Configuration conf = getConf(new Path(path));
-
+
if (metrics == null) {
metrics = MetricsHolder.metrics;
}
-
+
boolean blockCacheEnabled = getConfig(BLOCKCACHE_ENABLED, true);
boolean blockCacheGlobal = getConfig(BLOCKCACHE_GLOBAL, true);
boolean blockCacheReadEnabled = getConfig(BLOCKCACHE_READ_ENABLED, true);
-
+
final HdfsDirectory hdfsDir;
final Directory dir;
if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
int numberOfBlocksPerBank = getConfig(NUMBEROFBLOCKSPERBANK, 16384);
-
+
int blockSize = BlockDirectory.BLOCK_SIZE;
-
+
int bankCount = getConfig(BLOCKCACHE_SLAB_COUNT, 1);
-
+
boolean directAllocation = getConfig(BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
-
+
int slabSize = numberOfBlocksPerBank * blockSize;
log.info(
"Number of slabs of block cache [{}] with direct memory allocation set to [{}]",
- bankCount, directAllocation);
+ bankCount,
+ directAllocation);
if (log.isInfoEnabled()) {
log.info(
"Block cache target memory usage, slab size of [{}] will allocate [{}] slabs and use ~[{}] bytes",
- new Object[]{slabSize, bankCount,
- ((long) bankCount * (long) slabSize)});
+ new Object[] {slabSize, bankCount, ((long) bankCount * (long) slabSize)});
}
-
+
int bsBufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", blockSize);
- int bsBufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 0); // this is actually total size
-
- BlockCache blockCache = getBlockDirectoryCache(numberOfBlocksPerBank,
- blockSize, bankCount, directAllocation, slabSize,
- bsBufferSize, bsBufferCount, blockCacheGlobal);
-
+ int bsBufferCount =
+ params.getInt(
+ "solr.hdfs.blockcache.bufferstore.buffercount", 0); // this is actually total size
+
+ BlockCache blockCache =
+ getBlockDirectoryCache(
+ numberOfBlocksPerBank,
+ blockSize,
+ bankCount,
+ directAllocation,
+ slabSize,
+ bsBufferSize,
+ bsBufferCount,
+ blockCacheGlobal);
+
Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
int readBufferSize = params.getInt("solr.hdfs.blockcache.read.buffersize", blockSize);
hdfsDir = new HdfsDirectory(new Path(path), lockFactory, conf, readBufferSize);
- dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
+ dir =
+ new BlockDirectory(
+ path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
} else {
hdfsDir = new HdfsDirectory(new Path(path), conf);
dir = hdfsDir;
@@ -262,7 +281,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
if (nrtCachingDirectory) {
double nrtCacheMaxMergeSizeMB = getConfig(NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
double nrtCacheMaxCacheMB = getConfig(NRTCACHINGDIRECTORY_MAXCACHEMB, 192);
-
+
return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB, nrtCacheMaxCacheMB);
}
return dir;
@@ -278,7 +297,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
return value == null ? defaultValue : value;
}
-
+
int getConfig(String name, int defaultValue) {
Integer value = params.getInt(name);
if (value == null) {
@@ -297,31 +316,55 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
return value == null ? defaultValue : value;
}
-
- private BlockCache getBlockDirectoryCache(int numberOfBlocksPerBank, int blockSize, int bankCount,
- boolean directAllocation, int slabSize, int bufferSize, int bufferCount, boolean staticBlockCache) {
+
+ private BlockCache getBlockDirectoryCache(
+ int numberOfBlocksPerBank,
+ int blockSize,
+ int bankCount,
+ boolean directAllocation,
+ int slabSize,
+ int bufferSize,
+ int bufferCount,
+ boolean staticBlockCache) {
if (!staticBlockCache) {
log.info("Creating new single instance HDFS BlockCache");
- return createBlockCache(numberOfBlocksPerBank, blockSize, bankCount, directAllocation, slabSize, bufferSize, bufferCount);
+ return createBlockCache(
+ numberOfBlocksPerBank,
+ blockSize,
+ bankCount,
+ directAllocation,
+ slabSize,
+ bufferSize,
+ bufferCount);
}
synchronized (HdfsDirectoryFactory.class) {
-
if (globalBlockCache == null) {
log.info("Creating new global HDFS BlockCache");
- globalBlockCache = createBlockCache(numberOfBlocksPerBank, blockSize, bankCount,
- directAllocation, slabSize, bufferSize, bufferCount);
+ globalBlockCache =
+ createBlockCache(
+ numberOfBlocksPerBank,
+ blockSize,
+ bankCount,
+ directAllocation,
+ slabSize,
+ bufferSize,
+ bufferCount);
}
}
return globalBlockCache;
}
- private BlockCache createBlockCache(int numberOfBlocksPerBank, int blockSize,
- int bankCount, boolean directAllocation, int slabSize, int bufferSize,
+ private BlockCache createBlockCache(
+ int numberOfBlocksPerBank,
+ int blockSize,
+ int bankCount,
+ boolean directAllocation,
+ int slabSize,
+ int bufferSize,
int bufferCount) {
BufferStore.initNewBuffer(bufferSize, bufferCount, metrics);
- long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
- * (long) blockSize;
-
+ long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank * (long) blockSize;
+
BlockCache blockCache;
try {
blockCache = new BlockCache(metrics, directAllocation, totalMemory, slabSize, blockSize);
@@ -330,12 +373,14 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
"The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args)"
+ " or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap,"
+ " your java heap size might not be large enough."
- + " Failed allocating ~" + totalMemory / 1000000.0 + " MB.",
+ + " Failed allocating ~"
+ + totalMemory / 1000000.0
+ + " MB.",
e);
}
return blockCache;
}
-
+
@Override
public boolean exists(String path) {
final Path hdfsDirPath = new Path(path);
@@ -348,21 +393,21 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
throw new RuntimeException("Error checking if hdfs path exists", e);
}
}
-
+
public Configuration getConf(Path path) {
Configuration conf = new Configuration();
confDir = getConfig(CONFIG_DIRECTORY, null);
HdfsUtil.addHdfsResources(conf, confDir);
- if(path != null) {
+ if (path != null) {
String fsScheme = path.toUri().getScheme();
- if(fsScheme != null) {
+ if (fsScheme != null) {
conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
}
}
return conf;
}
-
+
protected synchronized void removeDirectory(final CacheValue cacheValue) {
FileSystem fileSystem = getCachedFileSystem(cacheValue.path);
@@ -373,62 +418,65 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
} catch (Exception e) {
log.error("Could not remove directory", e);
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "Could not remove directory", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not remove directory", e);
}
}
-
+
@Override
public boolean isAbsolute(String path) {
- if(path.startsWith("/")) {
+ if (path.startsWith("/")) {
return false;
}
return new Path(path).isAbsolute();
}
-
+
@Override
public boolean isPersistent() {
return true;
}
-
+
@Override
public boolean isSharedStorage() {
return true;
}
-
+
@Override
public boolean searchersReserveCommitPoints() {
return true;
}
-
+
@Override
public String getDataHome(CoreDescriptor cd) throws IOException {
if (hdfsDataDir == null) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "You must set the "
- + this.getClass().getSimpleName() + " param " + HDFS_HOME
- + " for relative dataDir paths to work");
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "You must set the "
+ + this.getClass().getSimpleName()
+ + " param "
+ + HDFS_HOME
+ + " for relative dataDir paths to work");
}
-
+
// by default, we go off the instance directory
String path;
if (cd.getCloudDescriptor() != null) {
- path = URLEncoder.encode(cd.getCloudDescriptor().getCollectionName(),
- "UTF-8")
- + "/"
- + URLEncoder.encode(cd.getCloudDescriptor().getCoreNodeName(),
- "UTF-8");
+ path =
+ URLEncoder.encode(cd.getCloudDescriptor().getCollectionName(), "UTF-8")
+ + "/"
+ + URLEncoder.encode(cd.getCloudDescriptor().getCoreNodeName(), "UTF-8");
} else {
path = cd.getName();
}
-
- return normalize(SolrPaths.normalizeDir(ZkController
- .trimLeadingAndTrailingSlashes(hdfsDataDir)
- + "/"
- + path
- + "/"
- + cd.getDataDir()));
+
+ return normalize(
+ SolrPaths.normalizeDir(
+ ZkController.trimLeadingAndTrailingSlashes(hdfsDataDir)
+ + "/"
+ + path
+ + "/"
+ + cd.getDataDir()));
}
-
+
/**
* @param directory to calculate size of
* @return size in bytes
@@ -439,7 +487,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
String hdfsDirPath = getPath(directory);
return size(hdfsDirPath);
}
-
+
/**
* @param path to calculate size of
* @return size in bytes
@@ -470,17 +518,17 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
public String getConfDir() {
return confDir;
}
-
+
private void initKerberos() {
String keytabFile = getConfig(KERBEROS_KEYTAB, "").trim();
if (keytabFile.length() == 0) {
- throw new IllegalArgumentException(KERBEROS_KEYTAB + " required because "
- + KERBEROS_ENABLED + " set to true");
+ throw new IllegalArgumentException(
+ KERBEROS_KEYTAB + " required because " + KERBEROS_ENABLED + " set to true");
}
String principal = getConfig(KERBEROS_PRINCIPAL, "");
if (principal.length() == 0) {
- throw new IllegalArgumentException(KERBEROS_PRINCIPAL
- + " required because " + KERBEROS_ENABLED + " set to true");
+ throw new IllegalArgumentException(
+ KERBEROS_PRINCIPAL + " required because " + KERBEROS_ENABLED + " set to true");
}
synchronized (HdfsDirectoryFactory.class) {
if (kerberosInit == null) {
@@ -489,9 +537,12 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
final String authVal = conf.get(HADOOP_SECURITY_AUTHENTICATION);
final String kerberos = "kerberos";
if (authVal != null && !authVal.equals(kerberos)) {
- throw new IllegalArgumentException(HADOOP_SECURITY_AUTHENTICATION
- + " set to: " + authVal + ", not kerberos, but attempting to "
- + " connect to HDFS via kerberos");
+ throw new IllegalArgumentException(
+ HADOOP_SECURITY_AUTHENTICATION
+ + " set to: "
+ + authVal
+ + ", not kerberos, but attempting to "
+ + " connect to HDFS via kerberos");
}
// let's avoid modifying the supplied configuration, just to be conservative
final Configuration ugiConf = new Configuration(getConf(null));
@@ -499,7 +550,8 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
UserGroupInformation.setConfiguration(ugiConf);
log.info(
"Attempting to acquire kerberos ticket with keytab: {}, principal: {} ",
- keytabFile, principal);
+ keytabFile,
+ principal);
try {
UserGroupInformation.loginUserFromKeytab(principal, keytabFile);
} catch (IOException ioe) {
@@ -532,7 +584,8 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
@Override
- public void cleanupOldIndexDirectories(final String dataDir, final String currentIndexDir, boolean afterReload) {
+ public void cleanupOldIndexDirectories(
+ final String dataDir, final String currentIndexDir, boolean afterReload) {
// Get the FileSystem object
final Path dataDirPath = new Path(dataDir);
@@ -545,28 +598,40 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
log.error("Error checking if hdfs path exists {}", dataDir, e);
}
if (!pathExists) {
- log.warn("{} does not point to a valid data directory; skipping clean-up of old index directories.", dataDir);
+ log.warn(
+ "{} does not point to a valid data directory; skipping clean-up of old index directories.",
+ dataDir);
return;
}
- final Path currentIndexDirPath = new Path(currentIndexDir); // make sure we don't delete the current
+ final Path currentIndexDirPath =
+ new Path(currentIndexDir); // make sure we don't delete the current
final FileSystem fs = fileSystem;
FileStatus[] oldIndexDirs = null;
try {
- oldIndexDirs = fileSystem.listStatus(dataDirPath, new PathFilter() {
- @Override
- public boolean accept(Path path) {
- boolean accept = false;
- String pathName = path.getName();
- try {
- accept = fs.getFileStatus(path).isDirectory() && !path.equals(currentIndexDirPath) &&
- (pathName.equals("index") || pathName.matches(INDEX_W_TIMESTAMP_REGEX));
- } catch (IOException e) {
- log.error("Error checking if path {} is an old index directory, caused by: {}", path, e);
- }
- return accept;
- }
- });
+ oldIndexDirs =
+ fileSystem.listStatus(
+ dataDirPath,
+ new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ boolean accept = false;
+ String pathName = path.getName();
+ try {
+ accept =
+ fs.getFileStatus(path).isDirectory()
+ && !path.equals(currentIndexDirPath)
+ && (pathName.equals("index")
+ || pathName.matches(INDEX_W_TIMESTAMP_REGEX));
+ } catch (IOException e) {
+ log.error(
+ "Error checking if path {} is an old index directory, caused by: {}",
+ path,
+ e);
+ }
+ return accept;
+ }
+ });
} catch (FileNotFoundException fnfe) {
// already deleted - ignore
log.debug("Old index directory already deleted - skipping...", fnfe);
@@ -574,8 +639,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
log.error("Error checking for old index directories to clean-up.", ioExc);
}
- if (oldIndexDirs == null || oldIndexDirs.length == 0)
- return; // nothing to clean-up
+ if (oldIndexDirs == null || oldIndexDirs.length == 0) return; // nothing to clean-up
List<Path> oldIndexPaths = new ArrayList<>(oldIndexDirs.length);
for (FileStatus ofs : oldIndexDirs) {
@@ -583,19 +647,25 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
Collections.sort(oldIndexPaths, Collections.reverseOrder());
-
+
Set<String> livePaths = getLivePaths();
-
+
int i = 0;
if (afterReload) {
log.info("Will not remove most recent old directory on reload {}", oldIndexDirs[0]);
i = 1;
}
- log.info("Found {} old index directories to clean-up under {} afterReload={}", oldIndexDirs.length - i, dataDirPath, afterReload);
+ log.info(
+ "Found {} old index directories to clean-up under {} afterReload={}",
+ oldIndexDirs.length - i,
+ dataDirPath,
+ afterReload);
for (; i < oldIndexPaths.size(); i++) {
Path oldDirPath = oldIndexPaths.get(i);
if (livePaths.contains(oldDirPath.toString())) {
- log.warn("Cannot delete directory {} because it is still being referenced in the cache.", oldDirPath);
+ log.warn(
+ "Cannot delete directory {} because it is still being referenced in the cache.",
+ oldDirPath);
} else {
try {
if (fileSystem.delete(oldDirPath, true)) {
@@ -609,20 +679,23 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
}
}
-
+
// perform an atomic rename if possible
- public void renameWithOverwrite(Directory dir, String fileName, String toName) throws IOException {
+ public void renameWithOverwrite(Directory dir, String fileName, String toName)
+ throws IOException {
String hdfsDirPath = getPath(dir);
FileContext fileContext = FileContext.getFileContext(getConf(new Path(hdfsDirPath)));
- fileContext.rename(new Path(hdfsDirPath, fileName), new Path(hdfsDirPath, toName), Options.Rename.OVERWRITE);
+ fileContext.rename(
+ new Path(hdfsDirPath, fileName), new Path(hdfsDirPath, toName), Options.Rename.OVERWRITE);
}
-
+
@Override
- public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
-
+ public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
+ throws IOException {
+
Directory baseFromDir = getBaseDir(fromDir);
Directory baseToDir = getBaseDir(toDir);
-
+
if (baseFromDir instanceof HdfsDirectory && baseToDir instanceof HdfsDirectory) {
Path dir1 = ((HdfsDirectory) baseFromDir).getHdfsDirPath();
Path dir2 = ((HdfsDirectory) baseToDir).getHdfsDirPath();
@@ -640,5 +713,4 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
public UpdateLog newDefaultUpdateLog() {
return new HdfsUpdateLog(getConfDir());
}
-
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java b/solr/modules/hdfs/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
index a1f9aeb..3df5038 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
@@ -17,16 +17,15 @@
package org.apache.solr.core.backup.repository;
+import com.google.common.base.Preconditions;
import java.io.IOException;
import java.io.OutputStream;
+import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.NoSuchFileException;
import java.util.Collection;
import java.util.Objects;
-import java.lang.invoke.MethodHandles;
-
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -65,16 +64,22 @@ public class HdfsBackupRepository implements BackupRepository {
// Configure the size of the buffer used for copying index files to/from HDFS, if specified.
if (args.get(HDFS_COPY_BUFFER_SIZE_PARAM) != null) {
- this.copyBufferSize = (Integer)args.get(HDFS_COPY_BUFFER_SIZE_PARAM);
+ this.copyBufferSize = (Integer) args.get(HDFS_COPY_BUFFER_SIZE_PARAM);
if (this.copyBufferSize <= 0) {
- throw new IllegalArgumentException("Value of " + HDFS_COPY_BUFFER_SIZE_PARAM + " must be > 0");
+ throw new IllegalArgumentException(
+ "Value of " + HDFS_COPY_BUFFER_SIZE_PARAM + " must be > 0");
}
}
- String hdfsSolrHome = (String) Objects.requireNonNull(args.get(HdfsDirectoryFactory.HDFS_HOME),
- "Please specify " + HdfsDirectoryFactory.HDFS_HOME + " property.");
+ String hdfsSolrHome =
+ (String)
+ Objects.requireNonNull(
+ args.get(HdfsDirectoryFactory.HDFS_HOME),
+ "Please specify " + HdfsDirectoryFactory.HDFS_HOME + " property.");
Path path = new Path(hdfsSolrHome);
- while (path != null) { // Compute the path of root file-system (without requiring an additional system property).
+ while (path
+ != null) { // Compute the path of root file-system (without requiring an additional system
+ // property).
baseHdfsPath = path;
path = path.getParent();
}
@@ -87,7 +92,7 @@ public class HdfsBackupRepository implements BackupRepository {
// Configure the umask mode if specified.
if (args.get(HDFS_UMASK_MODE_PARAM) != null) {
- String umaskVal = (String)args.get(HDFS_UMASK_MODE_PARAM);
+ String umaskVal = (String) args.get(HDFS_UMASK_MODE_PARAM);
this.hdfsConfig.set(FsPermission.UMASK_LABEL, umaskVal);
}
@@ -188,25 +193,30 @@ public class HdfsBackupRepository implements BackupRepository {
}
@Override
- public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException {
- try (HdfsDirectory dir = new HdfsDirectory(new Path(destDir), NoLockFactory.INSTANCE,
- hdfsConfig, copyBufferSize)) {
+ public void copyIndexFileFrom(
+ Directory sourceDir, String sourceFileName, URI destDir, String destFileName)
+ throws IOException {
+ try (HdfsDirectory dir =
+ new HdfsDirectory(new Path(destDir), NoLockFactory.INSTANCE, hdfsConfig, copyBufferSize)) {
copyIndexFileFrom(sourceDir, sourceFileName, dir, destFileName);
}
}
@Override
- public void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException {
- try (HdfsDirectory dir = new HdfsDirectory(new Path(sourceRepo), NoLockFactory.INSTANCE,
- hdfsConfig, copyBufferSize)) {
+ public void copyIndexFileTo(
+ URI sourceRepo, String sourceFileName, Directory dest, String destFileName)
+ throws IOException {
+ try (HdfsDirectory dir =
+ new HdfsDirectory(
+ new Path(sourceRepo), NoLockFactory.INSTANCE, hdfsConfig, copyBufferSize)) {
dest.copyFrom(dir, sourceFileName, destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
}
}
@Override
- public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException) throws IOException {
- if (files.isEmpty())
- return;
+ public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException)
+ throws IOException {
+ if (files.isEmpty()) return;
for (String file : files) {
Path filePath = new Path(new Path(path), file);
@@ -216,5 +226,4 @@ public class HdfsBackupRepository implements BackupRepository {
}
}
}
-
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java b/solr/modules/hdfs/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java
index eef0ffe..23a769c 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java
@@ -17,6 +17,7 @@
package org.apache.solr.core.snapshots;
+import com.google.common.base.Preconditions;
import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@@ -36,7 +37,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
-
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@@ -59,14 +59,11 @@ import org.apache.solr.util.CLIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-
-/**
- * This class provides utility functions required for Solr snapshots functionality.
- */
+/** This class provides utility functions required for Solr snapshots functionality. */
public class SolrSnapshotsTool implements Closeable, CLIO {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final DateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());
+ private static final DateFormat dateFormat =
+ new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());
private static final String CREATE = "create";
private static final String DELETE = "delete";
@@ -82,14 +79,29 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
private static final String HDFS_PATH_PREFIX = "p";
private static final String BACKUP_REPO_NAME = "r";
private static final String ASYNC_REQ_ID = "i";
- private static final List<String> OPTION_HELP_ORDER = Arrays.asList(CREATE, DELETE, LIST, DESCRIBE,
- PREPARE_FOR_EXPORT, EXPORT_SNAPSHOT, HELP, SOLR_ZK_ENSEMBLE, COLLECTION, DEST_DIR, BACKUP_REPO_NAME,
- ASYNC_REQ_ID, TEMP_DIR, HDFS_PATH_PREFIX);
+ private static final List<String> OPTION_HELP_ORDER =
+ Arrays.asList(
+ CREATE,
+ DELETE,
+ LIST,
+ DESCRIBE,
+ PREPARE_FOR_EXPORT,
+ EXPORT_SNAPSHOT,
+ HELP,
+ SOLR_ZK_ENSEMBLE,
+ COLLECTION,
+ DEST_DIR,
+ BACKUP_REPO_NAME,
+ ASYNC_REQ_ID,
+ TEMP_DIR,
+ HDFS_PATH_PREFIX);
private final CloudSolrClient solrClient;
public SolrSnapshotsTool(String solrZkEnsemble) {
- solrClient = new CloudSolrClient.Builder(Collections.singletonList(solrZkEnsemble), Optional.empty()).build();
+ solrClient =
+ new CloudSolrClient.Builder(Collections.singletonList(solrZkEnsemble), Optional.empty())
+ .build();
}
@Override
@@ -100,51 +112,90 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
public void createSnapshot(String collectionName, String snapshotName) {
- CollectionAdminRequest.CreateSnapshot createSnap = new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName);
+ CollectionAdminRequest.CreateSnapshot createSnap =
+ new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName);
CollectionAdminResponse resp;
try {
resp = createSnap.process(solrClient);
- Preconditions.checkState(resp.getStatus() == 0, "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());
- CLIO.out("Successfully created snapshot with name " + snapshotName + " for collection " + collectionName);
+ Preconditions.checkState(
+ resp.getStatus() == 0,
+ "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());
+ CLIO.out(
+ "Successfully created snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName);
} catch (Exception e) {
- log.error("Failed to create a snapshot with name {} for collection {}", snapshotName, collectionName, e);
- CLIO.out("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName
- +" due to following error : "+e.getLocalizedMessage());
+ log.error(
+ "Failed to create a snapshot with name {} for collection {}",
+ snapshotName,
+ collectionName,
+ e);
+ CLIO.out(
+ "Failed to create a snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
}
}
public void deleteSnapshot(String collectionName, String snapshotName) {
- CollectionAdminRequest.DeleteSnapshot deleteSnap = new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName);
+ CollectionAdminRequest.DeleteSnapshot deleteSnap =
+ new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName);
CollectionAdminResponse resp;
try {
resp = deleteSnap.process(solrClient);
- Preconditions.checkState(resp.getStatus() == 0, "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus());
- CLIO.out("Successfully deleted snapshot with name " + snapshotName + " for collection " + collectionName);
+ Preconditions.checkState(
+ resp.getStatus() == 0,
+ "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus());
+ CLIO.out(
+ "Successfully deleted snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName);
} catch (Exception e) {
- log.error("Failed to delete a snapshot with name {} for collection {}", snapshotName, collectionName, e);
- CLIO.out("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName
- +" due to following error : "+e.getLocalizedMessage());
+ log.error(
+ "Failed to delete a snapshot with name {} for collection {}",
+ snapshotName,
+ collectionName,
+ e);
+ CLIO.out(
+ "Failed to delete a snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
}
}
public void listSnapshots(String collectionName) {
- CollectionAdminRequest.ListSnapshots listSnaps = new CollectionAdminRequest.ListSnapshots(collectionName);
+ CollectionAdminRequest.ListSnapshots listSnaps =
+ new CollectionAdminRequest.ListSnapshots(collectionName);
CollectionAdminResponse resp;
try {
resp = listSnaps.process(solrClient);
- Preconditions.checkState(resp.getStatus() == 0, "The LISTSNAPSHOTS request failed. The status code is " + resp.getStatus());
+ Preconditions.checkState(
+ resp.getStatus() == 0,
+ "The LISTSNAPSHOTS request failed. The status code is " + resp.getStatus());
- NamedList<?> apiResult = (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
+ NamedList<?> apiResult =
+ (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
for (int i = 0; i < apiResult.size(); i++) {
CLIO.out(apiResult.getName(i));
}
} catch (Exception e) {
log.error("Failed to list snapshots for collection {}", collectionName, e);
- CLIO.out("Failed to list snapshots for collection " + collectionName
- +" due to following error : "+e.getLocalizedMessage());
+ CLIO.out(
+ "Failed to list snapshots for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
}
}
@@ -175,14 +226,14 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
} catch (Exception e) {
log.error("Failed to fetch snapshot details", e);
- CLIO.out("Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());
+ CLIO.out(
+ "Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());
}
}
- /**
- * @param pathPrefix optional
- */
- public Map<String, List<String>> getIndexFilesPathForSnapshot(String collectionName, String snapshotName, String pathPrefix)
+ /** @param pathPrefix optional */
+ public Map<String, List<String>> getIndexFilesPathForSnapshot(
+ String collectionName, String snapshotName, String pathPrefix)
throws SolrServerException, IOException {
Map<String, List<String>> result = new HashMap<>();
@@ -195,14 +246,16 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
}
if (meta != null) {
- throw new IllegalArgumentException("The snapshot named " + snapshotName
- + " is not found for collection " + collectionName);
+ throw new IllegalArgumentException(
+ "The snapshot named " + snapshotName + " is not found for collection " + collectionName);
}
- DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
+ DocCollection collectionState =
+ solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
for (Slice s : collectionState.getSlices()) {
List<CoreSnapshotMetaData> replicaSnaps = meta.getReplicaSnapshotsForShard(s.getName());
- // Prepare a list of *existing* replicas (since one or more replicas could have been deleted after the snapshot creation).
+ // Prepare a list of *existing* replicas (since one or more replicas could have been deleted
+ // after the snapshot creation).
List<CoreSnapshotMetaData> availableReplicas = new ArrayList<>();
for (CoreSnapshotMetaData m : replicaSnaps) {
if (isReplicaAvailable(s, m.getCoreName())) {
@@ -212,8 +265,12 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
if (availableReplicas.isEmpty()) {
throw new IllegalArgumentException(
- "The snapshot named " + snapshotName + " not found for shard "
- + s.getName() + " of collection " + collectionName);
+ "The snapshot named "
+ + snapshotName
+ + " not found for shard "
+ + s.getName()
+ + " of collection "
+ + collectionName);
}
// Prefer a leader replica (at the time when the snapshot was created).
@@ -242,41 +299,56 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
return result;
}
- /**
- * @param pathPrefix optional
- */
- public void buildCopyListings(String collectionName, String snapshotName, String localFsPath, String pathPrefix)
+ /** @param pathPrefix optional */
+ public void buildCopyListings(
+ String collectionName, String snapshotName, String localFsPath, String pathPrefix)
throws SolrServerException, IOException {
- Map<String, List<String>> paths = getIndexFilesPathForSnapshot(collectionName, snapshotName, pathPrefix);
- for (Map.Entry<String,List<String>> entry : paths.entrySet()) {
+ Map<String, List<String>> paths =
+ getIndexFilesPathForSnapshot(collectionName, snapshotName, pathPrefix);
+ for (Map.Entry<String, List<String>> entry : paths.entrySet()) {
// TODO: this used to trim - check if that's needed
// Using Paths.get instead of Path.of because of conflict with o.a.hadoop.fs.Path
Files.write(Paths.get(localFsPath, entry.getKey()), entry.getValue());
}
}
- public void backupCollectionMetaData(String collectionName, String snapshotName, String backupLoc) throws SolrServerException, IOException {
+ public void backupCollectionMetaData(String collectionName, String snapshotName, String backupLoc)
+ throws SolrServerException, IOException {
// Backup the collection meta-data
- CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName);
+ CollectionAdminRequest.Backup backup =
+ new CollectionAdminRequest.Backup(collectionName, snapshotName);
backup.setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
backup.setLocation(backupLoc);
CollectionAdminResponse resp = backup.process(solrClient);
- Preconditions.checkState(resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus());
+ Preconditions.checkState(
+ resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus());
}
- /**
- * @param pathPrefix optional
- */
- public void prepareForExport(String collectionName, String snapshotName, String localFsPath, String pathPrefix, String destPath) {
+ /** @param pathPrefix optional */
+ public void prepareForExport(
+ String collectionName,
+ String snapshotName,
+ String localFsPath,
+ String pathPrefix,
+ String destPath) {
try {
buildCopyListings(collectionName, snapshotName, localFsPath, pathPrefix);
CLIO.out("Successfully prepared copylisting for the snapshot export.");
} catch (Exception e) {
- log.error("Failed to prepare a copylisting for snapshot with name {} for collection {}", snapshotName, collectionName, e);
-
- CLIO.out("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
- + collectionName + " due to following error : " + e.getLocalizedMessage());
+ log.error(
+ "Failed to prepare a copylisting for snapshot with name {} for collection {}",
+ snapshotName,
+ collectionName,
+ e);
+
+ CLIO.out(
+ "Failed to prepare a copylisting for snapshot with name "
+ + snapshotName
+ + " for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
System.exit(1);
}
@@ -285,16 +357,24 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
CLIO.out("Successfully backed up collection meta-data");
} catch (Exception e) {
log.error("Failed to backup collection meta-data for collection {}", collectionName, e);
- CLIO.out("Failed to backup collection meta-data for collection " + collectionName
- + " due to following error : " + e.getLocalizedMessage());
+ CLIO.out(
+ "Failed to backup collection meta-data for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
System.exit(1);
}
}
- public void exportSnapshot(String collectionName, String snapshotName, String destPath, Optional<String> backupRepo,
+ public void exportSnapshot(
+ String collectionName,
+ String snapshotName,
+ String destPath,
+ Optional<String> backupRepo,
Optional<String> asyncReqId) {
try {
- CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName);
+ CollectionAdminRequest.Backup backup =
+ new CollectionAdminRequest.Backup(collectionName, snapshotName);
backup.setCommitName(snapshotName);
backup.setIndexBackupStrategy(CollectionAdminParams.COPY_FILES_STRATEGY);
backup.setLocation(destPath);
@@ -305,8 +385,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
backup.processAsync(asyncReqId.orElse(null), solrClient);
} catch (Exception e) {
log.error("Failed to backup collection meta-data for collection {}", collectionName, e);
- CLIO.out("Failed to backup collection meta-data for collection " + collectionName
- + " due to following error : " + e.getLocalizedMessage());
+ CLIO.out(
+ "Failed to backup collection meta-data for collection "
+ + collectionName
+ + " due to following error : "
+ + e.getLocalizedMessage());
System.exit(1);
}
}
@@ -315,27 +398,68 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
CommandLineParser parser = new PosixParser();
Options options = new Options();
- options.addOption(null, CREATE, true, "This command will create a snapshot with the specified name");
- options.addOption(null, DELETE, true, "This command will delete a snapshot with the specified name");
- options.addOption(null, LIST, false, "This command will list all the named snapshots for the specified collection.");
- options.addOption(null, DESCRIBE, true, "This command will print details for a named snapshot for the specified collection.");
- options.addOption(null, PREPARE_FOR_EXPORT, true, "This command will prepare copylistings for the specified snapshot."
- + " This command should only be used only if Solr is deployed with Hadoop and collection index files are stored on a shared"
- + " file-system e.g. HDFS");
- options.addOption(null, EXPORT_SNAPSHOT, true, "This command will create a backup for the specified snapshot.");
- options.addOption(null, HELP, false, "This command will print the help message for the snapshots related commands.");
- options.addOption(TEMP_DIR, true, "This parameter specifies the path of a temporary directory on local filesystem"
- + " during prepare-snapshot-export command.");
- options.addOption(DEST_DIR, true, "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related"
- + " information should be stored.");
- options.addOption(COLLECTION, true, "This parameter specifies the name of the collection to be used during snapshot operation");
- options.addOption(SOLR_ZK_ENSEMBLE, true, "This parameter specifies the Solr Zookeeper ensemble address");
- options.addOption(HDFS_PATH_PREFIX, true, "This parameter specifies the HDFS URI prefix to be used"
- + " during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS.");
- options.addOption(BACKUP_REPO_NAME, true, "This parameter specifies the name of the backup repository to be used"
- + " during snapshot export preparation");
- options.addOption(ASYNC_REQ_ID, true, "This parameter specifies the async request identifier to be used"
- + " during snapshot export preparation");
+ options.addOption(
+ null, CREATE, true, "This command will create a snapshot with the specified name");
+ options.addOption(
+ null, DELETE, true, "This command will delete a snapshot with the specified name");
+ options.addOption(
+ null,
+ LIST,
+ false,
+ "This command will list all the named snapshots for the specified collection.");
+ options.addOption(
+ null,
+ DESCRIBE,
+ true,
+ "This command will print details for a named snapshot for the specified collection.");
+ options.addOption(
+ null,
+ PREPARE_FOR_EXPORT,
+ true,
+ "This command will prepare copylistings for the specified snapshot."
+ + " This command should only be used only if Solr is deployed with Hadoop and collection index files are stored on a shared"
+ + " file-system e.g. HDFS");
+ options.addOption(
+ null,
+ EXPORT_SNAPSHOT,
+ true,
+ "This command will create a backup for the specified snapshot.");
+ options.addOption(
+ null,
+ HELP,
+ false,
+ "This command will print the help message for the snapshots related commands.");
+ options.addOption(
+ TEMP_DIR,
+ true,
+ "This parameter specifies the path of a temporary directory on local filesystem"
+ + " during prepare-snapshot-export command.");
+ options.addOption(
+ DEST_DIR,
+ true,
+ "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related"
+ + " information should be stored.");
+ options.addOption(
+ COLLECTION,
+ true,
+ "This parameter specifies the name of the collection to be used during snapshot operation");
+ options.addOption(
+ SOLR_ZK_ENSEMBLE, true, "This parameter specifies the Solr Zookeeper ensemble address");
+ options.addOption(
+ HDFS_PATH_PREFIX,
+ true,
+ "This parameter specifies the HDFS URI prefix to be used"
+ + " during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS.");
+ options.addOption(
+ BACKUP_REPO_NAME,
+ true,
+ "This parameter specifies the name of the backup repository to be used"
+ + " during snapshot export preparation");
+ options.addOption(
+ ASYNC_REQ_ID,
+ true,
+ "This parameter specifies the async request identifier to be used"
+ + " during snapshot export preparation");
CommandLine cmd = null;
try {
@@ -346,9 +470,14 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
System.exit(1);
}
- if (cmd.hasOption(CREATE) || cmd.hasOption(DELETE) || cmd.hasOption(LIST) || cmd.hasOption(DESCRIBE)
- || cmd.hasOption(PREPARE_FOR_EXPORT) || cmd.hasOption(EXPORT_SNAPSHOT)) {
- try (SolrSnapshotsTool tool = new SolrSnapshotsTool(requiredArg(options, cmd, SOLR_ZK_ENSEMBLE))) {
+ if (cmd.hasOption(CREATE)
+ || cmd.hasOption(DELETE)
+ || cmd.hasOption(LIST)
+ || cmd.hasOption(DESCRIBE)
+ || cmd.hasOption(PREPARE_FOR_EXPORT)
+ || cmd.hasOption(EXPORT_SNAPSHOT)) {
+ try (SolrSnapshotsTool tool =
+ new SolrSnapshotsTool(requiredArg(options, cmd, SOLR_ZK_ENSEMBLE))) {
if (cmd.hasOption(CREATE)) {
String snapshotName = cmd.getOptionValue(CREATE);
String collectionName = requiredArg(options, cmd, COLLECTION);
@@ -380,14 +509,16 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
new URI(pathPrefix);
} catch (URISyntaxException e) {
CLIO.out(
- "The specified File system path prefix " + pathPrefix
- + " is invalid. The error is " + e.getLocalizedMessage());
+ "The specified File system path prefix "
+ + pathPrefix
+ + " is invalid. The error is "
+ + e.getLocalizedMessage());
System.exit(1);
}
}
tool.prepareForExport(collectionName, snapshotName, localFsDir, pathPrefix, hdfsOpDir);
- } else if (cmd.hasOption(EXPORT_SNAPSHOT)) {
+ } else if (cmd.hasOption(EXPORT_SNAPSHOT)) {
String snapshotName = cmd.getOptionValue(EXPORT_SNAPSHOT);
String collectionName = requiredArg(options, cmd, COLLECTION);
String destDir = requiredArg(options, cmd, DEST_DIR);
@@ -397,7 +528,7 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
tool.exportSnapshot(collectionName, snapshotName, destDir, backupRepo, asyncReqId);
}
}
- } else if (cmd.hasOption(HELP)) {
+ } else if (cmd.hasOption(HELP)) {
printHelp(options);
} else {
CLIO.out("Unknown command specified.");
@@ -414,8 +545,8 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
return cmd.getOptionValue(optVal);
}
- private static boolean isReplicaAvailable (Slice s, String coreName) {
- for (Replica r: s.getReplicas()) {
+ private static boolean isReplicaAvailable(Slice s, String coreName) {
+ for (Replica r : s.getReplicas()) {
if (coreName.equals(r.getCoreName())) {
return true;
}
@@ -425,16 +556,18 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(String collectionName)
throws SolrServerException, IOException {
- CollectionAdminRequest.ListSnapshots listSnapshots = new CollectionAdminRequest.ListSnapshots(collectionName);
+ CollectionAdminRequest.ListSnapshots listSnapshots =
+ new CollectionAdminRequest.ListSnapshots(collectionName);
CollectionAdminResponse resp = listSnapshots.process(solrClient);
Preconditions.checkState(resp.getStatus() == 0);
- NamedList<?> apiResult = (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
+ NamedList<?> apiResult =
+ (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
for (int i = 0; i < apiResult.size(); i++) {
- result.add(new CollectionSnapshotMetaData((NamedList<?>)apiResult.getVal(i)));
+ result.add(new CollectionSnapshotMetaData((NamedList<?>) apiResult.getVal(i)));
}
return result;
@@ -446,7 +579,8 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
helpFooter.append("snapshotscli.sh --create snapshot-1 -c books -z localhost:2181 \n");
helpFooter.append("snapshotscli.sh --list -c books -z localhost:2181 \n");
helpFooter.append("snapshotscli.sh --describe snapshot-1 -c books -z localhost:2181 \n");
- helpFooter.append("snapshotscli.sh --export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath -i req_0 \n");
+ helpFooter.append(
+ "snapshotscli.sh --export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath -i req_0 \n");
helpFooter.append("snapshotscli.sh --delete snapshot-1 -c books -z localhost:2181 \n");
HelpFormatter formatter = new HelpFormatter();
@@ -459,8 +593,7 @@ public class SolrSnapshotsTool implements Closeable, CLIO {
public int compare(T o1, T o2) {
String s1 = o1.hasLongOpt() ? o1.getLongOpt() : o1.getOpt();
String s2 = o2.hasLongOpt() ? o2.getLongOpt() : o2.getOpt();
- return OPTION_HELP_ORDER.indexOf(s1) - OPTION_HELP_ORDER.indexOf(s2);
+ return OPTION_HELP_ORDER.indexOf(s1) - OPTION_HELP_ORDER.indexOf(s2);
}
-}
-
+ }
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java b/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java
index 8f8b12d..5ab5765 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/CheckHdfsIndex.java
@@ -17,7 +17,6 @@
package org.apache.solr.index.hdfs;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.lucene.index.CheckIndex;
@@ -45,11 +44,13 @@ public class CheckHdfsIndex {
}
if (!CheckIndex.assertsOn()) {
- System.out.println("\nNOTE: testing will be more thorough if you run java with '-ea:org.apache.lucene...', so assertions are enabled");
+ System.out.println(
+ "\nNOTE: testing will be more thorough if you run java with '-ea:org.apache.lucene...', so assertions are enabled");
}
if (opts.getDirImpl() != null) {
- System.out.println("\nIgnoring specified -dir-impl, instead using " + HdfsDirectory.class.getSimpleName());
+ System.out.println(
+ "\nIgnoring specified -dir-impl, instead using " + HdfsDirectory.class.getSimpleName());
}
Path indexPath = new Path(opts.getIndexPath());
@@ -64,7 +65,8 @@ public class CheckHdfsIndex {
return 1;
}
- try (Directory dir = directory; CheckIndex checker = new CheckIndex(dir)) {
+ try (Directory dir = directory;
+ CheckIndex checker = new CheckIndex(dir)) {
opts.setOut(System.out);
return checker.doCheck(opts);
}
@@ -76,7 +78,7 @@ public class CheckHdfsIndex {
HdfsUtil.addHdfsResources(conf, confDir);
String fsScheme = path.toUri().getScheme();
- if(fsScheme != null) {
+ if (fsScheme != null) {
conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
}
return conf;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/package-info.java
index 9b23898..3142414 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/package-info.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/index/hdfs/package-info.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-/**
- * An HDFS CheckIndex implementation.
- */
-package org.apache.solr.index.hdfs;
+/** An HDFS CheckIndex implementation. */
+package org.apache.solr.index.hdfs;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCache.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCache.java
index 9deef6c..3ea8fb8 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCache.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCache.java
@@ -16,24 +16,21 @@
*/
package org.apache.solr.store.blockcache;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicInteger;
-
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
import com.github.benmanes.caffeine.cache.RemovalListener;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class BlockCache {
-
+
public static final int _128M = 134217728;
public static final int _32K = 32768;
- private final Cache<BlockCacheKey,BlockCacheLocation> cache;
+ private final Cache<BlockCacheKey, BlockCacheLocation> cache;
private final ByteBuffer[] banks;
private final BlockLocks[] locks;
private final AtomicInteger[] lockCounters;
@@ -46,22 +43,21 @@ public class BlockCache {
public static interface OnRelease {
public void release(BlockCacheKey blockCacheKey);
}
-
+
public BlockCache(Metrics metrics, boolean directAllocation, long totalMemory) {
this(metrics, directAllocation, totalMemory, _128M);
}
-
- public BlockCache(Metrics metrics, boolean directAllocation,
- long totalMemory, int slabSize) {
+
+ public BlockCache(Metrics metrics, boolean directAllocation, long totalMemory, int slabSize) {
this(metrics, directAllocation, totalMemory, slabSize, _32K);
}
-
- public BlockCache(Metrics metrics, boolean directAllocation,
- long totalMemory, int slabSize, int blockSize) {
+
+ public BlockCache(
+ Metrics metrics, boolean directAllocation, long totalMemory, int slabSize, int blockSize) {
this.metrics = metrics;
numberOfBlocksPerBank = slabSize / blockSize;
int numberOfBanks = (int) (totalMemory / slabSize);
-
+
banks = new ByteBuffer[numberOfBanks];
locks = new BlockLocks[numberOfBanks];
lockCounters = new AtomicInteger[numberOfBanks];
@@ -76,20 +72,20 @@ public class BlockCache {
lockCounters[i] = new AtomicInteger();
}
- RemovalListener<BlockCacheKey,BlockCacheLocation> listener = (blockCacheKey, blockCacheLocation, removalCause) -> releaseLocation(blockCacheKey, blockCacheLocation, removalCause);
+ RemovalListener<BlockCacheKey, BlockCacheLocation> listener =
+ (blockCacheKey, blockCacheLocation, removalCause) ->
+ releaseLocation(blockCacheKey, blockCacheLocation, removalCause);
- cache = Caffeine.newBuilder()
- .removalListener(listener)
- .maximumSize(maxEntries)
- .build();
+ cache = Caffeine.newBuilder().removalListener(listener).maximumSize(maxEntries).build();
this.blockSize = blockSize;
}
-
+
public void release(BlockCacheKey key) {
cache.invalidate(key);
}
-
- private void releaseLocation(BlockCacheKey blockCacheKey, BlockCacheLocation location, RemovalCause removalCause) {
+
+ private void releaseLocation(
+ BlockCacheKey blockCacheKey, BlockCacheLocation location, RemovalCause removalCause) {
if (location == null) {
return;
}
@@ -111,9 +107,10 @@ public class BlockCache {
}
/**
- * This is only best-effort... it's possible for false to be returned, meaning the block was not able to be cached.
- * NOTE: blocks may not currently be updated (false will be returned if the block is already cached)
- * The blockCacheKey is cloned before it is inserted into the map, so it may be reused by clients if desired.
+ * This is only best-effort... it's possible for false to be returned, meaning the block was not
+ * able to be cached. NOTE: blocks may not currently be updated (false will be returned if the
+ * block is already cached) The blockCacheKey is cloned before it is inserted into the map, so it
+ * may be reused by clients if desired.
*
* @param blockCacheKey the key for the block
* @param blockOffset the offset within the block
@@ -122,28 +119,39 @@ public class BlockCache {
* @param length the number of bytes to write.
* @return true if the block was cached/updated
*/
- public boolean store(BlockCacheKey blockCacheKey, int blockOffset,
- byte[] data, int offset, int length) {
+ public boolean store(
+ BlockCacheKey blockCacheKey, int blockOffset, byte[] data, int offset, int length) {
if (length + blockOffset > blockSize) {
- throw new RuntimeException("Buffer size exceeded, expecting max ["
- + blockSize + "] got length [" + length + "] with blockOffset ["
- + blockOffset + "]");
+ throw new RuntimeException(
+ "Buffer size exceeded, expecting max ["
+ + blockSize
+ + "] got length ["
+ + length
+ + "] with blockOffset ["
+ + blockOffset
+ + "]");
}
BlockCacheLocation location = cache.getIfPresent(blockCacheKey);
if (location == null) {
location = new BlockCacheLocation();
if (!findEmptyLocation(location)) {
- // YCS: it looks like when the cache is full (a normal scenario), then two concurrent writes will result in one of them failing
- // because no eviction is done first. The code seems to rely on leaving just a single block empty.
+ // YCS: it looks like when the cache is full (a normal scenario), then two concurrent writes
+ // will result in one of them failing
+ // because no eviction is done first. The code seems to rely on leaving just a single block
+ // empty.
// TODO: simplest fix would be to leave more than one block empty
metrics.blockCacheStoreFail.incrementAndGet();
return false;
}
} else {
- // If we allocated a new block, then it has never been published and is thus never in danger of being concurrently removed.
- // On the other hand, if this is an existing block we are updating, it may concurrently be removed and reused for another
- // purpose (and then our write may overwrite that). This can happen even if clients never try to update existing blocks,
- // since two clients can try to cache the same block concurrently. Because of this, the ability to update an existing
+ // If we allocated a new block, then it has never been published and is thus never in danger
+ // of being concurrently removed.
+ // On the other hand, if this is an existing block we are updating, it may concurrently be
+ // removed and reused for another
+ // purpose (and then our write may overwrite that). This can happen even if clients never try
+ // to update existing blocks,
+ // since two clients can try to cache the same block concurrently. Because of this, the
+ // ability to update an existing
// block has been removed for the time being (see SOLR-10121).
// No metrics to update: we don't count a redundant store as a store fail.
@@ -170,8 +178,8 @@ public class BlockCache {
* @param length the number of bytes to read
* @return true if the block was cached and the bytes were read
*/
- public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer,
- int blockOffset, int off, int length) {
+ public boolean fetch(
+ BlockCacheKey blockCacheKey, byte[] buffer, int blockOffset, int off, int length) {
BlockCacheLocation location = cache.getIfPresent(blockCacheKey);
if (location == null) {
metrics.blockCacheMiss.incrementAndGet();
@@ -195,17 +203,18 @@ public class BlockCache {
metrics.blockCacheHit.incrementAndGet();
return true;
}
-
+
public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer) {
checkLength(buffer);
return fetch(blockCacheKey, buffer, 0, 0, blockSize);
}
-
+
private boolean findEmptyLocation(BlockCacheLocation location) {
// This is a tight loop that will try and find a location to
// place the block before giving up
for (int j = 0; j < 10; j++) {
- OUTER: for (int bankId = 0; bankId < banks.length; bankId++) {
+ OUTER:
+ for (int bankId = 0; bankId < banks.length; bankId++) {
AtomicInteger bitSetCounter = lockCounters[bankId];
BlockLocks bitSet = locks[bankId];
if (bitSetCounter.get() == numberOfBlocksPerBank) {
@@ -215,7 +224,8 @@ public class BlockCache {
// this check needs to spin, if a lock was attempted but not obtained
// the rest of the bank should not be skipped
int bit = bitSet.nextClearBit(0);
- INNER: while (bit != -1) {
+ INNER:
+ while (bit != -1) {
if (bit >= numberOfBlocksPerBank) {
// bit set is full
continue OUTER;
@@ -238,15 +248,18 @@ public class BlockCache {
}
return false;
}
-
+
private void checkLength(byte[] buffer) {
if (buffer.length != blockSize) {
- throw new RuntimeException("Buffer wrong size, expecting [" + blockSize
- + "] got [" + buffer.length + "]");
+ throw new RuntimeException(
+ "Buffer wrong size, expecting [" + blockSize + "] got [" + buffer.length + "]");
}
}
- /** Returns a new copy of the ByteBuffer for the given bank, so it's safe to call position() on w/o additional synchronization */
+ /**
+ * Returns a new copy of the ByteBuffer for the given bank, so it's safe to call position() on w/o
+ * additional synchronization
+ */
private ByteBuffer getBank(int bankId) {
return banks[bankId].duplicate();
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java
index c5397b1..2f2f25f 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java
@@ -16,15 +16,13 @@
*/
package org.apache.solr.store.blockcache;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class BlockCacheKey implements Cloneable {
-
+
private long block;
private int file;
private String path;
-
+
public String getPath() {
return path;
}
@@ -36,19 +34,19 @@ public class BlockCacheKey implements Cloneable {
public long getBlock() {
return block;
}
-
+
public int getFile() {
return file;
}
-
+
public void setBlock(long block) {
this.block = block;
}
-
+
public void setFile(int file) {
this.file = file;
}
-
+
@Override
public int hashCode() {
final int prime = 31;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java
index 7792d90..7ef4688 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java
@@ -17,14 +17,11 @@
package org.apache.solr.store.blockcache;
import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.solr.common.util.SuppressForbidden;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class BlockCacheLocation {
-
+
private int block;
private int bankId;
private long lastAccess;
@@ -35,20 +32,20 @@ public class BlockCacheLocation {
touch();
}
- /** The block within the bank. This has no relationship to the blockId in BlockCacheKey */
+ /** The block within the bank. This has no relationship to the blockId in BlockCacheKey */
public void setBlock(int block) {
this.block = block;
}
-
+
public void setBankId(int bankId) {
this.bankId = bankId;
}
- /** The block within the bank. This has no relationship to the blockId in BlockCacheKey */
+ /** The block within the bank. This has no relationship to the blockId in BlockCacheKey */
public int getBlock() {
return block;
}
-
+
public int getBankId() {
return bankId;
}
@@ -62,17 +59,16 @@ public class BlockCacheLocation {
public long getLastAccess() {
return lastAccess;
}
-
+
public long getNumberOfAccesses() {
return accesses;
}
-
+
public boolean isRemoved() {
return removed.get();
}
-
+
public void setRemoved(boolean removed) {
this.removed.set(removed);
}
-
-}
\ No newline at end of file
+}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectory.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
index fe19f92..b149da7 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
@@ -21,7 +21,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Set;
-
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -34,58 +33,60 @@ import org.apache.solr.store.hdfs.HdfsDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+
public static final long BLOCK_SHIFT = Integer.getInteger("solr.hdfs.blockcache.blockshift", 13);
public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT;
public static final long BLOCK_MOD = BLOCK_SIZE - 1;
-
+
public static long getBlock(long pos) {
return pos >>> BLOCK_SHIFT;
}
-
+
public static long getPosition(long pos) {
return pos & BLOCK_MOD;
}
-
+
public static long getRealPosition(long block, long positionInBlock) {
return (block << BLOCK_SHIFT) + positionInBlock;
}
-
- public static Cache NO_CACHE = new Cache() {
-
- @Override
- public void update(String name, long blockId, int blockOffset,
- byte[] buffer, int offset, int length) {}
-
- @Override
- public boolean fetch(String name, long blockId, int blockOffset, byte[] b,
- int off, int lengthToReadInBlock) {
- return false;
- }
-
- @Override
- public void delete(String name) {
-
- }
-
- @Override
- public long size() {
- return 0;
- }
-
- @Override
- public void renameCacheFile(String source, String dest) {}
- @Override
- public void releaseResources() {}
- };
-
+ public static Cache NO_CACHE =
+ new Cache() {
+
+ @Override
+ public void update(
+ String name, long blockId, int blockOffset, byte[] buffer, int offset, int length) {}
+
+ @Override
+ public boolean fetch(
+ String name,
+ long blockId,
+ int blockOffset,
+ byte[] b,
+ int off,
+ int lengthToReadInBlock) {
+ return false;
+ }
+
+ @Override
+ public void delete(String name) {}
+
+ @Override
+ public long size() {
+ return 0;
+ }
+
+ @Override
+ public void renameCacheFile(String source, String dest) {}
+
+ @Override
+ public void releaseResources() {}
+ };
+
private final int blockSize;
private final String dirName;
private final Cache cache;
@@ -96,15 +97,35 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
private boolean cacheMerges;
private boolean cacheReadOnce;
- public BlockDirectory(String dirName, Directory directory, Cache cache,
- Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
- boolean blockCacheWriteEnabled) throws IOException {
- this(dirName, directory, cache, blockCacheFileTypes, blockCacheReadEnabled, blockCacheWriteEnabled, true, true);
+ public BlockDirectory(
+ String dirName,
+ Directory directory,
+ Cache cache,
+ Set<String> blockCacheFileTypes,
+ boolean blockCacheReadEnabled,
+ boolean blockCacheWriteEnabled)
+ throws IOException {
+ this(
+ dirName,
+ directory,
+ cache,
+ blockCacheFileTypes,
+ blockCacheReadEnabled,
+ blockCacheWriteEnabled,
+ true,
+ true);
}
-
- public BlockDirectory(String dirName, Directory directory, Cache cache,
- Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
- boolean blockCacheWriteEnabled, boolean cacheMerges, boolean cacheReadOnce) throws IOException {
+
+ public BlockDirectory(
+ String dirName,
+ Directory directory,
+ Cache cache,
+ Set<String> blockCacheFileTypes,
+ boolean blockCacheReadEnabled,
+ boolean blockCacheWriteEnabled,
+ boolean cacheMerges,
+ boolean cacheReadOnce)
+ throws IOException {
super(directory);
this.cacheMerges = cacheMerges;
this.cacheReadOnce = cacheReadOnce;
@@ -125,17 +146,16 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
log.info("Block cache on write is disabled");
}
}
-
- private IndexInput openInput(String name, int bufferSize, IOContext context)
- throws IOException {
+
+ private IndexInput openInput(String name, int bufferSize, IOContext context) throws IOException {
final IndexInput source = super.openInput(name, context);
if (useReadCache(name, context)) {
- return new CachedIndexInput(source, blockSize, name,
- getFileCacheName(name), cache, bufferSize);
+ return new CachedIndexInput(
+ source, blockSize, name, getFileCacheName(name), cache, bufferSize);
}
return source;
}
-
+
private boolean isCachableFile(String name) {
for (String ext : blockCacheFileTypes) {
if (name.endsWith(ext)) {
@@ -144,13 +164,12 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
}
return false;
}
-
+
@Override
- public IndexInput openInput(final String name, IOContext context)
- throws IOException {
+ public IndexInput openInput(final String name, IOContext context) throws IOException {
return openInput(name, blockSize, context);
}
-
+
static class CachedIndexInput extends CustomBufferedIndexInput {
private final Store store;
private IndexInput source;
@@ -158,9 +177,14 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
private final long fileLength;
private final String cacheName;
private final Cache cache;
-
- public CachedIndexInput(IndexInput source, int blockSize, String name,
- String cacheName, Cache cache, int bufferSize) {
+
+ public CachedIndexInput(
+ IndexInput source,
+ int blockSize,
+ String name,
+ String cacheName,
+ Cache cache,
+ int bufferSize) {
super(name, bufferSize);
this.source = source;
this.blockSize = blockSize;
@@ -169,22 +193,22 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
this.cache = cache;
store = BufferStore.instance(blockSize);
}
-
+
@Override
public IndexInput clone() {
CachedIndexInput clone = (CachedIndexInput) super.clone();
clone.source = source.clone();
return clone;
}
-
+
@Override
public long length() {
return source.length();
}
-
+
@Override
protected void seekInternal(long pos) throws IOException {}
-
+
@Override
protected void readInternal(byte[] b, int off, int len) throws IOException {
long position = getFilePointer();
@@ -195,9 +219,8 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
off += length;
}
}
-
- private int fetchBlock(long position, byte[] b, int off, int len)
- throws IOException {
+
+ private int fetchBlock(long position, byte[] b, int off, int len) throws IOException {
// read whole block into cache and then provide needed data
long blockId = getBlock(position);
int blockOffset = (int) getPosition(position);
@@ -205,53 +228,52 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
if (checkCache(blockId, blockOffset, b, off, lengthToReadInBlock)) {
return lengthToReadInBlock;
} else {
- readIntoCacheAndResult(blockId, blockOffset, b, off,
- lengthToReadInBlock);
+ readIntoCacheAndResult(blockId, blockOffset, b, off, lengthToReadInBlock);
}
return lengthToReadInBlock;
}
-
- private void readIntoCacheAndResult(long blockId, int blockOffset,
- byte[] b, int off, int lengthToReadInBlock) throws IOException {
+
+ private void readIntoCacheAndResult(
+ long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock)
+ throws IOException {
long position = getRealPosition(blockId, 0);
int length = (int) Math.min(blockSize, fileLength - position);
source.seek(position);
-
+
byte[] buf = store.takeBuffer(blockSize);
source.readBytes(buf, 0, length);
System.arraycopy(buf, blockOffset, b, off, lengthToReadInBlock);
cache.update(cacheName, blockId, 0, buf, 0, blockSize);
store.putBuffer(buf);
}
-
- private boolean checkCache(long blockId, int blockOffset, byte[] b,
- int off, int lengthToReadInBlock) {
- return cache.fetch(cacheName, blockId, blockOffset, b, off,
- lengthToReadInBlock);
+
+ private boolean checkCache(
+ long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) {
+ return cache.fetch(cacheName, blockId, blockOffset, b, off, lengthToReadInBlock);
}
-
+
@Override
protected void closeInternal() throws IOException {
source.close();
}
}
-
+
@Override
public void closeOnShutdown() throws IOException {
log.info("BlockDirectory closing on shutdown");
// we are shutting down, no need to clean up cache
super.close();
}
-
+
@Override
public void close() throws IOException {
try {
String[] files = listAll();
-
+
for (String file : files) {
cache.delete(getFileCacheName(file));
}
-
+
} catch (FileNotFoundException e) {
// the local file system folder may be gone
} finally {
@@ -259,11 +281,11 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
cache.releaseResources();
}
}
-
+
String getFileCacheName(String name) throws IOException {
return getFileCacheLocation(name) + ":" + getFileModified(name);
}
-
+
private long getFileModified(String name) throws IOException {
if (in instanceof FSDirectory) {
File directory = ((FSDirectory) in).getDirectory().toFile();
@@ -278,24 +300,21 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
throw new UnsupportedOperationException();
}
}
-
+
String getFileCacheLocation(String name) {
return dirName + "/" + name;
}
-
+
/**
* Expert: mostly for tests
- *
+ *
* @lucene.experimental
*/
public Cache getCache() {
return cache;
}
-
- /**
- * Determine whether read caching should be used for a particular
- * file/context.
- */
+
+ /** Determine whether read caching should be used for a particular file/context. */
boolean useReadCache(String name, IOContext context) {
if (!blockCacheReadEnabled) {
return false;
@@ -304,27 +323,27 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
return false;
}
switch (context.context) {
- // depending on params, we don't cache on merges or when only reading once
- case MERGE: {
- return cacheMerges;
- }
- case READ: {
- if (context.readOnce) {
- return cacheReadOnce;
- } else {
+ // depending on params, we don't cache on merges or when only reading once
+ case MERGE:
+ {
+ return cacheMerges;
+ }
+ case READ:
+ {
+ if (context.readOnce) {
+ return cacheReadOnce;
+ } else {
+ return true;
+ }
+ }
+ default:
+ {
return true;
}
- }
- default: {
- return true;
- }
}
}
-
- /**
- * Determine whether write caching should be used for a particular
- * file/context.
- */
+
+ /** Determine whether write caching should be used for a particular file/context. */
boolean useWriteCache(String name, IOContext context) {
if (!blockCacheWriteEnabled || name.startsWith(IndexFileNames.PENDING_SEGMENTS)) {
// for safety, don't bother caching pending commits.
@@ -335,19 +354,20 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
return false;
}
switch (context.context) {
- case MERGE: {
- // we currently don't cache any merge context writes
- return false;
- }
- default: {
- return true;
- }
+ case MERGE:
+ {
+ // we currently don't cache any merge context writes
+ return false;
+ }
+ default:
+ {
+ return true;
+ }
}
}
-
+
@Override
- public IndexOutput createOutput(String name, IOContext context)
- throws IOException {
+ public IndexOutput createOutput(String name, IOContext context) throws IOException {
final IndexOutput dest = super.createOutput(name, context);
if (useWriteCache(name, context)) {
return new CachedIndexOutput(this, dest, blockSize, name, cache, blockSize);
@@ -359,7 +379,7 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
cache.delete(getFileCacheName(name));
super.deleteFile(name);
}
-
+
public boolean isBlockCacheReadEnabled() {
return blockCacheReadEnabled;
}
@@ -367,5 +387,4 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDire
public boolean isBlockCacheWriteEnabled() {
return blockCacheWriteEnabled;
}
-
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
index 6e999d5..b0aafa2 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java
@@ -16,67 +16,66 @@
*/
package org.apache.solr.store.blockcache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.solr.store.blockcache.BlockCache.OnRelease;
-import com.github.benmanes.caffeine.cache.Caffeine;
-
-
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class BlockDirectoryCache implements Cache {
private final BlockCache blockCache;
private final AtomicInteger counter = new AtomicInteger();
- private final com.github.benmanes.caffeine.cache.Cache<String,Integer> names;
+ private final com.github.benmanes.caffeine.cache.Cache<String, Integer> names;
private Set<BlockCacheKey> keysToRelease;
private final String path;
private final Metrics metrics;
-
+
public BlockDirectoryCache(BlockCache blockCache, String path, Metrics metrics) {
this(blockCache, path, metrics, false);
}
-
- public BlockDirectoryCache(BlockCache blockCache, String path, Metrics metrics, boolean releaseBlocks) {
+
+ public BlockDirectoryCache(
+ BlockCache blockCache, String path, Metrics metrics, boolean releaseBlocks) {
this.blockCache = blockCache;
this.path = path;
this.metrics = metrics;
-
+
names = Caffeine.newBuilder().maximumSize(50000).build();
-
+
if (releaseBlocks) {
- keysToRelease = Collections.newSetFromMap(new ConcurrentHashMap<BlockCacheKey,Boolean>(1024, 0.75f, 512));
- blockCache.setOnRelease(new OnRelease() {
-
- @Override
- public void release(BlockCacheKey key) {
- keysToRelease.remove(key);
- }
- });
+ keysToRelease =
+ Collections.newSetFromMap(
+ new ConcurrentHashMap<BlockCacheKey, Boolean>(1024, 0.75f, 512));
+ blockCache.setOnRelease(
+ new OnRelease() {
+
+ @Override
+ public void release(BlockCacheKey key) {
+ keysToRelease.remove(key);
+ }
+ });
}
}
/**
* Expert: mostly for tests
- *
+ *
* @lucene.experimental
*/
public BlockCache getBlockCache() {
return blockCache;
}
-
+
@Override
public void delete(String name) {
names.invalidate(name);
}
-
+
@Override
- public void update(String name, long blockId, int blockOffset, byte[] buffer,
- int offset, int length) {
+ public void update(
+ String name, long blockId, int blockOffset, byte[] buffer, int offset, int length) {
Integer file = names.getIfPresent(name);
if (file == null) {
file = counter.incrementAndGet();
@@ -86,14 +85,15 @@ public class BlockDirectoryCache implements Cache {
blockCacheKey.setPath(path);
blockCacheKey.setBlock(blockId);
blockCacheKey.setFile(file);
- if (blockCache.store(blockCacheKey, blockOffset, buffer, offset, length) && keysToRelease != null) {
+ if (blockCache.store(blockCacheKey, blockOffset, buffer, offset, length)
+ && keysToRelease != null) {
keysToRelease.add(blockCacheKey);
}
}
-
+
@Override
- public boolean fetch(String name, long blockId, int blockOffset, byte[] b,
- int off, int lengthToReadInBlock) {
+ public boolean fetch(
+ String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) {
Integer file = names.getIfPresent(name);
if (file == null) {
return false;
@@ -102,16 +102,15 @@ public class BlockDirectoryCache implements Cache {
blockCacheKey.setPath(path);
blockCacheKey.setBlock(blockId);
blockCacheKey.setFile(file);
- boolean fetch = blockCache.fetch(blockCacheKey, b, blockOffset, off,
- lengthToReadInBlock);
+ boolean fetch = blockCache.fetch(blockCacheKey, b, blockOffset, off, lengthToReadInBlock);
return fetch;
}
-
+
@Override
public long size() {
return blockCache.getSize();
}
-
+
@Override
public void renameCacheFile(String source, String dest) {
Integer file = names.getIfPresent(source);
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockLocks.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockLocks.java
index 766a8fc..023bc74 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockLocks.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BlockLocks.java
@@ -17,28 +17,24 @@
package org.apache.solr.store.blockcache;
import java.util.concurrent.atomic.AtomicLongArray;
-
import org.apache.lucene.util.LongBitSet;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class BlockLocks {
-
+
private AtomicLongArray bits;
private int wlen;
-
+
public BlockLocks(long numBits) {
int length = LongBitSet.bits2words(numBits);
bits = new AtomicLongArray(length);
wlen = length;
}
-
+
/**
* Find the next clear bit in the bit set.
- *
- * @param index
- * index
+ *
+ * @param index index
* @return next next bit
*/
public int nextClearBit(int index) {
@@ -46,7 +42,7 @@ public class BlockLocks {
if (i >= wlen) return -1;
int subIndex = index & 0x3f; // index within the word
long word = ~bits.get(i) >> subIndex; // skip all the bits to the right of
- // index
+ // index
if (word != 0) {
return (i << 6) + subIndex + Long.numberOfTrailingZeros(word);
}
@@ -58,13 +54,11 @@ public class BlockLocks {
}
return -1;
}
-
+
/**
- * Thread safe set operation that will set the bit if and only if the bit was
- * not previously set.
- *
- * @param index
- * the index position to set.
+ * Thread safe set operation that will set the bit if and only if the bit was not previously set.
+ *
+ * @param index the index position to set.
* @return returns true if the bit was set and false if it was already set.
*/
public boolean set(int index) {
@@ -83,7 +77,7 @@ public class BlockLocks {
} while (!bits.compareAndSet(wordNum, oword, word));
return true;
}
-
+
public void clear(int index) {
int wordNum = index >> 6;
int bit = index & 0x03f;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BufferStore.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BufferStore.java
index cf93c9d..9c49164 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BufferStore.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/BufferStore.java
@@ -22,24 +22,23 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class BufferStore implements Store {
- private static final Store EMPTY = new Store() {
+ private static final Store EMPTY =
+ new Store() {
- @Override
- public byte[] takeBuffer(int bufferSize) {
- return new byte[bufferSize];
- }
+ @Override
+ public byte[] takeBuffer(int bufferSize) {
+ return new byte[bufferSize];
+ }
- @Override
- public void putBuffer(byte[] buffer) {
- }
- };
+ @Override
+ public void putBuffer(byte[] buffer) {}
+ };
- private final static ConcurrentMap<Integer, BufferStore> bufferStores = new ConcurrentHashMap<>(8192, 0.75f, 512);
+ private static final ConcurrentMap<Integer, BufferStore> bufferStores =
+ new ConcurrentHashMap<>(8192, 0.75f, 512);
private final BlockingQueue<byte[]> buffers;
@@ -56,12 +55,12 @@ public class BufferStore implements Store {
static void clearBufferStores() {
bufferStores.clear();
}
-
- public synchronized static void initNewBuffer(int bufferSize, long totalAmount) {
+
+ public static synchronized void initNewBuffer(int bufferSize, long totalAmount) {
initNewBuffer(bufferSize, totalAmount, null);
}
- public synchronized static void initNewBuffer(int bufferSize, long totalAmount, Metrics metrics) {
+ public static synchronized void initNewBuffer(int bufferSize, long totalAmount, Metrics metrics) {
if (totalAmount == 0) {
return;
}
@@ -77,12 +76,17 @@ public class BufferStore implements Store {
shardBuffercacheLost = metrics.shardBuffercacheLost;
shardBuffercacheAllocate = metrics.shardBuffercacheAllocate;
}
- BufferStore store = new BufferStore(bufferSize, (int) count, shardBuffercacheAllocate, shardBuffercacheLost);
+ BufferStore store =
+ new BufferStore(bufferSize, (int) count, shardBuffercacheAllocate, shardBuffercacheLost);
bufferStores.put(bufferSize, store);
}
}
- private BufferStore(int bufferSize, int count, AtomicLong shardBuffercacheAllocate, AtomicLong shardBuffercacheLost) {
+ private BufferStore(
+ int bufferSize,
+ int count,
+ AtomicLong shardBuffercacheAllocate,
+ AtomicLong shardBuffercacheLost) {
this.bufferSize = bufferSize;
this.shardBuffercacheAllocate = shardBuffercacheAllocate;
this.shardBuffercacheLost = shardBuffercacheLost;
@@ -108,8 +112,12 @@ public class BufferStore implements Store {
@Override
public byte[] takeBuffer(int bufferSize) {
if (this.bufferSize != bufferSize) {
- throw new RuntimeException("Buffer with length [" + bufferSize + "] does not match buffer size of ["
- + bufferSize + "]");
+ throw new RuntimeException(
+ "Buffer with length ["
+ + bufferSize
+ + "] does not match buffer size of ["
+ + bufferSize
+ + "]");
}
return newBuffer(buffers.poll());
}
@@ -120,8 +128,12 @@ public class BufferStore implements Store {
return;
}
if (buffer.length != bufferSize) {
- throw new RuntimeException("Buffer with length [" + buffer.length + "] does not match buffer size of ["
- + bufferSize + "]");
+ throw new RuntimeException(
+ "Buffer with length ["
+ + buffer.length
+ + "] does not match buffer size of ["
+ + bufferSize
+ + "]");
}
checkReturn(buffers.offer(buffer));
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Cache.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Cache.java
index 71c05af..0974bad 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Cache.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Cache.java
@@ -16,54 +16,39 @@
*/
package org.apache.solr.store.blockcache;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public interface Cache {
-
+
/**
* Remove a file from the cache.
- *
- * @param name
- * cache file name
+ *
+ * @param name cache file name
*/
void delete(String name);
-
- /**
- * Update the content of the specified cache file. Creates cache entry if
- * necessary.
- *
- */
- void update(String name, long blockId, int blockOffset, byte[] buffer,
- int offset, int length);
-
+
+ /** Update the content of the specified cache file. Creates cache entry if necessary. */
+ void update(String name, long blockId, int blockOffset, byte[] buffer, int offset, int length);
+
/**
* Fetch the specified cache file content.
- *
+ *
* @return true if cached content found, otherwise return false
*/
- boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off,
- int lengthToReadInBlock);
-
- /**
- * Number of entries in the cache.
- */
+ boolean fetch(
+ String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock);
+
+ /** Number of entries in the cache. */
long size();
-
+
/**
- * Expert: Rename the specified file in the cache. Allows a file to be moved
- * without invalidating the cache.
- *
- * @param source
- * original name
- * @param dest
- * final name
+ * Expert: Rename the specified file in the cache. Allows a file to be moved without invalidating
+ * the cache.
+ *
+ * @param source original name
+ * @param dest final name
*/
void renameCacheFile(String source, String dest);
- /**
- * Release any resources associated with the cache.
- */
+ /** Release any resources associated with the cache. */
void releaseResources();
-
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
index b0ffa9e..9750e2b 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
@@ -17,13 +17,13 @@
package org.apache.solr.store.blockcache;
import java.io.IOException;
-
import org.apache.lucene.store.IndexOutput;
/**
- * Cache the blocks as they are written. The cache file name is the name of
- * the file until the file is closed, at which point the cache is updated
- * to include the last modified date (which is unknown until that point).
+ * Cache the blocks as they are written. The cache file name is the name of the file until the file
+ * is closed, at which point the cache is updated to include the last modified date (which is
+ * unknown until that point).
+ *
* @lucene.experimental
*/
public class CachedIndexOutput extends ReusedBufferedIndexOutput {
@@ -33,9 +33,14 @@ public class CachedIndexOutput extends ReusedBufferedIndexOutput {
private final String name;
private final String location;
private final Cache cache;
-
- public CachedIndexOutput(BlockDirectory directory, IndexOutput dest,
- int blockSize, String name, Cache cache, int bufferSize) {
+
+ public CachedIndexOutput(
+ BlockDirectory directory,
+ IndexOutput dest,
+ int blockSize,
+ String name,
+ Cache cache,
+ int bufferSize) {
super("dest=" + dest + " name=" + name, name, bufferSize);
this.directory = directory;
this.dest = dest;
@@ -50,25 +55,22 @@ public class CachedIndexOutput extends ReusedBufferedIndexOutput {
dest.close();
cache.renameCacheFile(location, directory.getFileCacheName(name));
}
-
- private int writeBlock(long position, byte[] b, int offset, int length)
- throws IOException {
+
+ private int writeBlock(long position, byte[] b, int offset, int length) throws IOException {
// read whole block into cache and then provide needed data
long blockId = BlockDirectory.getBlock(position);
int blockOffset = (int) BlockDirectory.getPosition(position);
int lengthToWriteInBlock = Math.min(length, blockSize - blockOffset);
-
+
// write the file and copy into the cache
dest.writeBytes(b, offset, lengthToWriteInBlock);
- cache.update(location, blockId, blockOffset, b, offset,
- lengthToWriteInBlock);
-
+ cache.update(location, blockId, blockOffset, b, offset, lengthToWriteInBlock);
+
return lengthToWriteInBlock;
}
-
+
@Override
- public void writeInternal(byte[] b, int offset, int length)
- throws IOException {
+ public void writeInternal(byte[] b, int offset, int length) throws IOException {
long position = getBufferStart();
while (length > 0) {
int len = writeBlock(position, b, offset, length);
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
index 50f8bf6..752ebe3 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
@@ -18,59 +18,57 @@ package org.apache.solr.store.blockcache;
import java.io.EOFException;
import java.io.IOException;
-
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public abstract class CustomBufferedIndexInput extends IndexInput {
-
- public static final int BUFFER_SIZE = Integer.getInteger("solr.hdfs.readbuffer.size.default", 32768);
-
+
+ public static final int BUFFER_SIZE =
+ Integer.getInteger("solr.hdfs.readbuffer.size.default", 32768);
+
private int bufferSize = BUFFER_SIZE;
-
+
protected byte[] buffer;
-
+
private long bufferStart = 0; // position in file of buffer
private int bufferLength = 0; // end of valid bytes
private int bufferPosition = 0; // next byte to read
-
+
private final Store store;
-
+
@Override
public byte readByte() throws IOException {
if (bufferPosition >= bufferLength) refill();
return buffer[bufferPosition++];
}
-
+
public CustomBufferedIndexInput(String resourceDesc) {
this(resourceDesc, BUFFER_SIZE);
}
-
+
public CustomBufferedIndexInput(String resourceDesc, int bufferSize) {
super(resourceDesc);
checkBufferSize(bufferSize);
this.bufferSize = bufferSize;
this.store = BufferStore.instance(bufferSize);
}
-
+
private void checkBufferSize(int bufferSize) {
- if (bufferSize <= 0) throw new IllegalArgumentException(
- "bufferSize must be greater than 0 (got " + bufferSize + ")");
+ if (bufferSize <= 0)
+ throw new IllegalArgumentException(
+ "bufferSize must be greater than 0 (got " + bufferSize + ")");
}
-
+
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
readBytes(b, offset, len, true);
}
-
+
@Override
- public void readBytes(byte[] b, int offset, int len, boolean useBuffer)
- throws IOException {
-
+ public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
+
if (len <= (bufferLength - bufferPosition)) {
// the buffer contains enough data to satisfy this request
if (len > 0) // to allow b to be null if len is 0...
@@ -154,7 +152,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
end = length();
int newLength = (int) (end - start);
if (newLength <= 0) throw new EOFException("read past EOF: " + this);
-
+
if (buffer == null) {
buffer = store.takeBuffer(bufferSize);
seekInternal(bufferStart);
@@ -164,40 +162,36 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
bufferStart = start;
bufferPosition = 0;
}
-
+
@Override
public final void close() throws IOException {
closeInternal();
store.putBuffer(buffer);
buffer = null;
}
-
+
protected abstract void closeInternal() throws IOException;
-
+
/**
- * Expert: implements buffer refill. Reads bytes from the current position in
- * the input.
- *
- * @param b
- * the array to read bytes into
- * @param offset
- * the offset in the array to start storing bytes
- * @param length
- * the number of bytes to read
+ * Expert: implements buffer refill. Reads bytes from the current position in the input.
+ *
+ * @param b the array to read bytes into
+ * @param offset the offset in the array to start storing bytes
+ * @param length the number of bytes to read
*/
- protected abstract void readInternal(byte[] b, int offset, int length)
- throws IOException;
-
+ protected abstract void readInternal(byte[] b, int offset, int length) throws IOException;
+
@Override
public long getFilePointer() {
return bufferStart + bufferPosition;
}
-
+
@Override
public void seek(long pos) throws IOException {
- if (pos >= bufferStart && pos < (bufferStart + bufferLength)) bufferPosition = (int) (pos - bufferStart); // seek
- // within
- // buffer
+ if (pos >= bufferStart && pos < (bufferStart + bufferLength))
+ bufferPosition = (int) (pos - bufferStart); // seek
+ // within
+ // buffer
else {
bufferStart = pos;
bufferPosition = 0;
@@ -205,39 +199,38 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
seekInternal(pos);
}
}
-
+
/**
- * Expert: implements seek. Sets current position in this file, where the next
- * {@link #readInternal(byte[],int,int)} will occur.
- *
+ * Expert: implements seek. Sets current position in this file, where the next {@link
+ * #readInternal(byte[],int,int)} will occur.
+ *
* @see #readInternal(byte[],int,int)
*/
protected abstract void seekInternal(long pos) throws IOException;
-
+
@Override
public IndexInput clone() {
CustomBufferedIndexInput clone = (CustomBufferedIndexInput) super.clone();
-
+
clone.buffer = null;
clone.bufferLength = 0;
clone.bufferPosition = 0;
clone.bufferStart = getFilePointer();
-
+
return clone;
}
-
+
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
}
/**
- * Flushes the in-memory bufer to the given output, copying at most
- * <code>numBytes</code>.
- * <p>
- * <b>NOTE:</b> this method does not refill the buffer, however it does
- * advance the buffer position.
- *
+ * Flushes the in-memory bufer to the given output, copying at most <code>numBytes</code>.
+ *
+ * <p><b>NOTE:</b> this method does not refill the buffer, however it does advance the buffer
+ * position.
+ *
* @return the number of bytes actually flushed from the in-memory buffer.
*/
protected int flushBuffer(IndexOutput out, long numBytes) throws IOException {
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Metrics.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Metrics.java
index 76bb265..9abdcb4 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Metrics.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Metrics.java
@@ -19,7 +19,6 @@ package org.apache.solr.store.blockcache;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricsContext;
@@ -32,7 +31,6 @@ import org.apache.solr.search.SolrCacheBase;
*/
public class Metrics extends SolrCacheBase implements SolrInfoBean {
-
public AtomicLong blockCacheSize = new AtomicLong(0);
public AtomicLong blockCacheHit = new AtomicLong(0);
public AtomicLong blockCacheMiss = new AtomicLong(0);
@@ -45,9 +43,9 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean {
private AtomicLong blockCacheEviction_last = new AtomicLong(0);
public AtomicLong blockCacheStoreFail_last = new AtomicLong(0);
-
// These are used by the BufferStore (just a generic cache of byte[]).
- // TODO: If this (the Store) is a good idea, we should make it more general and use it across more places in Solr.
+ // TODO: If this (the Store) is a good idea, we should make it more general and use it across more
+ // places in Solr.
public AtomicLong shardBuffercacheAllocate = new AtomicLong(0);
public AtomicLong shardBuffercacheLost = new AtomicLong(0);
@@ -59,49 +57,63 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean {
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- metricsMap = new MetricsMap(map -> {
- long now = System.nanoTime();
- long delta = Math.max(now - previous, 1);
- double seconds = delta / 1000000000.0;
-
- long hits_total = blockCacheHit.get();
- long hits_delta = hits_total - blockCacheHit_last.get();
- blockCacheHit_last.set(hits_total);
-
- long miss_total = blockCacheMiss.get();
- long miss_delta = miss_total - blockCacheMiss_last.get();
- blockCacheMiss_last.set(miss_total);
-
- long evict_total = blockCacheEviction.get();
- long evict_delta = evict_total - blockCacheEviction_last.get();
- blockCacheEviction_last.set(evict_total);
-
- long storeFail_total = blockCacheStoreFail.get();
- long storeFail_delta = storeFail_total - blockCacheStoreFail_last.get();
- blockCacheStoreFail_last.set(storeFail_total);
-
- long lookups_delta = hits_delta + miss_delta;
- long lookups_total = hits_total + miss_total;
-
- map.put("size", blockCacheSize.get());
- map.put("lookups", lookups_total);
- map.put("hits", hits_total);
- map.put("evictions", evict_total);
- map.put("storeFails", storeFail_total);
- map.put("hitratio_current", calcHitRatio(lookups_delta, hits_delta)); // hit ratio since the last call
- map.put("lookups_persec", getPerSecond(lookups_delta, seconds)); // lookups per second since the last call
- map.put("hits_persec", getPerSecond(hits_delta, seconds)); // hits per second since the last call
- map.put("evictions_persec", getPerSecond(evict_delta, seconds)); // evictions per second since the last call
- map.put("storeFails_persec", getPerSecond(storeFail_delta, seconds)); // evictions per second since the last call
- map.put("time_delta", seconds); // seconds since last call
-
- // TODO: these aren't really related to the BlockCache
- map.put("buffercache.allocations", getPerSecond(shardBuffercacheAllocate.getAndSet(0), seconds));
- map.put("buffercache.lost", getPerSecond(shardBuffercacheLost.getAndSet(0), seconds));
-
- previous = now;
-
- });
+ metricsMap =
+ new MetricsMap(
+ map -> {
+ long now = System.nanoTime();
+ long delta = Math.max(now - previous, 1);
+ double seconds = delta / 1000000000.0;
+
+ long hits_total = blockCacheHit.get();
+ long hits_delta = hits_total - blockCacheHit_last.get();
+ blockCacheHit_last.set(hits_total);
+
+ long miss_total = blockCacheMiss.get();
+ long miss_delta = miss_total - blockCacheMiss_last.get();
+ blockCacheMiss_last.set(miss_total);
+
+ long evict_total = blockCacheEviction.get();
+ long evict_delta = evict_total - blockCacheEviction_last.get();
+ blockCacheEviction_last.set(evict_total);
+
+ long storeFail_total = blockCacheStoreFail.get();
+ long storeFail_delta = storeFail_total - blockCacheStoreFail_last.get();
+ blockCacheStoreFail_last.set(storeFail_total);
+
+ long lookups_delta = hits_delta + miss_delta;
+ long lookups_total = hits_total + miss_total;
+
+ map.put("size", blockCacheSize.get());
+ map.put("lookups", lookups_total);
+ map.put("hits", hits_total);
+ map.put("evictions", evict_total);
+ map.put("storeFails", storeFail_total);
+ map.put(
+ "hitratio_current",
+ calcHitRatio(lookups_delta, hits_delta)); // hit ratio since the last call
+ map.put(
+ "lookups_persec",
+ getPerSecond(lookups_delta, seconds)); // lookups per second since the last call
+ map.put(
+ "hits_persec",
+ getPerSecond(hits_delta, seconds)); // hits per second since the last call
+ map.put(
+ "evictions_persec",
+ getPerSecond(evict_delta, seconds)); // evictions per second since the last call
+ map.put(
+ "storeFails_persec",
+ getPerSecond(
+ storeFail_delta, seconds)); // evictions per second since the last call
+ map.put("time_delta", seconds); // seconds since last call
+
+ // TODO: these aren't really related to the BlockCache
+ map.put(
+ "buffercache.allocations",
+ getPerSecond(shardBuffercacheAllocate.getAndSet(0), seconds));
+ map.put("buffercache.lost", getPerSecond(shardBuffercacheLost.getAndSet(0), seconds));
+
+ previous = now;
+ });
solrMetricsContext.gauge(metricsMap, true, getName(), getCategory().toString(), scope);
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
index d862539..5637da1 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java
@@ -17,20 +17,17 @@
package org.apache.solr.store.blockcache;
import java.io.IOException;
-
import org.apache.lucene.store.IndexOutput;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public abstract class ReusedBufferedIndexOutput extends IndexOutput {
-
+
public static final int BUFFER_SIZE = 1024;
-
+
private int bufferSize = BUFFER_SIZE;
-
+
protected byte[] buffer;
-
+
/** position in the file of buffer */
private long bufferStart = 0;
/** end of valid bytes */
@@ -39,13 +36,13 @@ public abstract class ReusedBufferedIndexOutput extends IndexOutput {
private int bufferPosition = 0;
/** total length of the file */
private long fileLength = 0;
-
+
private final Store store;
-
+
public ReusedBufferedIndexOutput(String resourceDescription, String name) {
this(resourceDescription, name, BUFFER_SIZE);
}
-
+
public ReusedBufferedIndexOutput(String resourceDescription, String name, int bufferSize) {
super(resourceDescription, name);
checkBufferSize(bufferSize);
@@ -53,27 +50,28 @@ public abstract class ReusedBufferedIndexOutput extends IndexOutput {
store = BufferStore.instance(bufferSize);
buffer = store.takeBuffer(this.bufferSize);
}
-
+
protected long getBufferStart() {
return bufferStart;
}
-
+
private void checkBufferSize(int bufferSize) {
- if (bufferSize <= 0) throw new IllegalArgumentException(
- "bufferSize must be greater than 0 (got " + bufferSize + ")");
+ if (bufferSize <= 0)
+ throw new IllegalArgumentException(
+ "bufferSize must be greater than 0 (got " + bufferSize + ")");
}
-
+
/** Write the buffered bytes to cache */
protected void flushBufferToCache() throws IOException {
writeInternal(buffer, 0, bufferLength);
-
+
bufferStart += bufferLength;
bufferLength = 0;
bufferPosition = 0;
}
-
+
protected abstract void closeInternal() throws IOException;
-
+
@Override
public void close() throws IOException {
flushBufferToCache();
@@ -81,12 +79,12 @@ public abstract class ReusedBufferedIndexOutput extends IndexOutput {
store.putBuffer(buffer);
buffer = null;
}
-
+
@Override
public long getFilePointer() {
return bufferStart + bufferPosition;
}
-
+
@Override
public void writeByte(byte b) throws IOException {
if (bufferPosition >= bufferSize) {
@@ -100,21 +98,17 @@ public abstract class ReusedBufferedIndexOutput extends IndexOutput {
bufferLength = bufferPosition;
}
}
-
+
/**
- * Expert: implements buffer flushing to cache. Writes bytes to the current
- * position in the output.
- *
- * @param b
- * the array of bytes to write
- * @param offset
- * the offset in the array of bytes to write
- * @param length
- * the number of bytes to write
+ * Expert: implements buffer flushing to cache. Writes bytes to the current position in the
+ * output.
+ *
+ * @param b the array of bytes to write
+ * @param offset the offset in the array of bytes to write
+ * @param length the number of bytes to write
*/
- protected abstract void writeInternal(byte[] b, int offset, int length)
- throws IOException;
-
+ protected abstract void writeInternal(byte[] b, int offset, int length) throws IOException;
+
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
if (getFilePointer() + length > fileLength) {
@@ -139,9 +133,9 @@ public abstract class ReusedBufferedIndexOutput extends IndexOutput {
bufferPosition = bufferSize;
bufferLength = bufferSize;
}
-
+
flushBufferToCache();
-
+
// and now, write the remaining 'length' bytes:
if (length < bufferSize) {
// If the amount left to write is small enough do it in the usual
@@ -158,7 +152,6 @@ public abstract class ReusedBufferedIndexOutput extends IndexOutput {
bufferPosition = 0;
bufferLength = 0;
}
-
}
}
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Store.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Store.java
index 74fe0fd..98e5f6a 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Store.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/Store.java
@@ -16,13 +16,10 @@
*/
package org.apache.solr.store.blockcache;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public interface Store {
byte[] takeBuffer(int bufferSize);
void putBuffer(byte[] buffer);
-
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/package-info.java
index 6da8bb3..14c7957 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/package-info.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/blockcache/package-info.java
@@ -14,10 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-/**
- * An HDFS blockcache implementation.
- */
-package org.apache.solr.store.blockcache;
-
+/** An HDFS blockcache implementation. */
+package org.apache.solr.store.blockcache;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
index 6efe2cd..2eff10a 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -47,11 +46,11 @@ import org.slf4j.LoggerFactory;
public class HdfsDirectory extends BaseDirectory {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final int DEFAULT_BUFFER_SIZE = 4096;
-
+
private static final String LF_EXT = ".lf";
protected final Path hdfsDirPath;
protected final Configuration configuration;
-
+
private final FileSystem fileSystem;
private final FileContext fileContext;
@@ -63,8 +62,9 @@ public class HdfsDirectory extends BaseDirectory {
public HdfsDirectory(Path hdfsDirPath, Configuration configuration) throws IOException {
this(hdfsDirPath, HdfsLockFactory.INSTANCE, configuration, DEFAULT_BUFFER_SIZE);
}
-
- public HdfsDirectory(Path hdfsDirPath, LockFactory lockFactory, Configuration configuration, int bufferSize)
+
+ public HdfsDirectory(
+ Path hdfsDirPath, LockFactory lockFactory, Configuration configuration, int bufferSize)
throws IOException {
super(lockFactory);
this.hdfsDirPath = hdfsDirPath;
@@ -72,7 +72,7 @@ public class HdfsDirectory extends BaseDirectory {
this.bufferSize = bufferSize;
fileSystem = FileSystem.get(hdfsDirPath.toUri(), configuration);
fileContext = FileContext.getFileContext(hdfsDirPath.toUri(), configuration);
-
+
if (fileSystem instanceof DistributedFileSystem) {
// Make sure dfs is not in safe mode
while (((DistributedFileSystem) fileSystem).setSafeMode(SafeModeAction.SAFEMODE_GET, true)) {
@@ -85,7 +85,7 @@ public class HdfsDirectory extends BaseDirectory {
}
}
}
-
+
try {
if (!fileSystem.exists(hdfsDirPath)) {
boolean success = fileSystem.mkdirs(hdfsDirPath);
@@ -98,7 +98,7 @@ public class HdfsDirectory extends BaseDirectory {
throw new RuntimeException("Problem creating directory: " + hdfsDirPath, e);
}
}
-
+
@Override
public void close() throws IOException {
log.info("Closing hdfs directory {}", hdfsDirPath);
@@ -107,26 +107,31 @@ public class HdfsDirectory extends BaseDirectory {
}
/**
- * Check whether this directory is open or closed. This check may return stale results in the form of false negatives.
- * @return true if the directory is definitely closed, false if the directory is open or is pending closure
+ * Check whether this directory is open or closed. This check may return stale results in the form
+ * of false negatives.
+ *
+ * @return true if the directory is definitely closed, false if the directory is open or is
+ * pending closure
*/
public boolean isClosed() {
return !isOpen;
}
-
+
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
try {
return new HdfsFileWriter(getFileSystem(), new Path(hdfsDirPath, name), name);
} catch (FileAlreadyExistsException e) {
- java.nio.file.FileAlreadyExistsException ex = new java.nio.file.FileAlreadyExistsException(e.getMessage());
+ java.nio.file.FileAlreadyExistsException ex =
+ new java.nio.file.FileAlreadyExistsException(e.getMessage());
ex.initCause(e);
throw ex;
}
}
@Override
- public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
+ throws IOException {
while (true) {
try {
String name = getTempFileName(prefix, suffix, nextTempFileCounter.getAndIncrement());
@@ -136,7 +141,7 @@ public class HdfsDirectory extends BaseDirectory {
}
}
}
-
+
private String[] getNormalNames(List<String> files) {
int size = files.size();
for (int i = 0; i < size; i++) {
@@ -145,28 +150,26 @@ public class HdfsDirectory extends BaseDirectory {
}
return files.toArray(new String[] {});
}
-
+
private String toNormalName(String name) {
if (name.endsWith(LF_EXT)) {
return name.substring(0, name.length() - 3);
}
return name;
}
-
+
@Override
- public IndexInput openInput(String name, IOContext context)
- throws IOException {
- return new HdfsIndexInput(name, getFileSystem(), new Path(
- hdfsDirPath, name), bufferSize);
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ return new HdfsIndexInput(name, getFileSystem(), new Path(hdfsDirPath, name), bufferSize);
}
-
+
@Override
public void deleteFile(String name) throws IOException {
Path path = new Path(hdfsDirPath, name);
log.debug("Deleting {}", path);
getFileSystem().delete(path, false);
}
-
+
@Override
public void rename(String source, String dest) throws IOException {
Path sourcePath = new Path(hdfsDirPath, source);
@@ -184,13 +187,12 @@ public class HdfsDirectory extends BaseDirectory {
FileStatus fileStatus = fileSystem.getFileStatus(new Path(hdfsDirPath, name));
return fileStatus.getLen();
}
-
+
public long fileModified(String name) throws IOException {
- FileStatus fileStatus = getFileSystem().getFileStatus(
- new Path(hdfsDirPath, name));
+ FileStatus fileStatus = getFileSystem().getFileStatus(new Path(hdfsDirPath, name));
return fileStatus.getModificationTime();
}
-
+
@Override
public String[] listAll() throws IOException {
FileStatus[] listStatus = getFileSystem().listStatus(hdfsDirPath);
@@ -207,11 +209,11 @@ public class HdfsDirectory extends BaseDirectory {
public Path getHdfsDirPath() {
return hdfsDirPath;
}
-
+
public FileSystem getFileSystem() {
return fileSystem;
}
-
+
public Configuration getConfiguration() {
return configuration;
}
@@ -223,14 +225,14 @@ public class HdfsDirectory extends BaseDirectory {
public static class HdfsIndexInput extends CustomBufferedIndexInput {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+
private final Path path;
private final FSDataInputStream inputStream;
private final long length;
private boolean clone = false;
-
- public HdfsIndexInput(String name, FileSystem fileSystem, Path path,
- int bufferSize) throws IOException {
+
+ public HdfsIndexInput(String name, FileSystem fileSystem, Path path, int bufferSize)
+ throws IOException {
super(name, bufferSize);
this.path = path;
log.debug("Opening normal index input on {}", path);
@@ -238,18 +240,15 @@ public class HdfsDirectory extends BaseDirectory {
length = fileStatus.getLen();
inputStream = fileSystem.open(path, bufferSize);
}
-
+
@Override
- protected void readInternal(byte[] b, int offset, int length)
- throws IOException {
+ protected void readInternal(byte[] b, int offset, int length) throws IOException {
inputStream.readFully(getFilePointer(), b, offset, length);
}
-
+
@Override
- protected void seekInternal(long pos) throws IOException {
+ protected void seekInternal(long pos) throws IOException {}
- }
-
@Override
protected void closeInternal() throws IOException {
log.debug("Closing normal index input on {}", path);
@@ -257,12 +256,12 @@ public class HdfsDirectory extends BaseDirectory {
inputStream.close();
}
}
-
+
@Override
public long length() {
return length;
}
-
+
@Override
public IndexInput clone() {
HdfsIndexInput clone = (HdfsIndexInput) super.clone();
@@ -270,19 +269,19 @@ public class HdfsDirectory extends BaseDirectory {
return clone;
}
}
-
+
@Override
public void sync(Collection<String> names) throws IOException {
if (log.isDebugEnabled()) {
log.debug("Sync called on {}", Arrays.toString(names.toArray()));
}
}
-
+
@Override
public int hashCode() {
return hdfsDirPath.hashCode();
}
-
+
@Override
public boolean equals(Object obj) {
if (obj == this) {
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
index 4099d95..7a2500a 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java
@@ -19,7 +19,6 @@ package org.apache.solr.store.hdfs;
import java.io.IOException;
import java.io.OutputStream;
import java.util.EnumSet;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileSystem;
@@ -28,19 +27,22 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.lucene.store.OutputStreamIndexOutput;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class HdfsFileWriter extends OutputStreamIndexOutput {
-
+
public static final String HDFS_SYNC_BLOCK = "solr.hdfs.sync.block";
public static final int BUFFER_SIZE = 16384;
-
+
public HdfsFileWriter(FileSystem fileSystem, Path path, String name) throws IOException {
- super("fileSystem=" + fileSystem + " path=" + path, name, getOutputStream(fileSystem, path), BUFFER_SIZE);
+ super(
+ "fileSystem=" + fileSystem + " path=" + path,
+ name,
+ getOutputStream(fileSystem, path),
+ BUFFER_SIZE);
}
-
- private static final OutputStream getOutputStream(FileSystem fileSystem, Path path) throws IOException {
+
+ private static final OutputStream getOutputStream(FileSystem fileSystem, Path path)
+ throws IOException {
Configuration conf = fileSystem.getConf();
FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path);
short replication = fileSystem.getDefaultReplication(path);
@@ -48,9 +50,13 @@ public class HdfsFileWriter extends OutputStreamIndexOutput {
if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) {
flags.add(CreateFlag.SYNC_BLOCK);
}
- return fileSystem.create(path, FsPermission.getDefault()
- .applyUMask(FsPermission.getUMask(conf)), flags, fsDefaults
- .getFileBufferSize(), replication, fsDefaults
- .getBlockSize(), null);
+ return fileSystem.create(
+ path,
+ FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)),
+ flags,
+ fsDefaults.getFileBufferSize(),
+ replication,
+ fsDefaults.getBlockSize(),
+ null);
}
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
index c95396d..1c0371e 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -46,7 +45,7 @@ public class HdfsLocalityReporter implements SolrInfoBean {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private String hostname;
- private final ConcurrentMap<HdfsDirectory,ConcurrentMap<FileStatus,BlockLocation[]>> cache;
+ private final ConcurrentMap<HdfsDirectory, ConcurrentMap<FileStatus, BlockLocation[]>> cache;
private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
private SolrMetricsContext solrMetricsContext;
@@ -57,12 +56,13 @@ public class HdfsLocalityReporter implements SolrInfoBean {
/**
* Set the host name to use when determining locality
+ *
* @param hostname The name of this host; should correspond to what HDFS Data Nodes think this is.
*/
public void setHost(String hostname) {
this.hostname = hostname;
}
-
+
@Override
public String getName() {
return "hdfs-locality";
@@ -83,70 +83,72 @@ public class HdfsLocalityReporter implements SolrInfoBean {
return solrMetricsContext;
}
- /**
- * Provide statistics on HDFS block locality, both in terms of bytes and block counts.
- */
+ /** Provide statistics on HDFS block locality, both in terms of bytes and block counts. */
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
- MetricsMap metricsMap = new MetricsMap(map -> {
- long totalBytes = 0;
- long localBytes = 0;
- int totalCount = 0;
- int localCount = 0;
-
- for (Iterator<HdfsDirectory> iterator = cache.keySet().iterator(); iterator.hasNext();) {
- HdfsDirectory hdfsDirectory = iterator.next();
-
- if (hdfsDirectory.isClosed()) {
- iterator.remove();
- } else {
- try {
- refreshDirectory(hdfsDirectory);
- Map<FileStatus,BlockLocation[]> blockMap = cache.get(hdfsDirectory);
-
- // For every block in every file in this directory, count it
- for (BlockLocation[] locations : blockMap.values()) {
- for (BlockLocation bl : locations) {
- totalBytes += bl.getLength();
- totalCount++;
-
- if (Arrays.asList(bl.getHosts()).contains(hostname)) {
- localBytes += bl.getLength();
- localCount++;
+ MetricsMap metricsMap =
+ new MetricsMap(
+ map -> {
+ long totalBytes = 0;
+ long localBytes = 0;
+ int totalCount = 0;
+ int localCount = 0;
+
+ for (Iterator<HdfsDirectory> iterator = cache.keySet().iterator();
+ iterator.hasNext(); ) {
+ HdfsDirectory hdfsDirectory = iterator.next();
+
+ if (hdfsDirectory.isClosed()) {
+ iterator.remove();
+ } else {
+ try {
+ refreshDirectory(hdfsDirectory);
+ Map<FileStatus, BlockLocation[]> blockMap = cache.get(hdfsDirectory);
+
+ // For every block in every file in this directory, count it
+ for (BlockLocation[] locations : blockMap.values()) {
+ for (BlockLocation bl : locations) {
+ totalBytes += bl.getLength();
+ totalCount++;
+
+ if (Arrays.asList(bl.getHosts()).contains(hostname)) {
+ localBytes += bl.getLength();
+ localCount++;
+ }
+ }
+ }
+ } catch (IOException e) {
+ log.warn(
+ "Could not retrieve locality information for {} due to exception: {}",
+ hdfsDirectory.getHdfsDirPath(),
+ e);
+ }
}
}
- }
- } catch (IOException e) {
- log.warn("Could not retrieve locality information for {} due to exception: {}",
- hdfsDirectory.getHdfsDirPath(), e);
- }
- }
- }
- map.put(LOCALITY_BYTES_TOTAL, totalBytes);
- map.put(LOCALITY_BYTES_LOCAL, localBytes);
- if (localBytes == 0) {
- map.put(LOCALITY_BYTES_RATIO, 0);
- } else {
- map.put(LOCALITY_BYTES_RATIO, localBytes / (double) totalBytes);
- }
- map.put(LOCALITY_BLOCKS_TOTAL, totalCount);
- map.put(LOCALITY_BLOCKS_LOCAL, localCount);
- if (localCount == 0) {
- map.put(LOCALITY_BLOCKS_RATIO, 0);
- } else {
- map.put(LOCALITY_BLOCKS_RATIO, localCount / (double) totalCount);
- }
- });
+ map.put(LOCALITY_BYTES_TOTAL, totalBytes);
+ map.put(LOCALITY_BYTES_LOCAL, localBytes);
+ if (localBytes == 0) {
+ map.put(LOCALITY_BYTES_RATIO, 0);
+ } else {
+ map.put(LOCALITY_BYTES_RATIO, localBytes / (double) totalBytes);
+ }
+ map.put(LOCALITY_BLOCKS_TOTAL, totalCount);
+ map.put(LOCALITY_BLOCKS_LOCAL, localCount);
+ if (localCount == 0) {
+ map.put(LOCALITY_BLOCKS_RATIO, 0);
+ } else {
+ map.put(LOCALITY_BLOCKS_RATIO, localCount / (double) totalCount);
+ }
+ });
solrMetricsContext.gauge(metricsMap, true, "hdfsLocality", getCategory().toString(), scope);
}
/**
- * Add a directory for block locality reporting. This directory will continue to be checked until its close method has
- * been called.
- *
- * @param dir
- * The directory to keep metrics on.
+ * Add a directory for block locality reporting. This directory will continue to be checked until
+ * its close method has been called.
+ *
+ * @param dir The directory to keep metrics on.
*/
public void registerDirectory(HdfsDirectory dir) {
if (log.isInfoEnabled()) {
@@ -158,16 +160,14 @@ public class HdfsLocalityReporter implements SolrInfoBean {
}
/**
- * Update the cached block locations for the given directory. This includes deleting any files that no longer exist in
- * the file system and adding any new files that have shown up.
- *
- * @param dir
- * The directory to refresh
- * @throws IOException
- * If there is a problem getting info from HDFS
+ * Update the cached block locations for the given directory. This includes deleting any files
+ * that no longer exist in the file system and adding any new files that have shown up.
+ *
+ * @param dir The directory to refresh
+ * @throws IOException If there is a problem getting info from HDFS
*/
private void refreshDirectory(HdfsDirectory dir) throws IOException {
- Map<FileStatus,BlockLocation[]> directoryCache = cache.get(dir);
+ Map<FileStatus, BlockLocation[]> directoryCache = cache.get(dir);
Set<FileStatus> cachedStatuses = directoryCache.keySet();
FileSystem fs = dir.getFileSystem();
@@ -187,5 +187,4 @@ public class HdfsLocalityReporter implements SolrInfoBean {
}
}
}
-
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
index c62b5f4..bf220e7 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java
@@ -18,7 +18,6 @@ package org.apache.solr.store.hdfs;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -36,21 +35,22 @@ import org.slf4j.LoggerFactory;
public class HdfsLockFactory extends LockFactory {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+
public static final HdfsLockFactory INSTANCE = new HdfsLockFactory();
-
+
private HdfsLockFactory() {}
-
+
@Override
public Lock obtainLock(Directory dir, String lockName) throws IOException {
if (!(dir instanceof HdfsDirectory)) {
- throw new UnsupportedOperationException("HdfsLockFactory can only be used with HdfsDirectory subclasses, got: " + dir);
+ throw new UnsupportedOperationException(
+ "HdfsLockFactory can only be used with HdfsDirectory subclasses, got: " + dir);
}
final HdfsDirectory hdfsDir = (HdfsDirectory) dir;
final Configuration conf = hdfsDir.getConfiguration();
final Path lockPath = hdfsDir.getHdfsDirPath();
final Path lockFile = new Path(lockPath, lockName);
-
+
FSDataOutputStream file = null;
final FileSystem fs = FileSystem.get(lockPath.toUri(), conf);
while (true) {
@@ -64,14 +64,13 @@ public class HdfsLockFactory extends LockFactory {
// just to check for safe mode
fs.mkdirs(lockPath);
}
-
+
file = fs.create(lockFile, false);
break;
} catch (FileAlreadyExistsException e) {
throw new LockObtainFailedException("Cannot obtain lock file: " + lockFile, e);
} catch (RemoteException e) {
- if (e.getClassName().equals(
- "org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
+ if (e.getClassName().equals("org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
try {
Thread.sleep(5000);
@@ -90,18 +89,18 @@ public class HdfsLockFactory extends LockFactory {
return new HdfsLock(conf, lockFile);
}
-
+
private static final class HdfsLock extends Lock {
private final Configuration conf;
private final Path lockFile;
private volatile boolean closed;
-
+
HdfsLock(Configuration conf, Path lockFile) {
this.conf = conf;
this.lockFile = lockFile;
}
-
+
@Override
public void close() throws IOException {
if (closed) {
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/package-info.java b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/package-info.java
index 5064bcd..d12f217 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/package-info.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/store/hdfs/package-info.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-/**
- * An HDFS Directory implementation.
- */
-package org.apache.solr.store.hdfs;
+/** An HDFS Directory implementation. */
+package org.apache.solr.store.hdfs;
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsTransactionLog.java b/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsTransactionLog.java
index 977c451..4896f36 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsTransactionLog.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsTransactionLog.java
@@ -24,7 +24,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -42,28 +41,24 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Log Format: List{Operation, Version, ...}
- * ADD, VERSION, DOC
- * DELETE, VERSION, ID_BYTES
- * DELETE_BY_QUERY, VERSION, String
- *
- * TODO: keep two files, one for [operation, version, id] and the other for the actual
- * document data. That way we could throw away document log files more readily
- * while retaining the smaller operation log files longer (and we can retrieve
- * the stored fields from the latest documents from the index).
+ * Log Format: List{Operation, Version, ...} ADD, VERSION, DOC DELETE, VERSION, ID_BYTES
+ * DELETE_BY_QUERY, VERSION, String
*
- * This would require keeping all source fields stored of course.
+ * <p>TODO: keep two files, one for [operation, version, id] and the other for the actual document
+ * data. That way we could throw away document log files more readily while retaining the smaller
+ * operation log files longer (and we can retrieve the stored fields from the latest documents from
+ * the index).
*
- * This would also allow to not log document data for requests with commit=true
- * in them (since we know that if the request succeeds, all docs will be committed)
+ * <p>This would require keeping all source fields stored of course.
*
+ * <p>This would also allow to not log document data for requests with commit=true in them (since we
+ * know that if the request succeeds, all docs will be committed)
*/
public class HdfsTransactionLog extends TransactionLog {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static boolean debug = log.isDebugEnabled();
private static boolean trace = log.isTraceEnabled();
-
Path tlogFile;
private long finalLogSize;
@@ -72,53 +67,65 @@ public class HdfsTransactionLog extends TransactionLog {
private volatile boolean isClosed = false;
- HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings, Integer tlogDfsReplication) {
+ HdfsTransactionLog(
+ FileSystem fs, Path tlogFile, Collection<String> globalStrings, Integer tlogDfsReplication) {
this(fs, tlogFile, globalStrings, false, tlogDfsReplication);
}
- HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings, boolean openExisting, Integer tlogDfsReplication) {
+ HdfsTransactionLog(
+ FileSystem fs,
+ Path tlogFile,
+ Collection<String> globalStrings,
+ boolean openExisting,
+ Integer tlogDfsReplication) {
super();
boolean success = false;
this.fs = fs;
try {
this.tlogFile = tlogFile;
-
+
if (fs.exists(tlogFile) && openExisting) {
- FSHDFSUtils.recoverFileLease(fs, tlogFile, fs.getConf(), new CallerInfo(){
+ FSHDFSUtils.recoverFileLease(
+ fs,
+ tlogFile,
+ fs.getConf(),
+ new CallerInfo() {
+
+ @Override
+ public boolean isCallerClosed() {
+ return isClosed;
+ }
+ });
- @Override
- public boolean isCallerClosed() {
- return isClosed;
- }});
-
tlogOutStream = fs.append(tlogFile);
} else {
fs.delete(tlogFile, false);
-
- tlogOutStream = fs.create(tlogFile, (short)tlogDfsReplication.intValue());
+
+ tlogOutStream = fs.create(tlogFile, (short) tlogDfsReplication.intValue());
tlogOutStream.hsync();
}
fos = new FastOutputStream(tlogOutStream, new byte[65536], 0);
- long start = tlogOutStream.getPos();
+ long start = tlogOutStream.getPos();
if (openExisting) {
if (start > 0) {
readHeader(null);
-
- // we should already be at the end
- // raf.seek(start);
- // assert channel.position() == start;
- fos.setWritten(start); // reflect that we aren't starting at the beginning
- //assert fos.size() == channel.size();
+ // we should already be at the end
+ // raf.seek(start);
+
+ // assert channel.position() == start;
+ fos.setWritten(start); // reflect that we aren't starting at the beginning
+ // assert fos.size() == channel.size();
} else {
addGlobalStrings(globalStrings);
}
} else {
if (start > 0) {
- log.error("New transaction log already exists:{} size={}", tlogFile, tlogOutStream.size());
+ log.error(
+ "New transaction log already exists:{} size={}", tlogFile, tlogOutStream.size());
}
addGlobalStrings(globalStrings);
@@ -128,7 +135,7 @@ public class HdfsTransactionLog extends TransactionLog {
assert ObjectReleaseTracker.track(this);
log.debug("Opening new tlog {}", this);
-
+
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
@@ -146,16 +153,16 @@ public class HdfsTransactionLog extends TransactionLog {
public boolean endsWithCommit() throws IOException {
ensureFlushed();
long size = getLogSize();
-
+
// the end of the file should have the end message (added during a commit) plus a 4 byte size
- byte[] buf = new byte[ END_MESSAGE.length() ];
+ byte[] buf = new byte[END_MESSAGE.length()];
long pos = size - END_MESSAGE.length() - 4;
if (pos < 0) return false;
-
+
FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile), pos);
try {
dis.read(buf);
- for (int i=0; i<buf.length; i++) {
+ for (int i = 0; i < buf.length; i++) {
if (buf[i] != END_MESSAGE.charAt(i)) return false;
}
} finally {
@@ -170,12 +177,12 @@ public class HdfsTransactionLog extends TransactionLog {
boolean closeFis = false;
if (fis == null) closeFis = true;
fis = fis != null ? fis : new FSDataFastInputStream(fs.open(tlogFile), 0);
- Map<?,?> header = null;
+ Map<?, ?> header = null;
try {
try (LogCodec codec = new LogCodec(resolver)) {
- header = (Map<?,?>) codec.unmarshal(fis);
+ header = (Map<?, ?>) codec.unmarshal(fis);
}
-
+
fis.readInt(); // skip size
} finally {
if (fis != null && closeFis) {
@@ -185,10 +192,10 @@ public class HdfsTransactionLog extends TransactionLog {
// needed to read other records
synchronized (this) {
- globalStringList = (List<String>)header.get("strings");
+ globalStringList = (List<String>) header.get("strings");
globalStringMap = new HashMap<>(globalStringList.size());
- for (int i=0; i<globalStringList.size(); i++) {
- globalStringMap.put( globalStringList.get(i), i+1);
+ for (int i = 0; i < globalStringList.size(); i++) {
+ globalStringMap.put(globalStringList.get(i), i + 1);
}
}
}
@@ -198,27 +205,27 @@ public class HdfsTransactionLog extends TransactionLog {
LogCodec codec = new LogCodec(resolver);
synchronized (this) {
try {
- long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
+ long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
if (pos == 0) {
writeLogHeader(codec);
pos = fos.size();
}
-
+
codec.init(fos);
codec.writeTag(JavaBinCodec.ARR, 3);
- codec.writeInt(UpdateLog.COMMIT); // should just take one byte
+ codec.writeInt(UpdateLog.COMMIT); // should just take one byte
codec.writeLong(cmd.getVersion());
- codec.writeStr(END_MESSAGE); // ensure these bytes are (almost) last in the file
+ codec.writeStr(END_MESSAGE); // ensure these bytes are (almost) last in the file
endRecord(pos);
-
- ensureFlushed(); // flush since this will be the last record in a log fill
+
+ ensureFlushed(); // flush since this will be the last record in a log fill
// now the commit command is written we will never write to this log again
closeOutput();
- //assert fos.size() == channel.size();
+ // assert fos.size() == channel.size();
return pos;
} catch (IOException e) {
@@ -227,7 +234,6 @@ public class HdfsTransactionLog extends TransactionLog {
}
}
-
/* This method is thread safe */
@Override
public Object lookup(long pos) {
@@ -239,8 +245,7 @@ public class HdfsTransactionLog extends TransactionLog {
// make sure any unflushed buffer has been flushed
ensureFlushed();
- FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile),
- pos);
+ FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile), pos);
try {
dis.seek(pos);
try (LogCodec codec = new LogCodec(resolver)) {
@@ -318,7 +323,7 @@ public class HdfsTransactionLog extends TransactionLog {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
-
+
@Override
public void close() {
try {
@@ -332,7 +337,7 @@ public class HdfsTransactionLog extends TransactionLog {
log.error("Exception closing tlog.", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
- isClosed = true;
+ isClosed = true;
assert ObjectReleaseTracker.release(this);
if (deleteOnClose) {
try {
@@ -348,9 +353,10 @@ public class HdfsTransactionLog extends TransactionLog {
return "hdfs tlog{file=" + tlogFile.toString() + " refcount=" + refcount.get() + "}";
}
- /** Returns a reader that can be used while a log is still in use.
- * Currently only *one* LogReader may be outstanding, and that log may only
- * be used from a single thread. */
+ /**
+ * Returns a reader that can be used while a log is still in use. Currently only *one* LogReader
+ * may be outstanding, and that log may only be used from a single thread.
+ */
@Override
public LogReader getReader(long startingPos) {
return new HDFSLogReader(startingPos);
@@ -366,8 +372,7 @@ public class HdfsTransactionLog extends TransactionLog {
return new HDFSReverseReader();
}
-
- public class HDFSLogReader extends LogReader{
+ public class HDFSLogReader extends LogReader {
FSDataFastInputStream fis;
private LogCodec codec = new LogCodec(resolver);
private long sz;
@@ -380,7 +385,7 @@ public class HdfsTransactionLog extends TransactionLog {
private void initStream(long pos) {
try {
-
+
synchronized (HdfsTransactionLog.this) {
ensureFlushed();
sz = getLogSize();
@@ -393,7 +398,8 @@ public class HdfsTransactionLog extends TransactionLog {
}
}
- /** Returns the next object from the log, or null if none available.
+ /**
+ * Returns the next object from the log, or null if none available.
*
* @return The log record, or null if EOF
* @throws IOException If there is a low-level I/O error.
@@ -410,20 +416,21 @@ public class HdfsTransactionLog extends TransactionLog {
return null;
}
}
-
- // we actually need a new reader to
+
+ // we actually need a new reader to
// see if any data was added by the writer
if (pos >= sz) {
log.info("Read available inputstream data, opening new inputstream pos={} sz={}", pos, sz);
-
+
fis.close();
initStream(pos);
}
-
+
if (pos == 0) {
readHeader(fis);
- // shouldn't currently happen - header and first record are currently written at the same time
+ // shouldn't currently happen - header and first record are currently written at the same
+ // time
synchronized (HdfsTransactionLog.this) {
if (fis.position() >= getLogSize()) {
return null;
@@ -453,23 +460,29 @@ public class HdfsTransactionLog extends TransactionLog {
@Override
public String toString() {
synchronized (HdfsTransactionLog.this) {
- return "LogReader{" + "file=" + tlogFile + ", position=" + fis.position() + ", end=" + getLogSize() + "}";
+ return "LogReader{"
+ + "file="
+ + tlogFile
+ + ", position="
+ + fis.position()
+ + ", end="
+ + getLogSize()
+ + "}";
}
}
-
+
@Override
public long currentPos() {
return fis.position();
}
-
+
@Override
public long currentSize() {
return getLogSize();
}
-
}
- public class HDFSSortedLogReader extends HDFSLogReader{
+ public class HDFSSortedLogReader extends HDFSLogReader {
private long startingPos;
private boolean inOrder = true;
private TreeMap<Long, Long> versionToPos;
@@ -488,7 +501,7 @@ public class HdfsTransactionLog extends TransactionLog {
long pos = startingPos;
long lastVersion = Long.MIN_VALUE;
- while ( (o = super.next()) != null) {
+ while ((o = super.next()) != null) {
List<?> entry = (List<?>) o;
long version = (Long) entry.get(UpdateLog.VERSION_IDX);
version = Math.abs(version);
@@ -515,17 +528,19 @@ public class HdfsTransactionLog extends TransactionLog {
public class HDFSReverseReader extends ReverseReader {
FSDataFastInputStream fis;
- private LogCodec codec = new LogCodec(resolver) {
- @Override
- public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
- // Given that the SolrInputDocument is last in an add record, it's OK to just skip
- // reading it completely.
- return null;
- }
- };
+ private LogCodec codec =
+ new LogCodec(resolver) {
+ @Override
+ public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
+ // Given that the SolrInputDocument is last in an add record, it's OK to just skip
+ // reading it completely.
+ return null;
+ }
+ };
- int nextLength; // length of the next record (the next one closer to the start of the log file)
- long prevPos; // where we started reading from last time (so prevPos - nextLength == start of next record)
+ int nextLength; // length of the next record (the next one closer to the start of the log file)
+ long prevPos; // where we started reading from last time (so prevPos - nextLength == start of
+ // next record)
public HDFSReverseReader() throws IOException {
incref();
@@ -537,8 +552,8 @@ public class HdfsTransactionLog extends TransactionLog {
}
fis = new FSDataFastInputStream(fs.open(tlogFile), 0);
-
- if (sz >=4) {
+
+ if (sz >= 4) {
// readHeader(fis); // should not be needed
prevPos = sz - 4;
fis.seek(prevPos);
@@ -546,8 +561,8 @@ public class HdfsTransactionLog extends TransactionLog {
}
}
-
- /** Returns the next object from the log, or null if none available.
+ /**
+ * Returns the next object from the log, or null if none available.
*
* @return The log record, or null if EOF
* @throws IOException If there is a low-level I/O error.
@@ -559,10 +574,10 @@ public class HdfsTransactionLog extends TransactionLog {
int thisLength = nextLength;
- long recordStart = prevPos - thisLength; // back up to the beginning of the next record
- prevPos = recordStart - 4; // back up 4 more to read the length of the next record
+ long recordStart = prevPos - thisLength; // back up to the beginning of the next record
+ prevPos = recordStart - 4; // back up 4 more to read the length of the next record
- if (prevPos <= 0) return null; // this record is the header
+ if (prevPos <= 0) return null; // this record is the header
long bufferPos = fis.getBufferPos();
if (prevPos >= bufferPos) {
@@ -570,26 +585,31 @@ public class HdfsTransactionLog extends TransactionLog {
} else {
// Position buffer so that this record is at the end.
// For small records, this will cause subsequent calls to next() to be within the buffer.
- long seekPos = endOfThisRecord - fis.getBufferSize();
- seekPos = Math.min(seekPos, prevPos); // seek to the start of the record if it's larger then the block size.
+ long seekPos = endOfThisRecord - fis.getBufferSize();
+ seekPos =
+ Math.min(
+ seekPos,
+ prevPos); // seek to the start of the record if it's larger then the block size.
seekPos = Math.max(seekPos, 0);
fis.seek(seekPos);
- fis.peek(); // cause buffer to be filled
+ fis.peek(); // cause buffer to be filled
}
fis.seek(prevPos);
- nextLength = fis.readInt(); // this is the length of the *next* record (i.e. closer to the beginning)
+ nextLength =
+ fis.readInt(); // this is the length of the *next* record (i.e. closer to the beginning)
// TODO: optionally skip document data
Object o = codec.readVal(fis);
- // assert fis.position() == prevPos + 4 + thisLength; // this is only true if we read all the data (and we currently skip reading SolrInputDocument
+ // assert fis.position() == prevPos + 4 + thisLength; // this is only true if we read all the
+ // data (and we currently skip reading SolrInputDocument
return o;
}
/* returns the position in the log file of the last record returned by next() */
public long position() {
- return prevPos + 4; // skip the length
+ return prevPos + 4; // skip the length
}
public void close() {
@@ -604,17 +624,19 @@ public class HdfsTransactionLog extends TransactionLog {
@Override
public String toString() {
synchronized (HdfsTransactionLog.this) {
- return "LogReader{" + "file=" + tlogFile + ", position=" + fis.position() + ", end=" + getLogSize() + "}";
+ return "LogReader{"
+ + "file="
+ + tlogFile
+ + ", position="
+ + fis.position()
+ + ", end="
+ + getLogSize()
+ + "}";
}
}
-
-
}
-
}
-
-
class FSDataFastInputStream extends FastInputStream {
private FSDataInputStream fis;
@@ -633,10 +655,12 @@ class FSDataFastInputStream extends FastInputStream {
public void seek(long position) throws IOException {
if (position <= readFromStream && position >= getBufferPos()) {
// seek within buffer
- pos = (int)(position - getBufferPos());
+ pos = (int) (position - getBufferPos());
} else {
- // long currSize = ch.size(); // not needed - underlying read should handle (unless read never done)
- // if (position > currSize) throw new EOFException("Read past EOF: seeking to " + position + " on file of size " + currSize + " file=" + ch);
+ // long currSize = ch.size(); // not needed - underlying read should handle (unless read
+ // never done)
+ // if (position > currSize) throw new EOFException("Read past EOF: seeking to " + position + "
+ // on file of size " + currSize + " file=" + ch);
readFromStream = position;
end = pos = 0;
}
@@ -656,9 +680,18 @@ class FSDataFastInputStream extends FastInputStream {
public void close() throws IOException {
fis.close();
}
-
+
@Override
public String toString() {
- return "readFromStream="+readFromStream +" pos="+pos +" end="+end + " bufferPos="+getBufferPos() + " position="+position() ;
+ return "readFromStream="
+ + readFromStream
+ + " pos="
+ + pos
+ + " end="
+ + end
+ + " bufferPos="
+ + getBufferPos()
+ + " position="
+ + position();
}
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsUpdateLog.java b/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsUpdateLog.java
index 0a83b42..0ca2bb1 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsUpdateLog.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/update/HdfsUpdateLog.java
@@ -24,7 +24,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -42,11 +41,9 @@ import org.apache.solr.util.HdfsUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * @lucene.experimental
- */
+/** @lucene.experimental */
public class HdfsUpdateLog extends UpdateLog {
-
+
private final Object fsLock = new Object();
private FileSystem fs;
private volatile Path tlogDir;
@@ -55,23 +52,23 @@ public class HdfsUpdateLog extends UpdateLog {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private boolean debug = log.isDebugEnabled();
-
+
// used internally by tests to track total count of failed tran log loads in init
public static AtomicLong INIT_FAILED_LOGS_COUNT = new AtomicLong();
public HdfsUpdateLog() {
this(null);
}
-
+
public HdfsUpdateLog(String confDir) {
this.confDir = confDir;
}
-
+
@Override
public void init(PluginInfo info) {
super.init(info);
-
- tlogDfsReplication = (Integer) info.initArgs.get( "tlogDfsReplication");
+
+ tlogDfsReplication = (Integer) info.initArgs.get("tlogDfsReplication");
if (tlogDfsReplication == null) tlogDfsReplication = 3;
log.info("Initializing HdfsUpdateLog: tlogDfsReplication={}", tlogDfsReplication);
@@ -84,15 +81,15 @@ public class HdfsUpdateLog extends UpdateLog {
}
String fsScheme = path.toUri().getScheme();
- if(fsScheme != null) {
+ if (fsScheme != null) {
conf.setBoolean("fs." + fsScheme + ".impl.disable.cache", true);
}
return conf;
}
-
+
@Override
public void init(UpdateHandler uhandler, SolrCore core) {
-
+
// ulogDir from CoreDescriptor overrides
String ulogDir = core.getCoreDescriptor().getUlogDir();
@@ -108,7 +105,7 @@ public class HdfsUpdateLog extends UpdateLog {
if (dataDir == null || dataDir.length() == 0) {
dataDir = core.getDataDir();
}
-
+
if (!core.getDirectoryFactory().isAbsolute(dataDir)) {
try {
dataDir = core.getDirectoryFactory().getDataHome(core.getCoreDescriptor());
@@ -116,7 +113,7 @@ public class HdfsUpdateLog extends UpdateLog {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
}
-
+
try {
Path dataDirPath = new Path(dataDir);
fs = FileSystem.get(dataDirPath.toUri(), getConf(dataDirPath));
@@ -125,14 +122,16 @@ public class HdfsUpdateLog extends UpdateLog {
}
} else {
if (debug) {
- log.debug("UpdateHandler init: tlogDir={}, next id={} this is a reopen or double init ... nothing else to do."
- , tlogDir, id);
+ log.debug(
+ "UpdateHandler init: tlogDir={}, next id={} this is a reopen or double init ... nothing else to do.",
+ tlogDir,
+ id);
}
versionInfo.reload();
return;
}
}
-
+
tlogDir = new Path(dataDir, TLOG_NAME);
while (true) {
try {
@@ -146,8 +145,7 @@ public class HdfsUpdateLog extends UpdateLog {
}
break;
} catch (RemoteException e) {
- if (e.getClassName().equals(
- "org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
+ if (e.getClassName().equals("org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
try {
Thread.sleep(5000);
@@ -156,8 +154,7 @@ public class HdfsUpdateLog extends UpdateLog {
}
continue;
}
- throw new RuntimeException(
- "Problem creating directory: " + tlogDir, e);
+ throw new RuntimeException("Problem creating directory: " + tlogDir, e);
} catch (IOException e) {
throw new RuntimeException("Problem creating directory: " + tlogDir, e);
}
@@ -167,27 +164,29 @@ public class HdfsUpdateLog extends UpdateLog {
if (oldBufferTlog != null && oldBufferTlog.length != 0) {
existOldBufferLog = true;
}
-
+
tlogFiles = getLogList(fs, tlogDir);
id = getLastLogId() + 1; // add 1 since we will create a new log for the
- // next update
-
+ // next update
+
if (debug) {
- log.debug("UpdateHandler init: tlogDir={}, existing tlogs={}, next id={}"
- , tlogDir, Arrays.asList(tlogFiles), id);
+ log.debug(
+ "UpdateHandler init: tlogDir={}, existing tlogs={}, next id={}",
+ tlogDir,
+ Arrays.asList(tlogFiles),
+ id);
}
-
+
TransactionLog oldLog = null;
for (String oldLogName : tlogFiles) {
Path f = new Path(tlogDir, oldLogName);
try {
oldLog = new HdfsTransactionLog(fs, f, null, true, tlogDfsReplication);
addOldLog(oldLog, false); // don't remove old logs on startup since more
- // than one may be uncapped.
+ // than one may be uncapped.
} catch (Exception e) {
INIT_FAILED_LOGS_COUNT.incrementAndGet();
- SolrException.log(log, "Failure to open existing log file (non fatal) "
- + f, e);
+ SolrException.log(log, "Failure to open existing log file (non fatal) " + f, e);
try {
fs.delete(f, false);
} catch (IOException e1) {
@@ -195,7 +194,7 @@ public class HdfsUpdateLog extends UpdateLog {
}
}
}
-
+
// Record first two logs (oldest first) at startup for potential tlog
// recovery.
// It's possible that at abnormal close both "tlog" and "prevTlog" were
@@ -209,15 +208,15 @@ public class HdfsUpdateLog extends UpdateLog {
ll.closeOutput();
}
}
-
+
try {
versionInfo = new VersionInfo(this, numVersionBuckets);
} catch (SolrException e) {
log.error("Unable to use updateLog: ", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Unable to use updateLog: " + e.getMessage(), e);
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Unable to use updateLog: " + e.getMessage(), e);
}
-
+
// TODO: these startingVersions assume that we successfully recover from all
// non-complete tlogs.
try (RecentUpdates startingUpdates = getRecentUpdates()) {
@@ -239,20 +238,19 @@ public class HdfsUpdateLog extends UpdateLog {
String q = (String) dbq.get(2);
trackDeleteByQuery(q, version);
}
-
}
// initialize metrics
core.getCoreMetricManager().registerMetricProducer(SolrInfoBean.Category.TLOG.toString(), this);
}
-
+
@Override
public String getLogDir() {
return tlogDir.toUri().toString();
}
public static String[] getBufferLogList(FileSystem fs, Path tlogDir) {
- final String prefix = BUFFER_TLOG_NAME+'.';
+ final String prefix = BUFFER_TLOG_NAME + '.';
assert fs != null;
FileStatus[] fileStatuses;
try {
@@ -267,19 +265,22 @@ public class HdfsUpdateLog extends UpdateLog {
}
return names;
}
-
+
public static String[] getLogList(FileSystem fs, Path tlogDir) {
final String prefix = TLOG_NAME + '.';
assert fs != null;
FileStatus[] fileStatuses;
try {
- fileStatuses = fs.listStatus(tlogDir, new PathFilter() {
-
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith(prefix);
- }
- });
+ fileStatuses =
+ fs.listStatus(
+ tlogDir,
+ new PathFilter() {
+
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith(prefix);
+ }
+ });
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -291,12 +292,12 @@ public class HdfsUpdateLog extends UpdateLog {
return names;
}
-
+
@Override
public void close(boolean committed) {
close(committed, false);
}
-
+
@Override
public void close(boolean committed, boolean deleteOnClose) {
try {
@@ -309,9 +310,11 @@ public class HdfsUpdateLog extends UpdateLog {
@Override
protected void ensureBufferTlog() {
if (bufferTlog != null) return;
- String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN, BUFFER_TLOG_NAME, System.nanoTime());
- bufferTlog = new HdfsTransactionLog(fs, new Path(tlogDir, newLogName),
- globalStrings, tlogDfsReplication);
+ String newLogName =
+ String.format(Locale.ROOT, LOG_FILENAME_PATTERN, BUFFER_TLOG_NAME, System.nanoTime());
+ bufferTlog =
+ new HdfsTransactionLog(
+ fs, new Path(tlogDir, newLogName), globalStrings, tlogDfsReplication);
bufferTlog.isBuffer = true;
}
@@ -338,17 +341,17 @@ public class HdfsUpdateLog extends UpdateLog {
@Override
protected void ensureLog() {
if (tlog == null) {
- String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN,
- TLOG_NAME, id);
- HdfsTransactionLog ntlog = new HdfsTransactionLog(fs, new Path(tlogDir, newLogName),
- globalStrings, tlogDfsReplication);
+ String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN, TLOG_NAME, id);
+ HdfsTransactionLog ntlog =
+ new HdfsTransactionLog(
+ fs, new Path(tlogDir, newLogName), globalStrings, tlogDfsReplication);
tlog = ntlog;
}
}
-
+
/**
* Clears the logs on the file system. Only call before init.
- *
+ *
* @param core the SolrCore
* @param ulogPluginInfo the init info for the UpdateHandler
*/
@@ -371,22 +374,25 @@ public class HdfsUpdateLog extends UpdateLog {
throw new RuntimeException(e);
}
}
-
+
@Override
public void preSoftCommit(CommitUpdateCommand cmd) {
debug = log.isDebugEnabled();
super.preSoftCommit(cmd);
}
-
+
public String[] getLogList(Path tlogDir) throws FileNotFoundException, IOException {
- final String prefix = TLOG_NAME+'.';
- FileStatus[] files = fs.listStatus(tlogDir, new PathFilter() {
-
- @Override
- public boolean accept(Path name) {
- return name.getName().startsWith(prefix);
- }
- });
+ final String prefix = TLOG_NAME + '.';
+ FileStatus[] files =
+ fs.listStatus(
+ tlogDir,
+ new PathFilter() {
+
+ @Override
+ public boolean accept(Path name) {
+ return name.getName().startsWith(prefix);
+ }
+ });
List<String> fileList = new ArrayList<>(files.length);
for (FileStatus file : files) {
fileList.add(file.getPath().getName());
@@ -394,10 +400,7 @@ public class HdfsUpdateLog extends UpdateLog {
return fileList.toArray(new String[0]);
}
- /**
- * Returns true if we were able to drop buffered updates and return to the
- * ACTIVE state
- */
+ /** Returns true if we were able to drop buffered updates and return to the ACTIVE state */
// public boolean dropBufferedUpdates() {
// versionInfo.blockUpdates();
// try {
@@ -426,9 +429,8 @@ public class HdfsUpdateLog extends UpdateLog {
// }
// return true;
// }
-
+
public String toString() {
return "HDFSUpdateLog{state=" + getState() + ", tlog=" + tlog + "}";
}
-
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/util/FSHDFSUtils.java b/solr/modules/hdfs/src/java/org/apache/solr/util/FSHDFSUtils.java
index d5a9b6d..57856f6 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/util/FSHDFSUtils.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/util/FSHDFSUtils.java
@@ -22,7 +22,6 @@ import java.io.InterruptedIOException;
import java.lang.invoke.MethodHandles;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -30,12 +29,10 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * Borrowed from Apache HBase to recover an HDFS lease.
- */
+/** Borrowed from Apache HBase to recover an HDFS lease. */
public class FSHDFSUtils {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+
// internal, for tests
public static final AtomicLong RECOVER_LEASE_SUCCESS_COUNT = new AtomicLong();
@@ -43,13 +40,13 @@ public class FSHDFSUtils {
boolean isCallerClosed();
}
- /**
- * Recover the lease from HDFS, retrying multiple times.
- */
- public static void recoverFileLease(final FileSystem fs, final Path p, Configuration conf, CallerInfo callerInfo) throws IOException {
+ /** Recover the lease from HDFS, retrying multiple times. */
+ public static void recoverFileLease(
+ final FileSystem fs, final Path p, Configuration conf, CallerInfo callerInfo)
+ throws IOException {
// lease recovery not needed for local file system case.
if (!(fs instanceof DistributedFileSystem)) return;
- recoverDFSFileLease((DistributedFileSystem)fs, p, conf, callerInfo);
+ recoverDFSFileLease((DistributedFileSystem) fs, p, conf, callerInfo);
}
/*
@@ -78,31 +75,41 @@ public class FSHDFSUtils {
*
* If HDFS-4525 is available, call it every second and we might be able to exit early.
*/
- static boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p, final Configuration conf, CallerInfo callerInfo)
- throws IOException {
+ static boolean recoverDFSFileLease(
+ final DistributedFileSystem dfs,
+ final Path p,
+ final Configuration conf,
+ CallerInfo callerInfo)
+ throws IOException {
log.info("Recovering lease on dfs file {}", p);
long startWaiting = System.nanoTime();
// Default is 15 minutes. It's huge, but the idea is that if we have a major issue, HDFS
// usually needs 10 minutes before marking the nodes as dead. So we're putting ourselves
// beyond that limit 'to be safe'.
- long recoveryTimeout = TimeUnit.NANOSECONDS.convert(conf.getInt("solr.hdfs.lease.recovery.timeout", 900000), TimeUnit.MILLISECONDS) + startWaiting;
+ long recoveryTimeout =
+ TimeUnit.NANOSECONDS.convert(
+ conf.getInt("solr.hdfs.lease.recovery.timeout", 900000), TimeUnit.MILLISECONDS)
+ + startWaiting;
// This setting should be a little bit above what the cluster dfs heartbeat is set to.
long firstPause = conf.getInt("solr.hdfs.lease.recovery.first.pause", 4000);
// This should be set to how long it'll take for us to timeout against primary datanode if it
// is dead. We set it to 61 seconds, 1 second than the default READ_TIMEOUT in HDFS, the
// default value for DFS_CLIENT_SOCKET_TIMEOUT_KEY.
- long subsequentPause = TimeUnit.NANOSECONDS.convert(conf.getInt("solr.hdfs.lease.recovery.dfs.timeout", 61 * 1000), TimeUnit.MILLISECONDS);
-
+ long subsequentPause =
+ TimeUnit.NANOSECONDS.convert(
+ conf.getInt("solr.hdfs.lease.recovery.dfs.timeout", 61 * 1000), TimeUnit.MILLISECONDS);
+
if (dfs.isFileClosed(p)) {
return true;
}
-
+
boolean recovered = false;
// We break the loop if we succeed the lease recovery, timeout, or we throw an exception.
for (int nbAttempt = 0; !recovered; nbAttempt++) {
recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
if (recovered) break;
- if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting) || callerInfo.isCallerClosed()) break;
+ if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)
+ || callerInfo.isCallerClosed()) break;
try {
// On the first time through wait the short 'firstPause'.
if (nbAttempt == 0) {
@@ -110,7 +117,8 @@ public class FSHDFSUtils {
} else {
// Cycle here until subsequentPause elapses. While spinning, check isFileClosed
long localStartWaiting = System.nanoTime();
- while ((System.nanoTime() - localStartWaiting) < subsequentPause && !callerInfo.isCallerClosed()) {
+ while ((System.nanoTime() - localStartWaiting) < subsequentPause
+ && !callerInfo.isCallerClosed()) {
Thread.sleep(conf.getInt("solr.hdfs.lease.recovery.pause", 1000));
if (dfs.isFileClosed(p)) {
@@ -131,12 +139,17 @@ public class FSHDFSUtils {
return recovered;
}
- static boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
- final int nbAttempt, final Path p, final long startWaiting) {
+ static boolean checkIfTimedout(
+ final Configuration conf,
+ final long recoveryTimeout,
+ final int nbAttempt,
+ final Path p,
+ final long startWaiting) {
if (recoveryTimeout < System.nanoTime()) {
- log.warn("Cannot recoverLease after trying for {}ms (solr.hdfs.lease.recovery.timeout); continuing, but may be DATALOSS!!!; {}"
- , conf.getInt("solr.hdfs.lease.recovery.timeout", 900000)
- , getLogMessageDetail(nbAttempt, p, startWaiting));
+ log.warn(
+ "Cannot recoverLease after trying for {}ms (solr.hdfs.lease.recovery.timeout); continuing, but may be DATALOSS!!!; {}",
+ conf.getInt("solr.hdfs.lease.recovery.timeout", 900000),
+ getLogMessageDetail(nbAttempt, p, startWaiting));
return true;
}
return false;
@@ -144,10 +157,12 @@ public class FSHDFSUtils {
/**
* Try to recover the lease.
+ *
* @return True if dfs#recoverLease came by true.
*/
- static boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p, final long startWaiting)
- throws FileNotFoundException {
+ static boolean recoverLease(
+ final DistributedFileSystem dfs, final int nbAttempt, final Path p, final long startWaiting)
+ throws FileNotFoundException {
boolean recovered = false;
try {
recovered = dfs.recoverLease(p);
@@ -159,18 +174,22 @@ public class FSHDFSUtils {
// This exception comes out instead of FNFE, fix it
throw new FileNotFoundException("The given transactionlog file wasn't found at " + p);
} else if (e instanceof FileNotFoundException) {
- throw (FileNotFoundException)e;
+ throw (FileNotFoundException) e;
}
log.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
}
return recovered;
}
- /**
- * @return Detail to append to any log message around lease recovering.
- */
- private static String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
- return "attempt=" + nbAttempt + " on file=" + p + " after " +
- TimeUnit.MILLISECONDS.convert(System.nanoTime() - startWaiting, TimeUnit.NANOSECONDS) + "ms";
+ /** @return Detail to append to any log message around lease recovering. */
+ private static String getLogMessageDetail(
+ final int nbAttempt, final Path p, final long startWaiting) {
+ return "attempt="
+ + nbAttempt
+ + " on file="
+ + p
+ + " after "
+ + TimeUnit.MILLISECONDS.convert(System.nanoTime() - startWaiting, TimeUnit.NANOSECONDS)
+ + "ms";
}
}
diff --git a/solr/modules/hdfs/src/java/org/apache/solr/util/HdfsUtil.java b/solr/modules/hdfs/src/java/org/apache/solr/util/HdfsUtil.java
index 799ddca..c4dc593 100644
--- a/solr/modules/hdfs/src/java/org/apache/solr/util/HdfsUtil.java
+++ b/solr/modules/hdfs/src/java/org/apache/solr/util/HdfsUtil.java
@@ -17,32 +17,39 @@
package org.apache.solr.util;
import java.io.File;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
public class HdfsUtil {
-
+
// Allows tests to easily add additional conf
public static Configuration TEST_CONF = null;
-
- private static final String[] HADOOP_CONF_FILES = {"core-site.xml",
- "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml", "hadoop-site.xml"};
-
+
+ private static final String[] HADOOP_CONF_FILES = {
+ "core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml", "hadoop-site.xml"
+ };
+
public static void addHdfsResources(Configuration conf, String confDir) {
if (confDir != null && confDir.length() > 0) {
File confDirFile = new File(confDir);
if (!confDirFile.exists()) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Resource directory does not exist: " + confDirFile.getAbsolutePath());
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Resource directory does not exist: " + confDirFile.getAbsolutePath());
}
if (!confDirFile.isDirectory()) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Specified resource directory is not a directory" + confDirFile.getAbsolutePath());
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Specified resource directory is not a directory" + confDirFile.getAbsolutePath());
}
if (!confDirFile.canRead()) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Resource directory must be readable by the Solr process: " + confDirFile.getAbsolutePath());
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Resource directory must be readable by the Solr process: "
+ + confDirFile.getAbsolutePath());
}
for (String file : HADOOP_CONF_FILES) {
if (new File(confDirFile, file).exists()) {
@@ -50,7 +57,7 @@ public class HdfsUtil {
}
}
}
-
+
if (TEST_CONF != null) {
conf.addResource(TEST_CONF);
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
index ee0ecc8..6577d6d 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
@@ -17,10 +17,9 @@
package org.apache.solr.cloud;
-import java.io.IOException;
-
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
@@ -40,11 +39,13 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
@Nightly // test is too long for non nightly
public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
private static MiniDFSCluster dfsCluster;
@@ -52,7 +53,8 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
@BeforeClass
public static void setupClass() throws Exception {
configureCluster(2)
- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
+ .addConfig(
+ "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
.configure();
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
@@ -82,12 +84,12 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
String hdfsUri = HdfsTestUtil.getURI(dfsCluster);
String dataDir = hdfsUri + "/dummyFolder/dataDir";
String ulogDir = hdfsUri + "/dummyFolder2/ulogDir";
- CollectionAdminResponse res = CollectionAdminRequest
- .addReplicaToShard(coll, "shard1")
- .setDataDir(dataDir)
- .setUlogDir(ulogDir)
- .setNode(cluster.getJettySolrRunner(0).getNodeName())
- .process(cluster.getSolrClient());
+ CollectionAdminResponse res =
+ CollectionAdminRequest.addReplicaToShard(coll, "shard1")
+ .setDataDir(dataDir)
+ .setUlogDir(ulogDir)
+ .setNode(cluster.getJettySolrRunner(0).getNodeName())
+ .process(cluster.getSolrClient());
ulogDir += "/tlog";
ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
@@ -95,28 +97,42 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
DocCollection docCollection = zkStateReader.getClusterState().getCollection(coll);
Replica replica = docCollection.getReplicas().iterator().next();
- assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/'));
- assertTrue(replica.getStr("dataDir"),replica.getStr("dataDir").equals(dataDir) || replica.getStr("dataDir").equals(dataDir+'/'));
+ assertTrue(
+ replica.getStr("ulogDir"),
+ replica.getStr("ulogDir").equals(ulogDir)
+ || replica.getStr("ulogDir").equals(ulogDir + '/'));
+ assertTrue(
+ replica.getStr("dataDir"),
+ replica.getStr("dataDir").equals(dataDir)
+ || replica.getStr("dataDir").equals(dataDir + '/'));
- new CollectionAdminRequest.MoveReplica(coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
+ new CollectionAdminRequest.MoveReplica(
+ coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
.process(cluster.getSolrClient());
assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
docCollection = zkStateReader.getClusterState().getCollection(coll);
assertEquals(1, docCollection.getSlice("shard1").getReplicas().size());
Replica newReplica = docCollection.getReplicas().iterator().next();
assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName());
- assertTrue(newReplica.getStr("ulogDir"), newReplica.getStr("ulogDir").equals(ulogDir) || newReplica.getStr("ulogDir").equals(ulogDir+'/'));
- assertTrue(newReplica.getStr("dataDir"),newReplica.getStr("dataDir").equals(dataDir) || newReplica.getStr("dataDir").equals(dataDir+'/'));
+ assertTrue(
+ newReplica.getStr("ulogDir"),
+ newReplica.getStr("ulogDir").equals(ulogDir)
+ || newReplica.getStr("ulogDir").equals(ulogDir + '/'));
+ assertTrue(
+ newReplica.getStr("dataDir"),
+ newReplica.getStr("dataDir").equals(dataDir)
+ || newReplica.getStr("dataDir").equals(dataDir + '/'));
assertEquals(replica.getName(), newReplica.getName());
assertEquals(replica.getCoreName(), newReplica.getCoreName());
assertFalse(replica.getNodeName().equals(newReplica.getNodeName()));
final int numDocs = 100;
- addDocs(coll, numDocs); // indexed but not committed
+ addDocs(coll, numDocs); // indexed but not committed
cluster.getJettySolrRunner(1).stop();
Thread.sleep(5000);
- new CollectionAdminRequest.MoveReplica(coll, newReplica.getName(), cluster.getJettySolrRunner(0).getNodeName())
+ new CollectionAdminRequest.MoveReplica(
+ coll, newReplica.getName(), cluster.getJettySolrRunner(0).getNodeName())
.process(cluster.getSolrClient());
assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
@@ -127,13 +143,21 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
assertEquals(1, docCollection.getReplicas().size());
newReplica = docCollection.getReplicas().iterator().next();
assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(0).getNodeName());
- assertTrue(newReplica.getStr("ulogDir"), newReplica.getStr("ulogDir").equals(ulogDir) || newReplica.getStr("ulogDir").equals(ulogDir+'/'));
- assertTrue(newReplica.getStr("dataDir"),newReplica.getStr("dataDir").equals(dataDir) || newReplica.getStr("dataDir").equals(dataDir+'/'));
+ assertTrue(
+ newReplica.getStr("ulogDir"),
+ newReplica.getStr("ulogDir").equals(ulogDir)
+ || newReplica.getStr("ulogDir").equals(ulogDir + '/'));
+ assertTrue(
+ newReplica.getStr("dataDir"),
+ newReplica.getStr("dataDir").equals(dataDir)
+ || newReplica.getStr("dataDir").equals(dataDir + '/'));
assertEquals(0, cluster.getJettySolrRunner(1).getCoreContainer().getCores().size());
cluster.getSolrClient().commit(coll);
- assertEquals(numDocs, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
+ assertEquals(
+ numDocs,
+ cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
CollectionAdminRequest.deleteCollection(coll).process(cluster.getSolrClient());
}
@@ -147,26 +171,38 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
Replica replica = getCollectionState(coll).getReplicas().iterator().next();
cluster.getJettySolrRunners().get(0).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllReplicasNotLive(
+ cluster.getSolrClient().getZkStateReader(), 20000));
// move replica from node0 -> node1
- new CollectionAdminRequest.MoveReplica(coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
+ new CollectionAdminRequest.MoveReplica(
+ coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
.process(cluster.getSolrClient());
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(
+ cluster.getSolrClient().getZkStateReader(), 20000));
cluster.getJettySolrRunners().get(1).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllReplicasNotLive(
+ cluster.getSolrClient().getZkStateReader(), 20000));
// node0 will delete it replica because of CloudUtil.checkSharedFSFailoverReplaced()
cluster.getJettySolrRunners().get(0).start();
Thread.sleep(5000);
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllReplicasNotLive(
+ cluster.getSolrClient().getZkStateReader(), 20000));
cluster.getJettySolrRunners().get(1).start();
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(
+ cluster.getSolrClient().getZkStateReader(), 20000));
assertEquals(1, getCollectionState(coll).getReplicas().size());
- assertEquals(2, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
+ assertEquals(
+ 2, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
CollectionAdminRequest.deleteCollection(coll).process(cluster.getSolrClient());
}
@@ -180,24 +216,34 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
Replica replica = getCollectionState(coll).getReplicas().iterator().next();
cluster.getJettySolrRunners().get(0).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllReplicasNotLive(
+ cluster.getSolrClient().getZkStateReader(), 20000));
// move replica from node0 -> node1
- new CollectionAdminRequest.MoveReplica(coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
+ new CollectionAdminRequest.MoveReplica(
+ coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
.process(cluster.getSolrClient());
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(
+ cluster.getSolrClient().getZkStateReader(), 20000));
cluster.getJettySolrRunners().get(1).stop();
- assertTrue(ClusterStateUtil.waitForAllReplicasNotLive(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllReplicasNotLive(
+ cluster.getSolrClient().getZkStateReader(), 20000));
cluster.getJettySolrRunners().get(1).start();
// node0 will delete it replica because of CloudUtil.checkSharedFSFailoverReplaced()
cluster.getJettySolrRunners().get(0).start();
Thread.sleep(5000);
- assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 20000));
+ assertTrue(
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(
+ cluster.getSolrClient().getZkStateReader(), 20000));
assertEquals(1, getCollectionState(coll).getReplicas().size());
- assertEquals(100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
+ assertEquals(
+ 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
CollectionAdminRequest.deleteCollection(coll).process(cluster.getSolrClient());
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
index 99c5232..bfd96ad 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
@@ -34,13 +34,15 @@ import org.junit.Test;
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
@TimeoutSuite(millis = TimeUnits.HOUR)
-@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13924")
+@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13924")
public class MoveReplicaHDFSTest extends AbstractMoveReplicaTestBase {
private static MiniDFSCluster dfsCluster;
@@ -70,18 +72,21 @@ public class MoveReplicaHDFSTest extends AbstractMoveReplicaTestBase {
}
@Test
- //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
- //commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
- //commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018
- // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018
- //commented 20-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
- //Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
+ // 2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
+ // 21-May-2018
+ // commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
+ // added 20-Jul-2018
+ // commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
+ // added 17-Aug-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
+ // added 23-Aug-2018
+ // commented 20-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
+ // 6-Sep-2018
+ // Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
+ // added 20-Sep-2018
+ @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
public void testNormalFailedMove() throws Exception {
inPlaceMove = false;
super.testFailedMove();
}
-
-
}
-
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
index 7db2ed9..317c017 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.cloud;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
@@ -31,13 +33,10 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrQuery;
@@ -58,9 +57,9 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.util.LogLevel;
import org.apache.solr.util.TestInjection;
@@ -69,43 +68,51 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Nightly
@Slow
@SuppressSSL
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
@LogLevel("org.apache.solr.cloud.*=DEBUG")
-@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
+@LuceneTestCase.BadApple(
+ bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final boolean DEBUG = true;
private static MiniDFSCluster dfsCluster;
- ThreadPoolExecutor executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0,
- Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
- new SolrNamedThreadFactory("testExecutor"));
-
+ ThreadPoolExecutor executor =
+ new ExecutorUtil.MDCAwareThreadPoolExecutor(
+ 0,
+ Integer.MAX_VALUE,
+ 5,
+ TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>(),
+ new SolrNamedThreadFactory("testExecutor"));
+
CompletionService<Object> completionService;
Set<Future<Object>> pending;
private final Map<String, String> collectionUlogDirMap = new HashMap<>();
-
+
@BeforeClass
public static void hdfsFailoverBeforeClass() throws Exception {
System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- System.setProperty("solr.hdfs.blockcache.global", "true"); // always use global cache, this test can create a lot of directories
- schemaString = "schema15.xml";
+ System.setProperty(
+ "solr.hdfs.blockcache.global",
+ "true"); // always use global cache, this test can create a lot of directories
+ schemaString = "schema15.xml";
}
-
+
@AfterClass
public static void hdfsFailoverAfterClass() throws Exception {
try {
@@ -122,18 +129,17 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
super.setUp();
collectionUlogDirMap.clear();
}
-
+
@Override
public void distribSetUp() throws Exception {
super.distribSetUp();
useJettyDataDir = false;
}
-
+
protected String getSolrXml() {
return "solr.xml";
}
-
public SharedFSAutoReplicaFailoverTest() {
completionService = new ExecutorCompletionService<>(executor);
pending = new HashSet<>();
@@ -142,7 +148,8 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
@Test
@ShardsFixed(num = 4)
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- //commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
+ // commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //
+ // added 20-Jul-2018
public void test() throws Exception {
try {
// to keep uncommitted docs during failover
@@ -160,38 +167,41 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
// serially
private void testBasics() throws Exception {
String collection1 = "solrj_collection";
- Create createCollectionRequest = CollectionAdminRequest.createCollection(collection1,"conf1",2,2)
+ Create createCollectionRequest =
+ CollectionAdminRequest.createCollection(collection1, "conf1", 2, 2)
.setRouterField("myOwnField");
CollectionAdminResponse response = createCollectionRequest.process(cloudClient);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
waitForRecoveriesToFinish(collection1, false);
-
+
String collection2 = "solrj_collection2";
- createCollectionRequest = CollectionAdminRequest.createCollection(collection2,"conf1",2,2)
+ createCollectionRequest =
+ CollectionAdminRequest.createCollection(collection2, "conf1", 2, 2)
.setRouterField("myOwnField");
CollectionAdminResponse response2 = createCollectionRequest.process(getCommonCloudSolrClient());
assertEquals(0, response2.getStatus());
assertTrue(response2.isSuccess());
-
+
waitForRecoveriesToFinish(collection2, false);
-
+
String collection3 = "solrj_collection3";
- createCollectionRequest = CollectionAdminRequest.createCollection(collection3,"conf1",5,1)
+ createCollectionRequest =
+ CollectionAdminRequest.createCollection(collection3, "conf1", 5, 1)
.setRouterField("myOwnField");
CollectionAdminResponse response3 = createCollectionRequest.process(getCommonCloudSolrClient());
assertEquals(0, response3.getStatus());
assertTrue(response3.isSuccess());
-
+
waitForRecoveriesToFinish(collection3, false);
// a collection has only 1 replica per a shard
String collection4 = "solrj_collection4";
- createCollectionRequest = CollectionAdminRequest.createCollection(collection4,"conf1",5,1)
- .setRouterField("text");
+ createCollectionRequest =
+ CollectionAdminRequest.createCollection(collection4, "conf1", 5, 1).setRouterField("text");
CollectionAdminResponse response4 = createCollectionRequest.process(getCommonCloudSolrClient());
assertEquals(0, response4.getStatus());
@@ -204,7 +214,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
// add some documents to collection4
final int numDocs = 100;
- addDocs(collection4, numDocs, false); // indexed but not committed
+ addDocs(collection4, numDocs, false); // indexed but not committed
// no result because not committed yet
queryAndAssertResultSize(collection4, 0, 10000);
@@ -218,14 +228,21 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
assertSliceAndReplicaCount(collection1, 2, 2, 120000);
- assertEquals(4, ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection1));
- assertTrue(ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection2) < 4);
+ assertEquals(
+ 4,
+ ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection1));
+ assertTrue(
+ ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection2)
+ < 4);
- // collection3 has maxShardsPerNode=1, there are 4 standard jetties and one control jetty and 2 nodes stopped
- ClusterStateUtil.waitForLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection3, 3, 30000);
+ // collection3 has maxShardsPerNode=1, there are 4 standard jetties and one control jetty and 2
+ // nodes stopped
+ ClusterStateUtil.waitForLiveAndActiveReplicaCount(
+ cloudClient.getZkStateReader(), collection3, 3, 30000);
// collection4 has maxShardsPerNode=5 and setMaxShardsPerNode=5
- ClusterStateUtil.waitForLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection4, 5, 30000);
+ ClusterStateUtil.waitForLiveAndActiveReplicaCount(
+ cloudClient.getZkStateReader(), collection4, 5, 30000);
// all docs should be queried after failover
cloudClient.commit(); // to query all docs
@@ -233,19 +250,29 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
queryAndAssertResultSize(collection4, numDocs, 10000);
// collection1 should still be at 4
- assertEquals(4, ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection1));
+ assertEquals(
+ 4,
+ ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection1));
// and collection2 less than 4
- assertTrue(ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection2) < 4);
+ assertTrue(
+ ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection2)
+ < 4);
assertUlogDir(collections);
boolean allowOverseerRestart = random().nextBoolean();
- List<JettySolrRunner> stoppedJetties = allowOverseerRestart
- ? jettys.stream().filter(jettySolrRunner -> random().nextBoolean()).collect(Collectors.toList()) : notOverseerJetties();
+ List<JettySolrRunner> stoppedJetties =
+ allowOverseerRestart
+ ? jettys.stream()
+ .filter(jettySolrRunner -> random().nextBoolean())
+ .collect(Collectors.toList())
+ : notOverseerJetties();
ChaosMonkey.stop(stoppedJetties);
controlJetty.stop();
- assertTrue("Timeout waiting for all not live", waitingForReplicasNotLive(cloudClient.getZkStateReader(), 45000, stoppedJetties));
+ assertTrue(
+ "Timeout waiting for all not live",
+ waitingForReplicasNotLive(cloudClient.getZkStateReader(), 45000, stoppedJetties));
ChaosMonkey.start(stoppedJetties);
controlJetty.start();
@@ -276,8 +303,9 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
long startTimestamp = System.nanoTime();
long actualResultSize = 0;
- while(true) {
- if (System.nanoTime() - startTimestamp > TimeUnit.MILLISECONDS.toNanos(timeoutMS) || actualResultSize > expectedResultSize) {
+ while (true) {
+ if (System.nanoTime() - startTimestamp > TimeUnit.MILLISECONDS.toNanos(timeoutMS)
+ || actualResultSize > expectedResultSize) {
fail("expected: " + expectedResultSize + ", actual: " + actualResultSize);
}
SolrParams queryAll = new SolrQuery("*:*");
@@ -285,18 +313,20 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
try {
QueryResponse queryResponse = cloudClient.query(queryAll);
actualResultSize = queryResponse.getResults().getNumFound();
- if(expectedResultSize == actualResultSize) {
+ if (expectedResultSize == actualResultSize) {
return;
}
} catch (SolrServerException | IOException e) {
- log.warn("Querying solr threw an exception. This can be expected to happen during restarts.", e);
+ log.warn(
+ "Querying solr threw an exception. This can be expected to happen during restarts.", e);
}
Thread.sleep(1000);
}
}
- private void addDocs(String collection, int numDocs, boolean commit) throws SolrServerException, IOException {
+ private void addDocs(String collection, int numDocs, boolean commit)
+ throws SolrServerException, IOException {
for (int docId = 1; docId <= numDocs; docId++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", docId);
@@ -309,12 +339,11 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
}
}
- /**
- * After failover, ulogDir should not be changed.
- */
+ /** After failover, ulogDir should not be changed. */
private void assertUlogDir(String... collections) {
for (String collection : collections) {
- Collection<Slice> slices = cloudClient.getZkStateReader().getClusterState().getCollection(collection).getSlices();
+ Collection<Slice> slices =
+ cloudClient.getZkStateReader().getClusterState().getCollection(collection).getSlices();
for (Slice slice : slices) {
for (Replica replica : slice.getReplicas()) {
Map<String, Object> properties = replica.getProperties();
@@ -335,19 +364,25 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
// Overseer failover is not currently guaranteed with MoveReplica or Policy Framework
private List<JettySolrRunner> notOverseerJetties() throws IOException, SolrServerException {
- CollectionAdminResponse response = CollectionAdminRequest.getOverseerStatus().process(cloudClient);
+ CollectionAdminResponse response =
+ CollectionAdminRequest.getOverseerStatus().process(cloudClient);
String overseerNode = (String) response.getResponse().get("leader");
- return jettys.stream().filter(jetty -> !(jetty.getCoreContainer() != null && overseerNode.equals(jetty.getNodeName())))
- .collect(Collectors.toList());
+ return jettys.stream()
+ .filter(
+ jetty ->
+ !(jetty.getCoreContainer() != null && overseerNode.equals(jetty.getNodeName())))
+ .collect(Collectors.toList());
}
- private boolean waitingForReplicasNotLive(ZkStateReader zkStateReader, int timeoutInMs, List<JettySolrRunner> jetties) {
- Set<String> nodeNames = jetties.stream()
- .filter(jetty -> jetty.getCoreContainer() != null)
- .map(JettySolrRunner::getNodeName)
- .collect(Collectors.toSet());
- long timeout = System.nanoTime()
- + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
+ private boolean waitingForReplicasNotLive(
+ ZkStateReader zkStateReader, int timeoutInMs, List<JettySolrRunner> jetties) {
+ Set<String> nodeNames =
+ jetties.stream()
+ .filter(jetty -> jetty.getCoreContainer() != null)
+ .map(JettySolrRunner::getNodeName)
+ .collect(Collectors.toSet());
+ long timeout =
+ System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
boolean success = false;
while (!success && System.nanoTime() < timeout) {
success = true;
@@ -363,8 +398,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
Collection<Replica> replicas = slice.getReplicas();
for (Replica replica : replicas) {
if (nodeNames.contains(replica.getNodeName())) {
- boolean live = clusterState.liveNodesContain(replica
- .getNodeName());
+ boolean live = clusterState.liveNodesContain(replica.getNodeName());
if (live) {
success = false;
}
@@ -387,7 +421,9 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
return success;
}
- private void assertSliceAndReplicaCount(String collection, int numSlices, int numReplicas, int timeOutInMs) throws InterruptedException {
+ private void assertSliceAndReplicaCount(
+ String collection, int numSlices, int numReplicas, int timeOutInMs)
+ throws InterruptedException {
TimeOut timeOut = new TimeOut(timeOutInMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
while (!timeOut.hasTimedOut()) {
ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
@@ -397,7 +433,8 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
for (Slice slice : slices) {
int count = 0;
for (Replica replica : slice.getReplicas()) {
- if (replica.getState() == Replica.State.ACTIVE && clusterState.liveNodesContain(replica.getNodeName())) {
+ if (replica.getState() == Replica.State.ACTIVE
+ && clusterState.liveNodesContain(replica.getNodeName())) {
count++;
}
}
@@ -409,7 +446,14 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
}
Thread.sleep(200);
}
- fail("Expected numSlices=" + numSlices + " numReplicas=" + numReplicas + " but found " + cloudClient.getZkStateReader().getClusterState().getCollection(collection) + " with /live_nodes: " + cloudClient.getZkStateReader().getClusterState().getLiveNodes());
+ fail(
+ "Expected numSlices="
+ + numSlices
+ + " numReplicas="
+ + numReplicas
+ + " but found "
+ + cloudClient.getZkStateReader().getClusterState().getCollection(collection)
+ + " with /live_nodes: "
+ + cloudClient.getZkStateReader().getClusterState().getLiveNodes());
}
-
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java
index 62fea83..4cbd70e 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java
@@ -17,12 +17,11 @@
package org.apache.solr.cloud.api.collections;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -37,112 +36,120 @@ import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-@LuceneTestCase.SuppressCodecs({"SimpleText"}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsCloudIncrementalBackupTest extends AbstractIncrementalBackupTest{
- public static final String SOLR_XML = "<solr>\n" +
- "\n" +
- " <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
- " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
- " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +
- "\n" +
- " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n" +
- " <str name=\"urlScheme\">${urlScheme:}</str>\n" +
- " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
- " <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
- " </shardHandlerFactory>\n" +
- "\n" +
- " <solrcloud>\n" +
- " <str name=\"host\">127.0.0.1</str>\n" +
- " <int name=\"hostPort\">${hostPort:8983}</int>\n" +
- " <str name=\"hostContext\">${hostContext:solr}</str>\n" +
- " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +
- " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +
- " <int name=\"leaderVoteWait\">10000</int>\n" +
- " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +
- " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
- " </solrcloud>\n" +
- " \n" +
- " <backup>\n" +
- " <repository name=\"trackingBackupRepository\" default=\"true\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n" +
- " <str name=\"delegateRepoName\">hdfs</str>\n" +
- " </repository>\n" +
- " <repository name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n" +
- " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n" +
- " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n" +
- " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n" +
- " </repository>\n" +
- " </backup>\n" +
- " \n" +
- "</solr>\n";
+@LuceneTestCase.SuppressCodecs({
+ "SimpleText"
+}) // Backups do checksum validation against a footer value not present in 'SimpleText'
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+public class HdfsCloudIncrementalBackupTest extends AbstractIncrementalBackupTest {
+ public static final String SOLR_XML =
+ "<solr>\n"
+ + "\n"
+ + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"
+ + " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"
+ + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
+ + "\n"
+ + " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n"
+ + " <str name=\"urlScheme\">${urlScheme:}</str>\n"
+ + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"
+ + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"
+ + " </shardHandlerFactory>\n"
+ + "\n"
+ + " <solrcloud>\n"
+ + " <str name=\"host\">127.0.0.1</str>\n"
+ + " <int name=\"hostPort\">${hostPort:8983}</int>\n"
+ + " <str name=\"hostContext\">${hostContext:solr}</str>\n"
+ + " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"
+ + " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"
+ + " <int name=\"leaderVoteWait\">10000</int>\n"
+ + " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
+ + " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
+ + " </solrcloud>\n"
+ + " \n"
+ + " <backup>\n"
+ + " <repository name=\"trackingBackupRepository\" default=\"true\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n"
+ + " <str name=\"delegateRepoName\">hdfs</str>\n"
+ + " </repository>\n"
+ + " <repository name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n"
+ + " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n"
+ + " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n"
+ + " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n"
+ + " </repository>\n"
+ + " </backup>\n"
+ + " \n"
+ + "</solr>\n";
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
+ private static MiniDFSCluster dfsCluster;
+ private static String hdfsUri;
+ private static FileSystem fs;
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
+ hdfsUri = HdfsTestUtil.getURI(dfsCluster);
+ try {
+ URI uri = new URI(hdfsUri);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ fs = FileSystem.get(uri, conf);
- if (fs instanceof DistributedFileSystem) {
- // Make sure dfs is not in safe mode
- while (((DistributedFileSystem) fs).setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- Thread.interrupted();
- // continue
- }
- }
- }
-
- fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
+ if (fs instanceof DistributedFileSystem) {
+ // Make sure dfs is not in safe mode
+ while (((DistributedFileSystem) fs)
+ .setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ // continue
+ }
}
+ }
- System.setProperty("solr.hdfs.default.backup.path", "/backup");
- System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
- useFactory("solr.StandardDirectoryFactory");
-
- configureCluster(NUM_SHARDS)// nodes
- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
- .withSolrXml(SOLR_XML)
- .configure();
+ fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
+ } catch (IOException | URISyntaxException e) {
+ throw new RuntimeException(e);
}
- @AfterClass
- public static void teardownClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- }
- }
+ System.setProperty("solr.hdfs.default.backup.path", "/backup");
+ System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
+ useFactory("solr.StandardDirectoryFactory");
- @Override
- public String getCollectionNamePrefix() {
- return "hdfsbackuprestore";
- }
+ configureCluster(NUM_SHARDS) // nodes
+ .addConfig(
+ "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .withSolrXml(SOLR_XML)
+ .configure();
+ }
- @Override
- public String getBackupLocation() {
- return null;
+ @AfterClass
+ public static void teardownClass() throws Exception {
+ IOUtils.closeQuietly(fs);
+ fs = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.default.backup.path");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
}
+ }
+
+ @Override
+ public String getCollectionNamePrefix() {
+ return "hdfsbackuprestore";
+ }
+
+ @Override
+ public String getBackupLocation() {
+ return null;
+ }
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
index 2cd9799..f2ee6c8 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
@@ -31,13 +31,16 @@ import org.junit.BeforeClass;
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
@TimeoutSuite(millis = TimeUnits.HOUR)
-public class HdfsCollectionsAPIDistributedZkTest extends AbstractCollectionsAPIDistributedZkTestBase {
+public class HdfsCollectionsAPIDistributedZkTest
+ extends AbstractCollectionsAPIDistributedZkTestBase {
private static MiniDFSCluster dfsCluster;
@BeforeClass
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index 346b543..a4badb9 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -16,6 +16,11 @@
*/
package org.apache.solr.cloud.api.collections;
+import static org.apache.solr.core.backup.BackupManager.CONFIG_STATE_DIR;
+import static org.apache.solr.core.backup.BackupManager.TRADITIONAL_BACKUP_PROPS_FILE;
+import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@@ -25,11 +30,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -37,6 +37,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -55,56 +57,56 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.core.backup.BackupManager.TRADITIONAL_BACKUP_PROPS_FILE;
-import static org.apache.solr.core.backup.BackupManager.CONFIG_STATE_DIR;
-import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR;
-
-/**
- * This class implements the tests for HDFS integration for Solr backup/restore capability.
- */
-@LuceneTestCase.SuppressCodecs({"SimpleText"}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+/** This class implements the tests for HDFS integration for Solr backup/restore capability. */
+@LuceneTestCase.SuppressCodecs({
+ "SimpleText"
+}) // Backups do checksum validation against a footer value not present in 'SimpleText'
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCase {
- public static final String SOLR_XML = "<solr>\n" +
- "\n" +
- " <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
- " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
- " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +
- "\n" +
- " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n" +
- " <str name=\"urlScheme\">${urlScheme:}</str>\n" +
- " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
- " <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
- " </shardHandlerFactory>\n" +
- "\n" +
- " <solrcloud>\n" +
- " <str name=\"host\">127.0.0.1</str>\n" +
- " <int name=\"hostPort\">${hostPort:8983}</int>\n" +
- " <str name=\"hostContext\">${hostContext:solr}</str>\n" +
- " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +
- " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +
- " <int name=\"leaderVoteWait\">10000</int>\n" +
- " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +
- " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
- " </solrcloud>\n" +
- " \n" +
- " <backup>\n" +
- " <repository name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n" +
- " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n" +
- " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n" +
- " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n" +
- " </repository>\n" +
- " <repository name=\"poisioned\" default=\"true\" "
- + "class=\"org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository\"> \n" +
- " </repository>\n" +
- " </backup>\n" +
- " \n" +
- "</solr>\n";
+ public static final String SOLR_XML =
+ "<solr>\n"
+ + "\n"
+ + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"
+ + " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"
+ + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
+ + "\n"
+ + " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n"
+ + " <str name=\"urlScheme\">${urlScheme:}</str>\n"
+ + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"
+ + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"
+ + " </shardHandlerFactory>\n"
+ + "\n"
+ + " <solrcloud>\n"
+ + " <str name=\"host\">127.0.0.1</str>\n"
+ + " <int name=\"hostPort\">${hostPort:8983}</int>\n"
+ + " <str name=\"hostContext\">${hostContext:solr}</str>\n"
+ + " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"
+ + " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"
+ + " <int name=\"leaderVoteWait\">10000</int>\n"
+ + " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
+ + " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
+ + " </solrcloud>\n"
+ + " \n"
+ + " <backup>\n"
+ + " <repository name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n"
+ + " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n"
+ + " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n"
+ + " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n"
+ + " </repository>\n"
+ + " <repository name=\"poisioned\" default=\"true\" "
+ + "class=\"org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository\"> \n"
+ + " </repository>\n"
+ + " </backup>\n"
+ + " \n"
+ + "</solr>\n";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static MiniDFSCluster dfsCluster;
@@ -142,12 +144,24 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
useFactory("solr.StandardDirectoryFactory");
- configureCluster(NUM_SHARDS)// nodes
- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"))
- .addConfig("confFaulty", TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"))
- .withSolrXml(SOLR_XML)
- .configure();
- cluster.getZkClient().delete(ZkConfigSetService.CONFIGS_ZKNODE + Path.SEPARATOR + "confFaulty" + Path.SEPARATOR + "solrconfig.xml", -1, true);
+ configureCluster(NUM_SHARDS) // nodes
+ .addConfig(
+ "conf1", TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"))
+ .addConfig(
+ "confFaulty",
+ TEST_PATH().resolve("configsets").resolve("cloud-subdirs").resolve("conf"))
+ .withSolrXml(SOLR_XML)
+ .configure();
+ cluster
+ .getZkClient()
+ .delete(
+ ZkConfigSetService.CONFIGS_ZKNODE
+ + Path.SEPARATOR
+ + "confFaulty"
+ + Path.SEPARATOR
+ + "solrconfig.xml",
+ -1,
+ true);
}
@AfterClass
@@ -184,13 +198,14 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
String backupName = "configonlybackup";
CloudSolrClient solrClient = cluster.getSolrClient();
- CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
- .setRepositoryName(getBackupRepoName())
- .setIncremental(false)
- .setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
+ CollectionAdminRequest.Backup backup =
+ CollectionAdminRequest.backupCollection(collectionName, backupName)
+ .setRepositoryName(getBackupRepoName())
+ .setIncremental(false)
+ .setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
backup.process(solrClient);
- Map<String,String> params = new HashMap<>();
+ Map<String, String> params = new HashMap<>();
params.put("location", "/backup");
params.put("solr.hdfs.home", hdfsUri + "/solr");
@@ -199,7 +214,9 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
URI baseLoc = repo.createDirectoryURI("/backup");
- BackupManager mgr = BackupManager.forRestore(repo, solrClient.getZkStateReader(), repo.resolve(baseLoc, backupName));
+ BackupManager mgr =
+ BackupManager.forRestore(
+ repo, solrClient.getZkStateReader(), repo.resolve(baseLoc, backupName));
BackupProperties props = mgr.readBackupProperties();
assertNotNull(props);
assertEquals(collectionName, props.getCollection());
@@ -210,7 +227,8 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
assertNotNull(collectionState);
assertEquals(collectionName, collectionState.getName());
- URI configDirLoc = repo.resolve(baseLoc, backupName, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
+ URI configDirLoc =
+ repo.resolve(baseLoc, backupName, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
assertTrue(repo.exists(configDirLoc));
Collection<String> expected = Arrays.asList(TRADITIONAL_BACKUP_PROPS_FILE, ZK_STATE_DIR);
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/DisableKeyStoresFactory.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/DisableKeyStoresFactory.java
index 088eb5f..e9c7a12 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/DisableKeyStoresFactory.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/DisableKeyStoresFactory.java
@@ -16,25 +16,20 @@
*/
package org.apache.solr.cloud.hdfs;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.TrustManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.ssl.KeyStoresFactory;
import org.apache.hadoop.security.ssl.SSLFactory;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.TrustManager;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-
public class DisableKeyStoresFactory implements KeyStoresFactory {
@Override
- public void init(SSLFactory.Mode mode) throws IOException, GeneralSecurityException {
-
- }
+ public void init(SSLFactory.Mode mode) throws IOException, GeneralSecurityException {}
@Override
- public void destroy() {
-
- }
+ public void destroy() {}
@Override
public KeyManager[] getKeyManagers() {
@@ -47,9 +42,7 @@ public class DisableKeyStoresFactory implements KeyStoresFactory {
}
@Override
- public void setConf(Configuration conf) {
-
- }
+ public void setConf(Configuration conf) {}
@Override
public Configuration getConf() {
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
index 5bad1c7..ce89466 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
@@ -33,12 +33,15 @@ import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HDFSCollectionsAPITest extends SolrCloudTestCase {
private static MiniDFSCluster dfsCluster;
@@ -52,7 +55,6 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase {
cluster.uploadConfigSet(configset("cloud-hdfs"), "conf1");
}
-
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -72,7 +74,8 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase {
String collection = "test";
cluster.getSolrClient().setDefaultCollection(collection);
CollectionAdminRequest.createCollection(collection, "conf1", 1, 1)
- .setCreateNodeSet(jettySolrRunner.getNodeName()).process(cluster.getSolrClient());
+ .setCreateNodeSet(jettySolrRunner.getNodeName())
+ .process(cluster.getSolrClient());
waitForState("", collection, clusterShape(1, 1));
cluster.getSolrClient().setDefaultCollection(collection);
cluster.getSolrClient().add(new SolrInputDocument("id", "1"));
@@ -81,19 +84,22 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase {
cluster.getSolrClient().add(new SolrInputDocument("id", "3"));
jettySolrRunner.stop();
- waitForState("", collection, (liveNodes, collectionState) -> {
- Replica replica = collectionState.getSlice("shard1").getReplicas().iterator().next();
- return replica.getState() == Replica.State.DOWN;
- });
+ waitForState(
+ "",
+ collection,
+ (liveNodes, collectionState) -> {
+ Replica replica = collectionState.getSlice("shard1").getReplicas().iterator().next();
+ return replica.getState() == Replica.State.DOWN;
+ });
CollectionAdminRequest.deleteCollection(collection).process(cluster.getSolrClient());
jettySolrRunner.start();
CollectionAdminRequest.createCollection(collection, "conf1", 1, 1)
- .setCreateNodeSet(cluster.getJettySolrRunner(1).getNodeName()).process(cluster.getSolrClient());
+ .setCreateNodeSet(cluster.getJettySolrRunner(1).getNodeName())
+ .process(cluster.getSolrClient());
waitForState("", collection, clusterShape(1, 1));
QueryResponse response = cluster.getSolrClient().query(collection, new SolrQuery("*:*"));
assertEquals(0L, response.getResults().getNumFound());
}
-
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java
index 8596e56..332e39a 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java
@@ -16,38 +16,38 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractBasicDistributedZk2TestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsBasicDistributedZk2Test extends AbstractBasicDistributedZk2TestBase {
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -56,10 +56,9 @@ public class HdfsBasicDistributedZk2Test extends AbstractBasicDistributedZk2Test
dfsCluster = null;
}
}
-
+
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
index 9a3183f..a66b255 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
@@ -16,12 +16,13 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractBasicDistributedZkTestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
@@ -29,17 +30,17 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsBasicDistributedZkTest extends AbstractBasicDistributedZkTestBase {
private static MiniDFSCluster dfsCluster;
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java
index 5642b85..f5878ca 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java
@@ -16,38 +16,41 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractChaosMonkeyNothingIsSafeTestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsChaosMonkeyNothingIsSafeTest extends AbstractChaosMonkeyNothingIsSafeTestBase {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.global", "true"); // always use global cache, this test can create a lot of directories
+ System.setProperty(
+ "solr.hdfs.blockcache.global",
+ "true"); // always use global cache, this test can create a lot of directories
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java
index 88fc163..c93f287 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java
@@ -16,40 +16,42 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractChaosMonkeySafeLeaderTestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsChaosMonkeySafeLeaderTest extends AbstractChaosMonkeySafeLeaderTestBase {
private static final String DIRECTORY_FACTORY = "org.apache.solr.core.HdfsDirectoryFactory";
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.global", "true"); // always use global cache, this test can create a lot of directories
+ System.setProperty(
+ "solr.hdfs.blockcache.global",
+ "true"); // always use global cache, this test can create a lot of directories
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -59,15 +61,15 @@ public class HdfsChaosMonkeySafeLeaderTest extends AbstractChaosMonkeySafeLeader
System.clearProperty("solr.hdfs.blockcache.global");
}
}
-
+
@Override
public void distribSetUp() throws Exception {
super.distribSetUp();
-
+
// super class may hard code directory
useFactory("org.apache.solr.core.HdfsDirectoryFactory");
}
-
+
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsFakeGroupMapping.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsFakeGroupMapping.java
index 2865d81..7ceb687 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsFakeGroupMapping.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsFakeGroupMapping.java
@@ -16,14 +16,11 @@
*/
package org.apache.solr.cloud.hdfs;
-import org.apache.hadoop.security.GroupMappingServiceProvider;
-
import java.util.Collections;
import java.util.List;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
-/**
- * Fake mapping for Hadoop to prevent falling back to Shell group provider
- */
+/** Fake mapping for Hadoop to prevent falling back to Shell group provider */
public class HdfsFakeGroupMapping implements GroupMappingServiceProvider {
@Override
public List<String> getGroups(String user) {
@@ -31,10 +28,8 @@ public class HdfsFakeGroupMapping implements GroupMappingServiceProvider {
}
@Override
- public void cacheGroupsRefresh() {
- }
+ public void cacheGroupsRefresh() {}
@Override
- public void cacheGroupsAdd(List<String> groups) {
- }
+ public void cacheGroupsAdd(List<String> groups) {}
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java
index 4b17cc3..a4ce9b9 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java
@@ -16,9 +16,9 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.QuickPatchThreadsFilter;
@@ -29,24 +29,25 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsNNFailoverTest extends AbstractBasicDistributedZkTestBase {
private static final String COLLECTION = "collection";
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false, true);
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -55,12 +56,12 @@ public class HdfsNNFailoverTest extends AbstractBasicDistributedZkTestBase {
dfsCluster = null;
}
}
-
+
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
+
public HdfsNNFailoverTest() {
super();
sliceCount = 1;
@@ -75,12 +76,11 @@ public class HdfsNNFailoverTest extends AbstractBasicDistributedZkTestBase {
@Override
public void test() throws Exception {
createCollection(COLLECTION, "conf1", 1, 1);
-
+
waitForRecoveriesToFinish(COLLECTION, false);
-
- // TODO: SOLR-7360 Enable HDFS NameNode failover testing.
-// dfsCluster.transitionToStandby(0);
-// dfsCluster.transitionToActive(1);
- }
+ // TODO: SOLR-7360 Enable HDFS NameNode failover testing.
+ // dfsCluster.transitionToStandby(0);
+ // dfsCluster.transitionToActive(1);
+ }
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
index f3742b4..bf0673c 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
@@ -16,12 +16,12 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
import java.net.URI;
import java.util.HashSet;
import java.util.Set;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -39,16 +39,17 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
-
+
private static MiniDFSCluster dfsCluster;
@BeforeClass
@@ -64,48 +65,51 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
dfsCluster = null;
}
}
-
+
@Before
public void setUp() throws Exception {
super.setUp();
}
-
+
@After
public void tearDown() throws Exception {
super.tearDown();
}
-
+
@Test
public void testBasic() throws IOException {
long startRecoverLeaseSuccessCount = FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get();
-
+
URI uri = dfsCluster.getURI();
Path path = new Path(uri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
FileSystem fs1 = FileSystem.get(path.toUri(), conf);
Path testFile = new Path(uri.toString() + "/testfile");
FSDataOutputStream out = fs1.create(testFile);
-
+
out.write(5);
out.hflush();
out.close();
- FSHDFSUtils.recoverFileLease(fs1, testFile, conf, new CallerInfo() {
-
- @Override
- public boolean isCallerClosed() {
- return false;
- }
- });
+ FSHDFSUtils.recoverFileLease(
+ fs1,
+ testFile,
+ conf,
+ new CallerInfo() {
+
+ @Override
+ public boolean isCallerClosed() {
+ return false;
+ }
+ });
assertEquals(0, FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get() - startRecoverLeaseSuccessCount);
-
+
fs1.close();
-
FileSystem fs2 = FileSystem.get(path.toUri(), conf);
Path testFile2 = new Path(uri.toString() + "/testfile2");
FSDataOutputStream out2 = fs2.create(testFile2);
-
+
if (random().nextBoolean()) {
int cnt = random().nextInt(100);
for (int i = 0; i < cnt; i++) {
@@ -114,38 +118,41 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
out2.hflush();
}
-
// closing the fs will close the file it seems
// fs2.close();
-
+
FileSystem fs3 = FileSystem.get(path.toUri(), conf);
- FSHDFSUtils.recoverFileLease(fs3, testFile2, conf, new CallerInfo() {
-
- @Override
- public boolean isCallerClosed() {
- return false;
- }
- });
+ FSHDFSUtils.recoverFileLease(
+ fs3,
+ testFile2,
+ conf,
+ new CallerInfo() {
+
+ @Override
+ public boolean isCallerClosed() {
+ return false;
+ }
+ });
assertEquals(1, FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get() - startRecoverLeaseSuccessCount);
-
+
fs3.close();
fs2.close();
}
-
+
@Test
public void testMultiThreaded() throws Exception {
long startRecoverLeaseSuccessCount = FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get();
-
+
final URI uri = dfsCluster.getURI();
final Path path = new Path(uri);
final Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
-
+
// n threads create files
class WriterThread extends Thread {
private FileSystem fs;
private int id;
-
+
public WriterThread(int id) {
this.id = id;
try {
@@ -154,14 +161,14 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
throw new RuntimeException(e);
}
}
-
+
@Override
public void run() {
Path testFile = new Path(uri.toString() + "/file-" + id);
FSDataOutputStream out;
try {
out = fs.create(testFile);
-
+
if (random().nextBoolean()) {
int cnt = random().nextInt(100);
for (int i = 0; i < cnt; i++) {
@@ -173,20 +180,20 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
throw new RuntimeException();
}
}
-
+
public void close() throws IOException {
fs.close();
}
-
+
public int getFileId() {
return id;
}
}
-
+
class RecoverThread extends Thread {
private FileSystem fs;
private int id;
-
+
public RecoverThread(int id) {
this.id = id;
try {
@@ -195,60 +202,63 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
throw new RuntimeException(e);
}
}
-
+
@Override
public void run() {
Path testFile = new Path(uri.toString() + "/file-" + id);
try {
- FSHDFSUtils.recoverFileLease(fs, testFile, conf, new CallerInfo() {
-
- @Override
- public boolean isCallerClosed() {
- return false;
- }
- });
+ FSHDFSUtils.recoverFileLease(
+ fs,
+ testFile,
+ conf,
+ new CallerInfo() {
+
+ @Override
+ public boolean isCallerClosed() {
+ return false;
+ }
+ });
} catch (IOException e) {
throw new RuntimeException(e);
}
}
-
+
public void close() throws IOException {
fs.close();
}
}
-
+
Set<WriterThread> writerThreads = new HashSet<WriterThread>();
Set<RecoverThread> recoverThreads = new HashSet<RecoverThread>();
-
+
int threadCount = 3;
for (int i = 0; i < threadCount; i++) {
WriterThread wt = new WriterThread(i);
writerThreads.add(wt);
wt.run();
}
-
+
for (WriterThread wt : writerThreads) {
wt.join();
}
-
+
Thread.sleep(2000);
-
+
for (WriterThread wt : writerThreads) {
RecoverThread rt = new RecoverThread(wt.getFileId());
recoverThreads.add(rt);
rt.run();
}
-
+
for (WriterThread wt : writerThreads) {
wt.close();
}
-
+
for (RecoverThread rt : recoverThreads) {
rt.close();
}
- assertEquals(threadCount, FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get() - startRecoverLeaseSuccessCount);
-
+ assertEquals(
+ threadCount, FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get() - startRecoverLeaseSuccessCount);
}
-
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
index d74f1bf..de8daa1 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
@@ -32,22 +32,25 @@ import org.junit.BeforeClass;
@Slow
@Nightly
@LuceneTestCase.AwaitsFix(bugUrl = "SOLR-15405")
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsRecoveryZkTest extends AbstractRecoveryZkTestBase {
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
cluster.uploadConfigSet(configset("cloud-hdfs"), "conf");
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -60,5 +63,4 @@ public class HdfsRecoveryZkTest extends AbstractRecoveryZkTestBase {
}
}
}
-
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
index b606090..23faa29 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
@@ -16,27 +16,29 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractRestartWhileUpdatingTestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsRestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTestBase {
private static MiniDFSCluster dfsCluster;
@@ -51,7 +53,7 @@ public class HdfsRestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTe
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -60,7 +62,7 @@ public class HdfsRestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTe
dfsCluster = null;
}
}
-
+
@Override
protected String getDataDir(String dataDir) {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java
index a06c07d..18319f1 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java
@@ -16,37 +16,38 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractSyncSliceTestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsSyncSliceTest extends AbstractSyncSliceTestBase {
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
index 0840ae9..38cf0f1 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
@@ -31,7 +31,6 @@ import java.util.TimerTask;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinWorkerThread;
import java.util.regex.Pattern;
-
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -62,13 +61,14 @@ import org.slf4j.LoggerFactory;
public class HdfsTestUtil {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final String SOLR_HACK_FOR_CLASS_VERIFICATION_FIELD = "SOLR_HACK_FOR_CLASS_VERIFICATION";
+ private static final String SOLR_HACK_FOR_CLASS_VERIFICATION_FIELD =
+ "SOLR_HACK_FOR_CLASS_VERIFICATION";
private static final String LOGICAL_HOSTNAME = "ha-nn-uri-%d";
private static final boolean HA_TESTING_ENABLED = false; // SOLR-XXX
- private static Map<MiniDFSCluster,Timer> timers = new HashMap<>();
+ private static Map<MiniDFSCluster, Timer> timers = new HashMap<>();
private static final Object TIMERS_LOCK = new Object();
private static FSDataOutputStream badTlogOutStream;
@@ -93,9 +93,9 @@ public class HdfsTestUtil {
/**
* If Hadoop home is set via environment variable HADOOP_HOME or Java system property
- * hadoop.home.dir, the behavior of test is undefined. Ensure that these are not set
- * before starting. It is not possible to easily unset environment variables so better
- * to bail out early instead of trying to test.
+ * hadoop.home.dir, the behavior of test is undefined. Ensure that these are not set before
+ * starting. It is not possible to easily unset environment variables so better to bail out early
+ * instead of trying to test.
*/
private static void ensureHadoopHomeNotSet() {
if (System.getenv("HADOOP_HOME") != null) {
@@ -106,24 +106,23 @@ public class HdfsTestUtil {
}
}
- /**
- * Hadoop integration tests fail on Windows without Hadoop NativeIO
- */
+ /** Hadoop integration tests fail on Windows without Hadoop NativeIO */
private static void checkHadoopWindows() {
- SolrTestCaseJ4.assumeTrue("Hadoop does not work on Windows without Hadoop NativeIO",
+ SolrTestCaseJ4.assumeTrue(
+ "Hadoop does not work on Windows without Hadoop NativeIO",
!Constants.WINDOWS || NativeIO.isAvailable());
}
- /**
- * Ensure that the tests are picking up the modified Hadoop classes
- */
+ /** Ensure that the tests are picking up the modified Hadoop classes */
private static void checkOverriddenHadoopClasses() {
- List<Class<?>> modifiedHadoopClasses = new ArrayList<>(Arrays.asList(
- DiskChecker.class,
- FileUtil.class,
- HardLink.class,
- NameNodeResourceChecker.class,
- RawLocalFileSystem.class));
+ List<Class<?>> modifiedHadoopClasses =
+ new ArrayList<>(
+ Arrays.asList(
+ DiskChecker.class,
+ FileUtil.class,
+ HardLink.class,
+ NameNodeResourceChecker.class,
+ RawLocalFileSystem.class));
// Dodge weird scope errors from the compiler (SOLR-14417)
try {
modifiedHadoopClasses.add(
@@ -134,38 +133,41 @@ public class HdfsTestUtil {
for (Class<?> clazz : modifiedHadoopClasses) {
try {
- SolrTestCaseJ4.assertNotNull("Field on " + clazz.getCanonicalName() + " should not have been null",
+ SolrTestCaseJ4.assertNotNull(
+ "Field on " + clazz.getCanonicalName() + " should not have been null",
clazz.getField(SOLR_HACK_FOR_CLASS_VERIFICATION_FIELD));
} catch (NoSuchFieldException e) {
- SolrTestCaseJ4.fail("Expected to load Solr modified Hadoop class " + clazz.getCanonicalName() +
- " , but it was not found.");
+ SolrTestCaseJ4.fail(
+ "Expected to load Solr modified Hadoop class "
+ + clazz.getCanonicalName()
+ + " , but it was not found.");
}
}
}
- /**
- * Checks that commons-lang3 FastDateFormat works with configured locale
- */
- @SuppressForbidden(reason="Call FastDateFormat.format same way Hadoop calls it")
+ /** Checks that commons-lang3 FastDateFormat works with configured locale */
+ @SuppressForbidden(reason = "Call FastDateFormat.format same way Hadoop calls it")
private static void checkFastDateFormat() {
try {
FastDateFormat.getInstance().format(System.currentTimeMillis());
} catch (ArrayIndexOutOfBoundsException e) {
- SolrTestCaseJ4.assumeNoException("commons-lang3 FastDateFormat doesn't work with " +
- Locale.getDefault().toLanguageTag(), e);
+ SolrTestCaseJ4.assumeNoException(
+ "commons-lang3 FastDateFormat doesn't work with " + Locale.getDefault().toLanguageTag(),
+ e);
}
}
- /**
- * Hadoop fails to generate locale agnostic ids - Checks that generated string matches
- */
+ /** Hadoop fails to generate locale agnostic ids - Checks that generated string matches */
private static void checkGeneratedIdMatches() {
- // This is basically how Namenode generates fsimage ids and checks that the fsimage filename matches
- SolrTestCaseJ4.assumeTrue("Check that generated id matches regex",
- Pattern.matches("(\\d+)", String.format(Locale.getDefault(),"%019d", 0)));
+ // This is basically how Namenode generates fsimage ids and checks that the fsimage filename
+ // matches
+ SolrTestCaseJ4.assumeTrue(
+ "Check that generated id matches regex",
+ Pattern.matches("(\\d+)", String.format(Locale.getDefault(), "%019d", 0)));
}
- public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting, boolean haTesting) throws Exception {
+ public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting, boolean haTesting)
+ throws Exception {
checkAssumptions();
if (!HA_TESTING_ENABLED) haTesting = false;
@@ -184,15 +186,19 @@ public class HdfsTestUtil {
System.setProperty("test.build.data", dir + File.separator + "hdfs" + File.separator + "build");
System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache");
System.setProperty("solr.lock.type", HdfsDirectoryFactory.LOCK_TYPE_HDFS);
- System.setProperty(SolrTestCaseJ4.UPDATELOG_SYSPROP, HdfsDirectoryFactory.HDFS_UPDATE_LOG_CLASS_NAME);
+ System.setProperty(
+ SolrTestCaseJ4.UPDATELOG_SYSPROP, HdfsDirectoryFactory.HDFS_UPDATE_LOG_CLASS_NAME);
- // test-files/solr/solr.xml sets this to be 15000. This isn't long enough for HDFS in some cases.
+ // test-files/solr/solr.xml sets this to be 15000. This isn't long enough for HDFS in some
+ // cases.
System.setProperty("socketTimeout", "90000");
- String blockcacheGlobal = System.getProperty("solr.hdfs.blockcache.global", Boolean.toString(SolrTestCaseJ4.random().nextBoolean()));
+ String blockcacheGlobal =
+ System.getProperty(
+ "solr.hdfs.blockcache.global", Boolean.toString(SolrTestCaseJ4.random().nextBoolean()));
System.setProperty("solr.hdfs.blockcache.global", blockcacheGlobal);
// Limit memory usage for HDFS tests
- if(Boolean.parseBoolean(blockcacheGlobal)) {
+ if (Boolean.parseBoolean(blockcacheGlobal)) {
System.setProperty("solr.hdfs.blockcache.blocksperbank", "4096");
} else {
System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
@@ -200,8 +206,8 @@ public class HdfsTestUtil {
}
int dataNodes = Integer.getInteger("tests.hdfs.numdatanodes", 2);
- final MiniDFSCluster.Builder dfsClusterBuilder = new MiniDFSCluster.Builder(conf)
- .numDataNodes(dataNodes).format(true);
+ final MiniDFSCluster.Builder dfsClusterBuilder =
+ new MiniDFSCluster.Builder(conf).numDataNodes(dataNodes).format(true);
if (haTesting) {
dfsClusterBuilder.nnTopology(MiniDFSNNTopology.simpleHATopology());
}
@@ -226,13 +232,15 @@ public class HdfsTestUtil {
}
timers.put(dfsCluster, timer);
}
- timer.schedule(new TimerTask() {
+ timer.schedule(
+ new TimerTask() {
- @Override
- public void run() {
- NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
- }
- }, rnd);
+ @Override
+ public void run() {
+ NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
+ }
+ },
+ rnd);
} else if (haTesting && rndMode == 2) {
int rnd = SolrTestCaseJ4.random().nextInt(30000);
Timer timer = new Timer();
@@ -242,25 +250,29 @@ public class HdfsTestUtil {
}
timers.put(dfsCluster, timer);
}
- timer.schedule(new TimerTask() {
-
- @Override
- public void run() {
- // TODO: randomly transition to standby
-// try {
-// dfsCluster.transitionToStandby(0);
-// dfsCluster.transitionToActive(1);
-// } catch (IOException e) {
-// throw new RuntimeException();
-// }
+ timer.schedule(
+ new TimerTask() {
+
+ @Override
+ public void run() {
+ // TODO: randomly transition to standby
+ // try {
+ // dfsCluster.transitionToStandby(0);
+ // dfsCluster.transitionToActive(1);
+ // } catch (IOException e) {
+ // throw new RuntimeException();
+ // }
- }
- }, rnd);
- } else {
+ }
+ },
+ rnd);
+ } else {
// TODO: we could do much better at testing this
// force a lease recovery by creating a tlog file and not closing it
URI uri = dfsCluster.getURI();
- Path hdfsDirPath = new Path(uri.toString() + "/solr/collection1/core_node1/data/tlog/tlog.0000000000000000000");
+ Path hdfsDirPath =
+ new Path(
+ uri.toString() + "/solr/collection1/core_node1/data/tlog/tlog.0000000000000000000");
// tran log already being created testing
badTlogOutStreamFs = FileSystem.get(hdfsDirPath.toUri(), getClientConfiguration(dfsCluster));
badTlogOutStream = badTlogOutStreamFs.create(hdfsDirPath);
@@ -344,9 +356,9 @@ public class HdfsTestUtil {
// Clear "solr.hdfs." system properties
Enumeration<?> propertyNames = System.getProperties().propertyNames();
- while(propertyNames.hasMoreElements()) {
+ while (propertyNames.hasMoreElements()) {
String propertyName = String.valueOf(propertyNames.nextElement());
- if(propertyName.startsWith("solr.hdfs.")) {
+ if (propertyName.startsWith("solr.hdfs.")) {
System.clearProperty(propertyName);
}
}
@@ -357,16 +369,24 @@ public class HdfsTestUtil {
if (dataDir == null) {
return null;
}
- String dir = "/"
- + new File(dataDir).toString().replaceAll(":", "_")
- .replaceAll("/", "_").replaceAll(" ", "_");
+ String dir =
+ "/"
+ + new File(dataDir)
+ .toString()
+ .replaceAll(":", "_")
+ .replaceAll("/", "_")
+ .replaceAll(" ", "_");
return getURI(dfsCluster) + dir;
}
public static String getURI(MiniDFSCluster dfsCluster) {
if (dfsCluster.getNameNodeInfos().length > 1) {
- String logicalName = String.format(Locale.ENGLISH, LOGICAL_HOSTNAME, dfsCluster.getInstanceId()); // NOTE: hdfs uses default locale
+ String logicalName =
+ String.format(
+ Locale.ENGLISH,
+ LOGICAL_HOSTNAME,
+ dfsCluster.getInstanceId()); // NOTE: hdfs uses default locale
return "hdfs://" + logicalName;
} else {
URI uri = dfsCluster.getURI(0);
@@ -376,11 +396,11 @@ public class HdfsTestUtil {
/**
* By default in JDK9+, the ForkJoinWorkerThreadFactory does not give SecurityManager permissions
- * to threads that are created. This works around that with a custom thread factory.
- * See SOLR-9515 and HDFS-14251
- * Used in org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice
+ * to threads that are created. This works around that with a custom thread factory. See SOLR-9515
+ * and HDFS-14251 Used in org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice
*/
- public static class HDFSForkJoinThreadFactory implements ForkJoinPool.ForkJoinWorkerThreadFactory {
+ public static class HDFSForkJoinThreadFactory
+ implements ForkJoinPool.ForkJoinWorkerThreadFactory {
@Override
public ForkJoinWorkerThread newThread(ForkJoinPool pool) {
ForkJoinWorkerThread worker = new SecurityManagerWorkerThread(pool);
@@ -395,10 +415,7 @@ public class HdfsTestUtil {
}
}
- /**
- * Ensures that we don't try to initialize metrics and read files outside
- * the source tree.
- */
+ /** Ensures that we don't try to initialize metrics and read files outside the source tree. */
public static class FakeMetricsSystem extends MetricsSystemImpl {
@Override
public synchronized MetricsSystem init(String prefix) {
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
index 9fa195b..658fcee 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
@@ -32,14 +33,15 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsThreadLeakTest extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
@@ -62,9 +64,9 @@ public class HdfsThreadLeakTest extends SolrTestCaseJ4 {
String uri = HdfsTestUtil.getURI(dfsCluster);
Path path = new Path(uri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- try(FileSystem fs = FileSystem.get(path.toUri(), conf)) {
+ try (FileSystem fs = FileSystem.get(path.toUri(), conf)) {
Path testFile = new Path(uri + "/testfile");
- try(FSDataOutputStream out = fs.create(testFile)) {
+ try (FSDataOutputStream out = fs.create(testFile)) {
out.write(5);
out.hflush();
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
index d817866..1d688c0 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
@@ -16,28 +16,31 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractTlogReplayBufferedWhileIndexingTestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
-public class HdfsTlogReplayBufferedWhileIndexingTest extends AbstractTlogReplayBufferedWhileIndexingTestBase {
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+public class HdfsTlogReplayBufferedWhileIndexingTest
+ extends AbstractTlogReplayBufferedWhileIndexingTestBase {
private static MiniDFSCluster dfsCluster;
public HdfsTlogReplayBufferedWhileIndexingTest() throws Exception {
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
index 93fb63a..6fb35a0 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
@@ -16,35 +16,37 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.Nightly;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.cloud.AbstractUnloadDistributedZkTestBase;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsUnloadDistributedZkTest extends AbstractUnloadDistributedZkTestBase {
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -53,7 +55,7 @@ public class HdfsUnloadDistributedZkTest extends AbstractUnloadDistributedZkTest
dfsCluster = null;
}
}
-
+
@Override
protected String getDataDir(String dataDir) {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
index f92fdf4..18224af 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
@@ -16,14 +16,12 @@
*/
package org.apache.solr.cloud.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -32,9 +30,9 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Nightly;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -45,11 +43,11 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.HdfsDirectoryFactory;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.store.blockcache.BlockCache;
import org.apache.solr.store.blockcache.BlockDirectory;
import org.apache.solr.store.blockcache.BlockDirectoryCache;
import org.apache.solr.store.blockcache.Cache;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.util.RefCounted;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -57,22 +55,25 @@ import org.junit.Test;
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsWriteToMultipleCollectionsTest extends AbstractBasicDistributedZkTestBase {
private static final String ACOLLECTION = "acollection";
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
- schemaString = "schema15.xml"; // we need a string id
+ schemaString = "schema15.xml"; // we need a string id
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -82,18 +83,18 @@ public class HdfsWriteToMultipleCollectionsTest extends AbstractBasicDistributed
schemaString = null;
}
}
-
+
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
+
public HdfsWriteToMultipleCollectionsTest() {
super();
sliceCount = 1;
fixShardCount(3);
}
-
+
protected String getSolrXml() {
return "solr.xml";
}
@@ -115,17 +116,18 @@ public class HdfsWriteToMultipleCollectionsTest extends AbstractBasicDistributed
CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress());
client.setDefaultCollection(ACOLLECTION + i);
cloudClients.add(client);
- StoppableIndexingThread indexThread = new StoppableIndexingThread(null, client, "1", true, docCount, 1, true);
+ StoppableIndexingThread indexThread =
+ new StoppableIndexingThread(null, client, "1", true, docCount, 1, true);
threads.add(indexThread);
indexThread.start();
}
-
+
int addCnt = 0;
for (StoppableIndexingThread thread : threads) {
thread.join();
addCnt += thread.getNumAdds() - thread.getNumDeletes();
}
-
+
long collectionsCount = 0;
for (CloudSolrClient client : cloudClients) {
client.commit();
@@ -135,45 +137,46 @@ public class HdfsWriteToMultipleCollectionsTest extends AbstractBasicDistributed
IOUtils.close(cloudClients);
assertEquals(addCnt, collectionsCount);
-
+
BlockCache lastBlockCache = null;
// assert that we are using the block directory and that write and read caching are being used
for (JettySolrRunner jetty : jettys) {
CoreContainer cores = jetty.getCoreContainer();
Collection<SolrCore> solrCores = cores.getCores();
for (SolrCore core : solrCores) {
- if (core.getCoreDescriptor().getCloudDescriptor().getCollectionName()
+ if (core.getCoreDescriptor()
+ .getCloudDescriptor()
+ .getCollectionName()
.startsWith(ACOLLECTION)) {
DirectoryFactory factory = core.getDirectoryFactory();
- assertTrue("Found: " + core.getDirectoryFactory().getClass().getName(), factory instanceof HdfsDirectoryFactory);
+ assertTrue(
+ "Found: " + core.getDirectoryFactory().getClass().getName(),
+ factory instanceof HdfsDirectoryFactory);
Directory dir = factory.get(core.getDataDir(), null, null);
try {
long dataDirSize = factory.size(dir);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- FileSystem fileSystem = FileSystem.newInstance(
- new Path(core.getDataDir()).toUri(), conf);
- long size = fileSystem.getContentSummary(
- new Path(core.getDataDir())).getLength();
+ FileSystem fileSystem =
+ FileSystem.newInstance(new Path(core.getDataDir()).toUri(), conf);
+ long size = fileSystem.getContentSummary(new Path(core.getDataDir())).getLength();
assertEquals(size, dataDirSize);
} finally {
core.getDirectoryFactory().release(dir);
}
-
- RefCounted<IndexWriter> iwRef = core.getUpdateHandler()
- .getSolrCoreState().getIndexWriter(core);
+
+ RefCounted<IndexWriter> iwRef =
+ core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
try {
IndexWriter iw = iwRef.get();
NRTCachingDirectory directory = (NRTCachingDirectory) iw.getDirectory();
- BlockDirectory blockDirectory = (BlockDirectory) directory
- .getDelegate();
+ BlockDirectory blockDirectory = (BlockDirectory) directory.getDelegate();
assertTrue(blockDirectory.isBlockCacheReadEnabled());
// see SOLR-6424
assertFalse(blockDirectory.isBlockCacheWriteEnabled());
Cache cache = blockDirectory.getCache();
// we know it's a BlockDirectoryCache, but future proof
assertTrue(cache instanceof BlockDirectoryCache);
- BlockCache blockCache = ((BlockDirectoryCache) cache)
- .getBlockCache();
+ BlockCache blockCache = ((BlockDirectoryCache) cache).getBlockCache();
if (lastBlockCache != null) {
if (Boolean.getBoolean("solr.hdfs.blockcache.global")) {
assertEquals(lastBlockCache, blockCache);
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
index 2c4c09d..99b1dbe 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
@@ -18,15 +18,22 @@ package org.apache.solr.cloud.hdfs;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
@@ -47,34 +54,28 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.TimeUnit;
-
@Slow
@Nightly
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
private static final String DELETE_DATA_DIR_COLLECTION = "delete_data_dir";
private static MiniDFSCluster dfsCluster;
private boolean testRestartIntoSafeMode;
-
+
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -83,12 +84,12 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
dfsCluster = null;
}
}
-
+
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
+
public StressHdfsTest() {
super();
sliceCount = 1;
@@ -104,7 +105,7 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
@Override
public void test() throws Exception {
randomlyEnableAutoSoftCommit();
-
+
int cnt = random().nextInt(2) + 1;
for (int i = 0; i < cnt; i++) {
createAndDeleteCollection();
@@ -112,29 +113,31 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
if (testRestartIntoSafeMode) {
Timer timer = new Timer();
-
+
try {
createCollection(DELETE_DATA_DIR_COLLECTION, "conf1", 1, 1);
-
+
waitForRecoveriesToFinish(DELETE_DATA_DIR_COLLECTION, false);
jettys.get(0).stop();
-
+
// enter safe mode and restart a node
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
-
+
int rnd = random().nextInt(10000);
-
- timer.schedule(new TimerTask() {
-
- @Override
- public void run() {
- NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
- }
- }, rnd);
-
+
+ timer.schedule(
+ new TimerTask() {
+
+ @Override
+ public void run() {
+ NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
+ }
+ },
+ rnd);
+
jettys.get(0).start();
-
+
waitForRecoveriesToFinish(DELETE_DATA_DIR_COLLECTION, false);
} finally {
timer.cancel();
@@ -154,15 +157,16 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
rep = 2;
if (nShards == 0) nShards = 1;
}
-
+
createCollection(DELETE_DATA_DIR_COLLECTION, "conf1", nShards, rep);
waitForRecoveriesToFinish(DELETE_DATA_DIR_COLLECTION, false);
-
+
// data dirs should be in zk, SOLR-8913
ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
- final DocCollection docCollection = clusterState.getCollectionOrNull(DELETE_DATA_DIR_COLLECTION);
- assertNotNull("Could not find :"+DELETE_DATA_DIR_COLLECTION, docCollection);
+ final DocCollection docCollection =
+ clusterState.getCollectionOrNull(DELETE_DATA_DIR_COLLECTION);
+ assertNotNull("Could not find :" + DELETE_DATA_DIR_COLLECTION, docCollection);
Slice slice = docCollection.getSlice("shard1");
assertNotNull(docCollection.getSlices().toString(), slice);
Collection<Replica> replicas = slice.getReplicas();
@@ -170,20 +174,21 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
assertNotNull(replica.getProperties().toString(), replica.get("dataDir"));
assertNotNull(replica.getProperties().toString(), replica.get("ulogDir"));
}
-
+
cloudClient.setDefaultCollection(DELETE_DATA_DIR_COLLECTION);
cloudClient.getZkStateReader().forceUpdateCollection(DELETE_DATA_DIR_COLLECTION);
-
+
for (int i = 1; i < nShards + 1; i++) {
cloudClient.getZkStateReader().getLeaderRetry(DELETE_DATA_DIR_COLLECTION, "shard" + i, 30000);
}
-
+
// collect the data dirs
List<String> dataDirs = new ArrayList<>();
-
+
int i = 0;
for (SolrClient client : clients) {
- try (HttpSolrClient c = getHttpSolrClient(getBaseUrl(client) + "/" + DELETE_DATA_DIR_COLLECTION, 30000)) {
+ try (HttpSolrClient c =
+ getHttpSolrClient(getBaseUrl(client) + "/" + DELETE_DATA_DIR_COLLECTION, 30000)) {
int docCnt = random().nextInt(1000) + 1;
for (int j = 0; j < docCnt; j++) {
c.add(getDoc("id", i++, "txt_t", "just some random text for a doc"));
@@ -194,9 +199,9 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
} else {
c.commit(true, true, true);
}
-
- NamedList<Object> response = c.query(
- new SolrQuery().setRequestHandler("/admin/system")).getResponse();
+
+ NamedList<Object> response =
+ c.query(new SolrQuery().setRequestHandler("/admin/system")).getResponse();
@SuppressWarnings({"unchecked"})
NamedList<Object> coreInfo = (NamedList<Object>) response.get("core");
@SuppressWarnings({"unchecked"})
@@ -204,17 +209,17 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
dataDirs.add(dataDir);
}
}
-
+
if (random().nextBoolean()) {
cloudClient.deleteByQuery("*:*");
cloudClient.commit();
-
+
assertEquals(0, cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound());
}
-
+
cloudClient.commit();
cloudClient.query(new SolrQuery("*:*"));
-
+
// delete collection
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.DELETE.toString());
@@ -224,18 +229,21 @@ public class StressHdfsTest extends AbstractBasicDistributedZkTestBase {
cloudClient.request(request);
final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while (cloudClient.getZkStateReader().getClusterState().hasCollection(DELETE_DATA_DIR_COLLECTION)) {
+ while (cloudClient
+ .getZkStateReader()
+ .getClusterState()
+ .hasCollection(DELETE_DATA_DIR_COLLECTION)) {
if (timeout.hasTimedOut()) {
throw new AssertionError("Timeout waiting to see removed collection leave clusterstate");
}
-
+
Thread.sleep(200);
}
-
+
// check that all dirs are gone
for (String dataDir : dataDirs) {
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- try(FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf)) {
+ try (FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf)) {
assertFalse(
"Data directory exists after collection removal : " + dataDir,
fs.exists(new Path(dataDir)));
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java b/solr/modules/hdfs/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
index 9b49b46..e523f19 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
@@ -16,7 +16,9 @@
*/
package org.apache.solr.core;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
@@ -27,8 +29,6 @@ import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
-
-import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.store.Directory;
@@ -42,32 +42,32 @@ import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.handler.SnapShooter;
+import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.store.hdfs.HdfsLocalityReporter;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.util.MockCoreContainer.MockCoreDescriptor;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-@ThreadLeakFilters(filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
-
+
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
}
-
+
@AfterClass
public static void teardownClass() throws Exception {
try {
@@ -85,9 +85,10 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
@Test
@SuppressWarnings({"try"})
public void testInitArgsOrSysPropConfig() throws Exception {
- try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
+ try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
// test sys prop config
- System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(
+ HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
@@ -104,7 +105,8 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
assertTrue(dataHome.endsWith("/solr2/mock/data"));
// test sys prop and init args config - init args wins
- System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(
+ HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(nl);
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
@@ -117,8 +119,11 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
- try (Directory dir = hdfsFactory
- .create(HdfsTestUtil.getURI(dfsCluster) + "/solr", NoLockFactory.INSTANCE, DirContext.DEFAULT)) {
+ try (Directory dir =
+ hdfsFactory.create(
+ HdfsTestUtil.getURI(dfsCluster) + "/solr",
+ NoLockFactory.INSTANCE,
+ DirContext.DEFAULT)) {
assertEquals(confDir.toString(), hdfsFactory.getConfDir());
}
@@ -131,20 +136,23 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
hdfsFactory.init(nl);
- assertEquals(4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertEquals(
+ 4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
assertTrue(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
nl = new NamedList<>();
hdfsFactory.init(nl);
System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
- assertEquals(3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertEquals(
+ 3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
assertTrue(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
- assertEquals(0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertEquals(
+ 0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
assertFalse(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
}
}
@@ -152,26 +160,31 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
@Test
public void testCleanupOldIndexDirectories() throws Exception {
try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
- System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(
+ HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr1/mock/data"));
System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
- try(FileSystem hdfs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
+ try (FileSystem hdfs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
org.apache.hadoop.fs.Path dataHomePath = new org.apache.hadoop.fs.Path(dataHome);
- org.apache.hadoop.fs.Path currentIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index");
- assertFalse(checkHdfsDirectory(hdfs,currentIndexDirPath));
+ org.apache.hadoop.fs.Path currentIndexDirPath =
+ new org.apache.hadoop.fs.Path(dataHomePath, "index");
+ assertFalse(checkHdfsDirectory(hdfs, currentIndexDirPath));
hdfs.mkdirs(currentIndexDirPath);
assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath));
- String timestamp1 = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());
- org.apache.hadoop.fs.Path oldIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1);
- assertFalse(checkHdfsDirectory(hdfs,oldIndexDirPath));
+ String timestamp1 =
+ new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());
+ org.apache.hadoop.fs.Path oldIndexDirPath =
+ new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1);
+ assertFalse(checkHdfsDirectory(hdfs, oldIndexDirPath));
hdfs.mkdirs(oldIndexDirPath);
assertTrue(checkHdfsDirectory(hdfs, oldIndexDirPath));
- hdfsFactory.cleanupOldIndexDirectories(dataHomePath.toString(), currentIndexDirPath.toString(), false);
+ hdfsFactory.cleanupOldIndexDirectories(
+ dataHomePath.toString(), currentIndexDirPath.toString(), false);
assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath));
assertFalse(checkHdfsDirectory(hdfs, oldIndexDirPath));
@@ -179,18 +192,19 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
}
}
- private boolean checkHdfsDirectory(FileSystem hdfs, org.apache.hadoop.fs.Path path) throws IOException {
+ private boolean checkHdfsDirectory(FileSystem hdfs, org.apache.hadoop.fs.Path path)
+ throws IOException {
try {
return hdfs.getFileStatus(path).isDirectory();
} catch (FileNotFoundException e) {
return false;
}
}
-
+
@Test
public void testLocalityReporter() throws Exception {
Random r = random();
- try(HdfsDirectoryFactory factory = new HdfsDirectoryFactory()) {
+ try (HdfsDirectoryFactory factory = new HdfsDirectoryFactory()) {
SolrMetricManager metricManager = new SolrMetricManager();
String registry = TestUtil.randomSimpleString(r, 2, 10);
String scope = TestUtil.randomSimpleString(r, 2, 10);
@@ -203,13 +217,25 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
factory.initializeMetrics(new SolrMetricsContext(metricManager, registry, "foo"), scope);
// get the metrics map for the locality bean
- MetricsMap metrics = (MetricsMap) ((SolrMetricManager.GaugeWrapper) metricManager.registry(registry).getMetrics().get("OTHER." + scope + ".hdfsLocality")).getGauge();
+ MetricsMap metrics =
+ (MetricsMap)
+ ((SolrMetricManager.GaugeWrapper)
+ metricManager
+ .registry(registry)
+ .getMetrics()
+ .get("OTHER." + scope + ".hdfsLocality"))
+ .getGauge();
// We haven't done anything, so there should be no data
Map<String, Object> statistics = metrics.getValue();
- assertEquals("Saw bytes that were not written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL), 0L,
+ assertEquals(
+ "Saw bytes that were not written: "
+ + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL),
+ 0L,
statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
assertEquals(
- "Counted bytes as local when none written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO), 0,
+ "Counted bytes as local when none written: "
+ + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO),
+ 0,
statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO));
// create a directory and a file
@@ -224,13 +250,21 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
// no locality because hostname not set
factory.setHost("bogus");
statistics = metrics.getValue();
- assertEquals("Wrong number of total bytes counted: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL),
- long_bytes, statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
- assertEquals("Wrong number of total blocks counted: " + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL),
- 1, statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL));
assertEquals(
- "Counted block as local when bad hostname set: " + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL),
- 0, statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL));
+ "Wrong number of total bytes counted: "
+ + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL),
+ long_bytes,
+ statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
+ assertEquals(
+ "Wrong number of total blocks counted: "
+ + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL),
+ 1,
+ statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL));
+ assertEquals(
+ "Counted block as local when bad hostname set: "
+ + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL),
+ 0,
+ statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL));
// set hostname and check again
factory.setHost("127.0.0.1");
@@ -238,7 +272,8 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
assertEquals(
"Did not count block as local after setting hostname: "
+ statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL),
- long_bytes, statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL));
+ long_bytes,
+ statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL));
}
}
}
@@ -246,11 +281,9 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
@Test
public void testIsAbsolute() throws Exception {
try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
- String pathToRepeat = RandomStrings.randomAsciiAlphanumOfLength(
- random(),
- random().nextInt(10) + 1
- ) + '/';
- String relativePath =pathToRepeat.repeat(random().nextInt(5) + 1);
+ String pathToRepeat =
+ RandomStrings.randomAsciiAlphanumOfLength(random(), random().nextInt(10) + 1) + '/';
+ String relativePath = pathToRepeat.repeat(random().nextInt(5) + 1);
assertFalse(hdfsFactory.isAbsolute(relativePath));
assertFalse(hdfsFactory.isAbsolute("/" + relativePath));
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryIntegrationTest.java b/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryIntegrationTest.java
index cd81902..e7f27b9 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryIntegrationTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryIntegrationTest.java
@@ -17,12 +17,11 @@
package org.apache.solr.core.backup.repository;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -38,78 +37,84 @@ import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-@LuceneTestCase.SuppressCodecs({"SimpleText"}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakFilters(defaultFilters = true, filters = {
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@LuceneTestCase.SuppressCodecs({
+ "SimpleText"
+}) // Backups do checksum validation against a footer value not present in 'SimpleText'
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class HdfsBackupRepositoryIntegrationTest extends AbstractBackupRepositoryTest {
- private static MiniDFSCluster dfsCluster;
- private static String hdfsUri;
- private static FileSystem fs;
+ private static MiniDFSCluster dfsCluster;
+ private static String hdfsUri;
+ private static FileSystem fs;
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- hdfsUri = HdfsTestUtil.getURI(dfsCluster);
- try {
- URI uri = new URI(hdfsUri);
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- fs = FileSystem.get(uri, conf);
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
+ hdfsUri = HdfsTestUtil.getURI(dfsCluster);
+ try {
+ URI uri = new URI(hdfsUri);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ fs = FileSystem.get(uri, conf);
- if (fs instanceof DistributedFileSystem) {
- // Make sure dfs is not in safe mode
- while (((DistributedFileSystem) fs).setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- Thread.interrupted();
- // continue
- }
- }
- }
-
- fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
- } catch (IOException | URISyntaxException e) {
- throw new RuntimeException(e);
+ if (fs instanceof DistributedFileSystem) {
+ // Make sure dfs is not in safe mode
+ while (((DistributedFileSystem) fs)
+ .setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ // continue
+ }
}
+ }
- System.setProperty("solr.hdfs.default.backup.path", "/backup");
- System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
- useFactory("solr.StandardDirectoryFactory");
+ fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
+ } catch (IOException | URISyntaxException e) {
+ throw new RuntimeException(e);
}
- @AfterClass
- public static void teardownClass() throws Exception {
- IOUtils.closeQuietly(fs);
- fs = null;
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- }
- }
+ System.setProperty("solr.hdfs.default.backup.path", "/backup");
+ System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
+ useFactory("solr.StandardDirectoryFactory");
+ }
- @Override
- protected BackupRepository getRepository() {
- HdfsBackupRepository repository = new HdfsBackupRepository();
- repository.init(getBaseBackupRepositoryConfiguration());
- return repository;
+ @AfterClass
+ public static void teardownClass() throws Exception {
+ IOUtils.closeQuietly(fs);
+ fs = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.default.backup.path");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
}
+ }
- @Override
- protected URI getBaseUri() throws URISyntaxException {
- return new URI(hdfsUri+"/solr/tmp");
- }
+ @Override
+ protected BackupRepository getRepository() {
+ HdfsBackupRepository repository = new HdfsBackupRepository();
+ repository.init(getBaseBackupRepositoryConfiguration());
+ return repository;
+ }
- @Override
- protected NamedList<Object> getBaseBackupRepositoryConfiguration() {
- NamedList<Object> config = new NamedList<>();
- config.add(HdfsDirectoryFactory.HDFS_HOME, hdfsUri + "/solr");
- return config;
- }
+ @Override
+ protected URI getBaseUri() throws URISyntaxException {
+ return new URI(hdfsUri + "/solr/tmp");
+ }
+
+ @Override
+ protected NamedList<Object> getBaseBackupRepositoryConfiguration() {
+ NamedList<Object> config = new NamedList<>();
+ config.add(HdfsDirectoryFactory.HDFS_HOME, hdfsUri + "/solr");
+ return config;
+ }
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryTest.java b/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryTest.java
index 398fb3b..dbc124e 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryTest.java
@@ -17,21 +17,20 @@
package org.apache.solr.core.backup.repository;
-import java.io.IOException;
+import static org.junit.Assert.assertEquals;
+import java.io.IOException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.HdfsDirectoryFactory;
import org.apache.solr.store.hdfs.HdfsDirectory;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
public class HdfsBackupRepositoryTest {
@Test(expected = NullPointerException.class)
public void testHdfsHomePropertyMissing() throws IOException {
- try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
+ try (HdfsBackupRepository hdfsBackupRepository = new HdfsBackupRepository()) {
NamedList<Object> namedList = new SimpleOrderedMap<>();
hdfsBackupRepository.init(namedList);
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/modules/hdfs/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
index 7604151..f079ef3 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
@@ -18,6 +18,12 @@ package org.apache.solr.handler;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -52,55 +58,54 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
-
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
-@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
-@LuceneTestCase.SuppressCodecs({"SimpleText"}) // Backups do checksum validation against a footer value not present in 'SimpleText'
-@ThreadLeakLingering(linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
+@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
+@LuceneTestCase.SuppressCodecs({
+ "SimpleText"
+}) // Backups do checksum validation against a footer value not present in 'SimpleText'
+@ThreadLeakLingering(
+ linger = 1000) // Wait at least 1 second for Netty GlobalEventExecutor to shutdown
public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
- public static final String HDFS_REPO_SOLR_XML = "<solr>\n" +
- "\n" +
- " <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
- " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
- " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +
- "\n" +
- " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n" +
- " <str name=\"urlScheme\">${urlScheme:}</str>\n" +
- " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
- " <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
- " </shardHandlerFactory>\n" +
- "\n" +
- " <solrcloud>\n" +
- " <str name=\"host\">127.0.0.1</str>\n" +
- " <int name=\"hostPort\">${hostPort:8983}</int>\n" +
- " <str name=\"hostContext\">${hostContext:solr}</str>\n" +
- " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +
- " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +
- " <int name=\"leaderVoteWait\">10000</int>\n" +
- " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +
- " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
- " </solrcloud>\n" +
- " \n" +
- " <backup>\n" +
- " <repository name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n" +
- " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n" +
- " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n" +
- " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n" +
- " <str name=\"solr.hdfs.permissions.umask-mode\">${solr.hdfs.permissions.umask-mode:000}</str>\n" +
- " </repository>\n" +
- " </backup>\n" +
- " \n" +
- "</solr>\n";
+ public static final String HDFS_REPO_SOLR_XML =
+ "<solr>\n"
+ + "\n"
+ + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"
+ + " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"
+ + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
+ + "\n"
+ + " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n"
+ + " <str name=\"urlScheme\">${urlScheme:}</str>\n"
+ + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"
+ + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"
+ + " </shardHandlerFactory>\n"
+ + "\n"
+ + " <solrcloud>\n"
+ + " <str name=\"host\">127.0.0.1</str>\n"
+ + " <int name=\"hostPort\">${hostPort:8983}</int>\n"
+ + " <str name=\"hostContext\">${hostContext:solr}</str>\n"
+ + " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"
+ + " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"
+ + " <int name=\"leaderVoteWait\">10000</int>\n"
+ + " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
+ + " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
+ + " </solrcloud>\n"
+ + " \n"
+ + " <backup>\n"
+ + " <repository name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n"
+ + " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n"
+ + " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n"
+ + " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n"
+ + " <str name=\"solr.hdfs.permissions.umask-mode\">${solr.hdfs.permissions.umask-mode:000}</str>\n"
+ + " </repository>\n"
+ + " </backup>\n"
+ + " \n"
+ + "</solr>\n";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static MiniDFSCluster dfsCluster;
@@ -139,11 +144,12 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
useFactory("solr.StandardDirectoryFactory");
- configureCluster(1)// nodes
- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
- .withSolrXml(HDFS_REPO_SOLR_XML)
- .configure();
-
+ configureCluster(1) // nodes
+ .addConfig(
+ "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .withSolrXml(HDFS_REPO_SOLR_XML)
+ .configure();
+
docsSeed = random().nextLong();
}
@@ -176,7 +182,8 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
int nDocs = BackupRestoreUtils.indexDocs(solrClient, collectionName, docsSeed);
- DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
+ DocCollection collectionState =
+ solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
assertEquals(1, collectionState.getActiveSlices().size());
Slice shard = collectionState.getActiveSlices().iterator().next();
assertEquals(1, shard.getReplicas().size());
@@ -194,39 +201,42 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
// Create a backup.
if (testViaReplicationHandler) {
log.info("Running Backup via replication handler");
- BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
- final BackupStatusChecker backupStatus
- = new BackupStatusChecker(leaderClient, "/" + coreName + "/replication");
+ BackupRestoreUtils.runReplicationHandlerCommand(
+ baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
+ final BackupStatusChecker backupStatus =
+ new BackupStatusChecker(leaderClient, "/" + coreName + "/replication");
backupStatus.waitForBackupSuccess(backupName, 30);
} else {
log.info("Running Backup via core admin api");
- Map<String,String> params = new HashMap<>();
+ Map<String, String> params = new HashMap<>();
params.put("name", backupName);
params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
params.put(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId);
- BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
+ BackupRestoreUtils.runCoreAdminCommand(
+ replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
}
int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
- for (int attempts=0; attempts<numRestoreTests; attempts++) {
- //Modify existing index before we call restore.
+ for (int attempts = 0; attempts < numRestoreTests; attempts++) {
+ // Modify existing index before we call restore.
if (nDocs > 0) {
- //Delete a few docs
+ // Delete a few docs
int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
- for(int i=0; i<numDeletes; i++) {
+ for (int i = 0; i < numDeletes; i++) {
leaderClient.deleteByQuery(collectionName, "id:" + i);
}
leaderClient.commit(collectionName);
- //Add a few more
+ // Add a few more
int moreAdds = TestUtil.nextInt(random(), 1, 100);
- for (int i=0; i<moreAdds; i++) {
+ for (int i = 0; i < moreAdds; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", i + nDocs);
doc.addField("name", "name = " + (i + nDocs));
leaderClient.add(collectionName, doc);
}
- //Purposely not calling commit once in a while. There can be some docs which are not committed
+ // Purposely not calling commit once in a while. There can be some docs which are not
+ // committed
if (usually()) {
leaderClient.commit(collectionName);
}
@@ -235,26 +245,30 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
if (testViaReplicationHandler) {
log.info("Running Restore via replication handler");
// Snapshooter prefixes "snapshot." to the backup name.
- BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName);
+ BackupRestoreUtils.runReplicationHandlerCommand(
+ baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName);
while (!TestRestoreCoreUtil.fetchRestoreStatus(baseUrl, coreName)) {
Thread.sleep(1000);
}
} else {
log.info("Running Restore via core admin api");
- Map<String,String> params = new HashMap<>();
+ Map<String, String> params = new HashMap<>();
params.put("name", "snapshot." + backupName);
params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
params.put(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId);
- BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
+ BackupRestoreUtils.runCoreAdminCommand(
+ replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
}
- //See if restore was successful by checking if all the docs are present again
+ // See if restore was successful by checking if all the docs are present again
BackupRestoreUtils.verifyDocs(nDocs, leaderClient, coreName);
// Verify the permissions on the backup folder.
- final String backupPath = (testViaReplicationHandler) ?
- "/backup/snapshot."+ backupName :
- "/backup/shard_backup_metadata";
- final FsAction expectedPerms = (testViaReplicationHandler) ? FsAction.ALL : FsAction.READ_EXECUTE;
+ final String backupPath =
+ (testViaReplicationHandler)
+ ? "/backup/snapshot." + backupName
+ : "/backup/shard_backup_metadata";
+ final FsAction expectedPerms =
+ (testViaReplicationHandler) ? FsAction.ALL : FsAction.READ_EXECUTE;
FileStatus status = fs.getFileStatus(new org.apache.hadoop.fs.Path(backupPath));
FsPermission perm = status.getPermission();
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/util/BadHdfsThreadsFilter.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/util/BadHdfsThreadsFilter.java
index 495f226..b13ec19 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/util/BadHdfsThreadsFilter.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/util/BadHdfsThreadsFilter.java
@@ -25,7 +25,8 @@ public class BadHdfsThreadsFilter implements ThreadFilter {
String name = t.getName();
if (name.startsWith("IPC Parameter Sending Thread ")) { // SOLR-5007
return true;
- } if (name.startsWith("IPC Client")) { // SOLR-5007
+ }
+ if (name.startsWith("IPC Client")) { // SOLR-5007
return true;
} else if (name.startsWith("org.apache.hadoop.hdfs.PeerCache")) { // SOLR-7288
return true;
@@ -42,7 +43,7 @@ public class BadHdfsThreadsFilter implements ThreadFilter {
} else if (name.startsWith("nioEventLoopGroup")) {
return true; // netty threads waiting for a web server stop confirm that won't happen
}
-
+
return false;
}
}
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/modules/hdfs/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
index 140f864..13fce0f 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
@@ -16,8 +16,8 @@
*/
package org.apache.solr.index.hdfs;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -32,8 +32,8 @@ import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.store.hdfs.HdfsDirectory;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
+import org.apache.solr.store.hdfs.HdfsDirectory;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -41,13 +41,13 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
@SolrTestCaseJ4.SuppressSSL
public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
private static MiniDFSCluster dfsCluster;
@@ -97,7 +97,7 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
directory.close();
}
} finally {
- try(FileSystem fs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
+ try (FileSystem fs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
fs.delete(path, true);
} finally {
super.tearDown();
@@ -122,7 +122,8 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
String[] args;
{
SolrClient client = clients.get(0);
- NamedList<Object> response = client.query(new SolrQuery().setRequestHandler("/admin/system")).getResponse();
+ NamedList<Object> response =
+ client.query(new SolrQuery().setRequestHandler("/admin/system")).getResponse();
@SuppressWarnings({"unchecked"})
NamedList<Object> coreInfo = (NamedList<Object>) response.get("core");
@SuppressWarnings({"unchecked"})
diff --git a/solr/modules/hdfs/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/modules/hdfs/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index 9c50eff..15999b4 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -16,6 +16,9 @@
*/
package org.apache.solr.search;
+import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -30,8 +33,6 @@ import java.util.Map;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -46,12 +47,12 @@ import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.HdfsUpdateLog;
import org.apache.solr.update.UpdateHandler;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;
-import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.junit.After;
@@ -60,19 +61,20 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class,
- QuickPatchThreadsFilter.class,
- BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
-})
+@ThreadLeakFilters(
+ defaultFilters = true,
+ filters = {
+ SolrIgnoredThreadsFilter.class,
+ QuickPatchThreadsFilter.class,
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+ })
// TODO: longer term this should be combined with TestRecovery somehow ??
public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// means that we've seen the leader and have version info (i.e. we are a non-leader replica)
private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();
- // acquire timeout in seconds. change this to a huge number when debugging to prevent threads from advancing.
+ // acquire timeout in seconds. change this to a huge number when debugging to prevent threads
+ // from advancing.
private static final int TIMEOUT = 60;
private static MiniDFSCluster dfsCluster;
@@ -99,7 +101,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");
- initCore("solrconfig-tlog.xml","schema15.xml");
+ initCore("solrconfig-tlog.xml", "schema15.xml");
}
@AfterClass
@@ -134,11 +136,11 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
String[] logList = ulog.getLogList(new Path(ulog.getLogDir()));
boolean foundRep2 = false;
for (String tl : logList) {
- FileStatus status = fs.getFileStatus(new Path(ulog.getLogDir(), tl));
- if (status.getReplication() == 2) {
- foundRep2 = true;
- break;
- }
+ FileStatus status = fs.getFileStatus(new Path(ulog.getLogDir(), tl));
+ if (status.getReplication() == 2) {
+ foundRep2 = true;
+ break;
+ }
}
assertTrue("Expected to find tlogs with a replication factor of 2", foundRep2);
@@ -151,13 +153,14 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
- UpdateLog.testing_logReplayHook = () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
+ UpdateLog.testing_logReplayHook =
+ () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
@@ -171,9 +174,9 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
versions.addFirst(deleteByQueryAndGetVersion("id:A11", null));
versions.addFirst(addAndGetVersion(sdoc("id", "A13"), null));
- assertJQ(req("q","*:*"),"/response/numFound==0");
+ assertJQ(req("q", "*:*"), "/response/numFound==0");
- assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
+ assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
h.close();
createCore();
@@ -182,31 +185,31 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// verify that previous close didn't do a commit
// recovery should be blocked by our hook
- assertJQ(req("q","*:*") ,"/response/numFound==0");
+ assertJQ(req("q", "*:*"), "/response/numFound==0");
// make sure we can still access versions after a restart
- assertJQ(req("qt","/get", "getVersions",""+versions.size()),"/versions==" + versions);
+ assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
// unblock recovery
logReplay.release(1000);
// make sure we can still access versions during recovery
- assertJQ(req("qt","/get", "getVersions",""+versions.size()),"/versions==" + versions);
+ assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
// wait until recovery has finished
assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q", "*:*"), "/response/numFound==3");
// make sure we can still access versions after recovery
- assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
+ assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);
- assertU(adoc("id","A2"));
- assertU(adoc("id","A3"));
+ assertU(adoc("id", "A2"));
+ assertU(adoc("id", "A3"));
assertU(delI("A2"));
- assertU(adoc("id","A4"));
+ assertU(adoc("id", "A4"));
- assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q", "*:*"), "/response/numFound==3");
h.close();
createCore();
@@ -215,8 +218,8 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// wait until recovery has finished
assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- assertJQ(req("q","*:*") ,"/response/numFound==5");
- assertJQ(req("q","id:A2") ,"/response/numFound==0");
+ assertJQ(req("q", "*:*"), "/response/numFound==5");
+ assertJQ(req("q", "id:A2"), "/response/numFound==0");
// no updates, so insure that recovery does not run
h.close();
@@ -225,11 +228,13 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// Solr should kick this off now
// h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
- assertJQ(req("q","*:*") ,"/response/numFound==5");
+ assertJQ(req("q", "*:*"), "/response/numFound==5");
Thread.sleep(100);
- assertEquals(permits, logReplay.availablePermits()); // no updates, so insure that recovery didn't run
+ assertEquals(
+ permits, logReplay.availablePermits()); // no updates, so insure that recovery didn't run
- assertEquals(UpdateLog.State.ACTIVE, h.getCore().getUpdateHandler().getUpdateLog().getState());
+ assertEquals(
+ UpdateLog.State.ACTIVE, h.getCore().getUpdateHandler().getUpdateLog().getState());
} finally {
UpdateLog.testing_logReplayHook = null;
@@ -243,13 +248,14 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
- UpdateLog.testing_logReplayHook = () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
+ UpdateLog.testing_logReplayHook =
+ () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
@@ -272,35 +278,40 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
// simulate updates from a leader
- updateJ(jsonAdd(sdoc("id","B1", "_version_","1010")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","B11", "_version_","1015")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonDelQ("id:B1 id:B11 id:B2 id:B3"), params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_","-1017"));
- updateJ(jsonAdd(sdoc("id","B2", "_version_","1020")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","B3", "_version_","1030")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_","-2010"));
-
- assertJQ(req("qt","/get", "getVersions","6")
- ,"=={'versions':[-2010,1030,1020,-1017,1015,1010]}"
- );
+ updateJ(
+ jsonAdd(sdoc("id", "B1", "_version_", "1010")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "B11", "_version_", "1015")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonDelQ("id:B1 id:B11 id:B2 id:B3"),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-1017"));
+ updateJ(
+ jsonAdd(sdoc("id", "B2", "_version_", "1020")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "B3", "_version_", "1030")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2010"));
+
+ assertJQ(
+ req("qt", "/get", "getVersions", "6"),
+ "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
assertU(commit());
- assertJQ(req("qt","/get", "getVersions","6")
- ,"=={'versions':[-2010,1030,1020,-1017,1015,1010]}"
- );
+ assertJQ(
+ req("qt", "/get", "getVersions", "6"),
+ "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
// updates should be buffered, so we should not see any results yet.
- assertJQ(req("q", "*:*")
- , "/response/numFound==0"
- );
+ assertJQ(req("q", "*:*"), "/response/numFound==0");
// real-time get should also not show anything (this could change in the future,
// but it's currently used for validating version numbers too, so it would
// be bad for updates to be visible if we're just buffering.
- assertJQ(req("qt","/get", "id","B3")
- ,"=={'doc':null}"
- );
-
+ assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}");
rinfoFuture = ulog.applyBufferedUpdates();
assertNotNull(rinfoFuture);
@@ -312,40 +323,57 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
+ assertJQ(
+ req("qt", "/get", "getVersions", "6"),
+ "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
- assertJQ(req("qt","/get", "getVersions","6")
- ,"=={'versions':[-2010,1030,1020,-1017,1015,1010]}"
- );
-
-
- assertJQ(req("q", "*:*")
- , "/response/numFound==2"
- );
+ assertJQ(req("q", "*:*"), "/response/numFound==2");
// move back to recovering
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
- Long ver = getVer(req("qt","/get", "id","B3"));
+ Long ver = getVer(req("qt", "/get", "id", "B3"));
assertEquals(1030L, ver.longValue());
// add a reordered doc that shouldn't overwrite one in the index
- updateJ(jsonAdd(sdoc("id","B3", "_version_","3")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "B3", "_version_", "3")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
// reorder two buffered updates
- updateJ(jsonAdd(sdoc("id","B4", "_version_","1040")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- deleteAndGetVersion("B4", params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_","-940")); // this update should not take affect
- updateJ(jsonAdd(sdoc("id","B6", "_version_","1060")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","B5", "_version_","1050")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","B8", "_version_","1080")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
-
- // test that delete by query is at least buffered along with everything else so it will delete the
+ updateJ(
+ jsonAdd(sdoc("id", "B4", "_version_", "1040")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ deleteAndGetVersion(
+ "B4",
+ params(
+ DISTRIB_UPDATE_PARAM,
+ FROM_LEADER,
+ "_version_",
+ "-940")); // this update should not take affect
+ updateJ(
+ jsonAdd(sdoc("id", "B6", "_version_", "1060")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "B5", "_version_", "1050")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "B8", "_version_", "1080")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+
+ // test that delete by query is at least buffered along with everything else so it will delete
+ // the
// currently buffered id:8 (even if it doesn't currently support versioning)
- updateJ("{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}", params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_","-3000"));
+ updateJ(
+ "{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}",
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-3000"));
- assertJQ(req("qt","/get", "getVersions","13")
- ,"=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the "3" appears because versions aren't checked while buffering
- );
+ assertJQ(
+ req("qt", "/get", "getVersions", "13"),
+ "=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the
+ // "3"
+ // appears because versions aren't checked while buffering
+ );
logReplay.drainPermits();
rinfoFuture = ulog.applyBufferedUpdates();
@@ -356,28 +384,31 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
logReplay.release(1);
// now add another update
- updateJ(jsonAdd(sdoc("id","B7", "_version_","1070")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "B7", "_version_", "1070")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
// a reordered update that should be dropped
- deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_","-950"));
+ deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-950"));
- deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_","-2060"));
+ deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2060"));
logReplay.release(1000);
UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();
- assertJQ(req("q", "*:*", "sort","id asc", "fl","id,_version_")
- , "/response/docs==["
- + "{'id':'B3','_version_':1030}"
- + ",{'id':'B4','_version_':1040}"
- + ",{'id':'B5','_version_':1050}"
- + ",{'id':'B7','_version_':1070}"
- +"]"
- );
+ assertJQ(
+ req("q", "*:*", "sort", "id asc", "fl", "id,_version_"),
+ "/response/docs==["
+ + "{'id':'B3','_version_':1030}"
+ + ",{'id':'B4','_version_':1040}"
+ + ",{'id':'B5','_version_':1050}"
+ + ",{'id':'B7','_version_':1070}"
+ + "]");
assertEquals(1, recInfo.deleteByQuery);
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
+ assertEquals(
+ UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
} finally {
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
@@ -393,17 +424,17 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
- UpdateLog.testing_logReplayHook = () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
+ UpdateLog.testing_logReplayHook =
+ () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
SolrQueryRequest req = req();
UpdateHandler uhandler = req.getCore().getUpdateHandler();
UpdateLog ulog = uhandler.getUpdateLog();
@@ -423,83 +454,104 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
// simulate updates from a leader
- updateJ(jsonAdd(sdoc("id","C1", "_version_","101")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","C2", "_version_","102")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","C3", "_version_","103")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertTrue(ulog.dropBufferedUpdates());
ulog.bufferUpdates();
- updateJ(jsonAdd(sdoc("id", "C4", "_version_","104")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id", "C5", "_version_","105")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
logReplay.release(1000);
rinfoFuture = ulog.applyBufferedUpdates();
UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
assertEquals(2, rinfo.adds);
- assertJQ(req("qt","/get", "getVersions","2")
- ,"=={'versions':[105,104]}"
- );
+ assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[105,104]}");
// this time add some docs first before buffering starts (so tlog won't be at pos 0)
- updateJ(jsonAdd(sdoc("id","C100", "_version_","200")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","C101", "_version_","201")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C100", "_version_", "200")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C101", "_version_", "201")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
ulog.bufferUpdates();
- updateJ(jsonAdd(sdoc("id","C103", "_version_","203")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","C104", "_version_","204")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C103", "_version_", "203")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C104", "_version_", "204")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertTrue(ulog.dropBufferedUpdates());
ulog.bufferUpdates();
- updateJ(jsonAdd(sdoc("id","C105", "_version_","205")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","C106", "_version_","206")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C105", "_version_", "205")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C106", "_version_", "206")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
rinfoFuture = ulog.applyBufferedUpdates();
rinfo = rinfoFuture.get();
assertEquals(2, rinfo.adds);
- assertJQ(req("q", "*:*", "sort","_version_ asc", "fl","id,_version_")
- , "/response/docs==["
- + "{'id':'C4','_version_':104}"
- + ",{'id':'C5','_version_':105}"
- + ",{'id':'C100','_version_':200}"
- + ",{'id':'C101','_version_':201}"
- + ",{'id':'C105','_version_':205}"
- + ",{'id':'C106','_version_':206}"
- +"]"
- );
-
- assertJQ(req("qt","/get", "getVersions","6")
- ,"=={'versions':[206,205,201,200,105,104]}"
- );
+ assertJQ(
+ req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"),
+ "/response/docs==["
+ + "{'id':'C4','_version_':104}"
+ + ",{'id':'C5','_version_':105}"
+ + ",{'id':'C100','_version_':200}"
+ + ",{'id':'C101','_version_':201}"
+ + ",{'id':'C105','_version_':205}"
+ + ",{'id':'C106','_version_':206}"
+ + "]");
+
+ assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[206,205,201,200,105,104]}");
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
- updateJ(jsonAdd(sdoc("id","C301", "_version_","998")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","C302", "_version_","999")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C301", "_version_", "998")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C302", "_version_", "999")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertTrue(ulog.dropBufferedUpdates());
// make sure we can overwrite with a lower version
// TODO: is this functionality needed?
- updateJ(jsonAdd(sdoc("id","C301", "_version_","301")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","C302", "_version_","302")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C301", "_version_", "301")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C302", "_version_", "302")),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertU(commit());
- assertJQ(req("qt","/get", "getVersions","2")
- ,"=={'versions':[302,301]}"
- );
+ assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[302,301]}");
- assertJQ(req("q", "*:*", "sort","_version_ desc", "fl","id,_version_", "rows","2")
- , "/response/docs==["
- + "{'id':'C302','_version_':302}"
- + ",{'id':'C301','_version_':301}"
- +"]"
- );
+ assertJQ(
+ req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"),
+ "/response/docs==["
+ + "{'id':'C302','_version_':302}"
+ + ",{'id':'C301','_version_':301}"
+ + "]");
- updateJ(jsonAdd(sdoc("id","C2", "_version_","302")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "C2", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
+ assertEquals(
+ UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
} finally {
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
@@ -524,9 +576,12 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
ulog.bufferUpdates();
// simulate updates from a leader
- updateJ(jsonAdd(sdoc("id","Q1", "_version_","101")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","Q2", "_version_","102")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","Q3", "_version_","103")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "Q1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "Q2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "Q3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
req.close();
@@ -538,7 +593,8 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
ulog = uhandler.getUpdateLog();
// the core no longer replay updates from buffer tlog on startup
- assertTrue(ulog.existOldBufferLog()); // since we died while buffering, we should see this last
+ assertTrue(
+ ulog.existOldBufferLog()); // since we died while buffering, we should see this last
// buffer tlog won't be removed on restart
req.close();
@@ -554,7 +610,11 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
ulog.bufferUpdates();
// simulate receiving no updates
ulog.applyBufferedUpdates();
- updateJ(jsonAdd(sdoc("id","Q7", "_version_","117")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); // do another add to make sure flags are back to normal
+ updateJ(
+ jsonAdd(sdoc("id", "Q7", "_version_", "117")),
+ params(
+ DISTRIB_UPDATE_PARAM,
+ FROM_LEADER)); // do another add to make sure flags are back to normal
req.close();
h.close();
@@ -565,11 +625,13 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
ulog = uhandler.getUpdateLog();
assertFalse(ulog.existOldBufferLog());
- // Timeout for Q7 get replayed, because it was added on tlog, therefore it will be replayed on restart
+ // Timeout for Q7 get replayed, because it was added on tlog, therefore it will be replayed on
+ // restart
TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- timeout.waitFor("Timeout waiting for finish replay updates",
+ timeout.waitFor(
+ "Timeout waiting for finish replay updates",
() -> h.getCore().getUpdateHandler().getUpdateLog().getState() == UpdateLog.State.ACTIVE);
- assertJQ(req("qt","/get", "id", "Q7") ,"/doc/id==Q7");
+ assertJQ(req("qt", "/get", "id", "Q7"), "/doc/id==Q7");
} finally {
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
@@ -584,24 +646,22 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
clearIndex();
assertU(commit());
- assertU(adoc("id","D1", "val_i","1"));
- assertU(adoc("id","D2", "val_i","1"));
+ assertU(adoc("id", "D1", "val_i", "1"));
+ assertU(adoc("id", "D2", "val_i", "1"));
assertU(commit());
- long v1 = getVer(req("q","id:D1"));
- long v1a = getVer(req("q","id:D2"));
+ long v1 = getVer(req("q", "id:D1"));
+ long v1a = getVer(req("q", "id:D2"));
h.close();
createCore();
- assertU(adoc("id","D1", "val_i","2"));
+ assertU(adoc("id", "D1", "val_i", "2"));
assertU(commit());
- long v2 = getVer(req("q","id:D1"));
+ long v2 = getVer(req("q", "id:D1"));
- assert(v2 > v1);
+ assert (v2 > v1);
- assertJQ(req("qt","/get", "getVersions","2")
- ,"/versions==[" + v2 + "," + v1a + "]"
- );
+ assertJQ(req("qt", "/get", "getVersions", "2"), "/versions==[" + v2 + "," + v1a + "]");
}
// make sure that log isn't needlessly replayed after a clean close
@@ -610,13 +670,14 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
- UpdateLog.testing_logReplayHook = () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
+ UpdateLog.testing_logReplayHook =
+ () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
@@ -628,8 +689,8 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
clearIndex();
assertU(commit());
- assertU(adoc("id","E1", "val_i","1"));
- assertU(adoc("id","E2", "val_i","1"));
+ assertU(adoc("id", "E1", "val_i", "1"));
+ assertU(adoc("id", "E2", "val_i", "1"));
// set to a high enough number so this test won't hang on a bug
logReplay.release(10);
@@ -638,7 +699,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
createCore();
// make sure the docs got committed
- assertJQ(req("q","*:*"),"/response/numFound==2");
+ assertJQ(req("q", "*:*"), "/response/numFound==2");
// make sure no replay happened
assertEquals(10, logReplay.availablePermits());
@@ -652,8 +713,8 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
private void addDocs(int nDocs, int start, LinkedList<Long> versions) throws Exception {
- for (int i=0; i<nDocs; i++) {
- versions.addFirst( addAndGetVersion( sdoc("id",Integer.toString(start + nDocs)) , null) );
+ for (int i = 0; i < nDocs; i++) {
+ versions.addFirst(addAndGetVersion(sdoc("id", Integer.toString(start + nDocs)), null));
}
}
@@ -664,17 +725,17 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
- UpdateLog.testing_logReplayHook = () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
+ UpdateLog.testing_logReplayHook =
+ () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
clearIndex();
assertU(commit());
@@ -695,62 +756,95 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
int maxReq = 50;
LinkedList<Long> versions = new LinkedList<>();
- addDocs(10, start, versions); start+=10;
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ addDocs(10, start, versions);
+ start += 10;
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
-
- addDocs(10, start, versions); start+=10;
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
+
+ addDocs(10, start, versions);
+ start += 10;
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertEquals(2, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
- addDocs(105, start, versions); start+=105;
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ addDocs(105, start, versions);
+ start += 105;
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
// previous two logs should be gone now
assertEquals(1, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
- addDocs(1, start, versions); start+=1;
+ addDocs(1, start, versions);
+ start += 1;
h.close();
- createCore(); // trigger recovery, make sure that tlog reference handling is correct
+ createCore(); // trigger recovery, make sure that tlog reference handling is correct
// test we can get versions while replay is happening
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
logReplay.release(1000);
assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
- addDocs(105, start, versions); start+=105;
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ addDocs(105, start, versions);
+ start += 105;
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
assertU(commit());
- assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
// previous logs should be gone now
assertEquals(1, HdfsUpdateLog.getLogList(fs, new Path(logDir)).length);
//
- // test that a corrupt tlog file doesn't stop us from coming up, or seeing versions before that tlog file.
+ // test that a corrupt tlog file doesn't stop us from coming up, or seeing versions before
+ // that tlog file.
//
- addDocs(1, start, new LinkedList<Long>()); // don't add this to the versions list because we are going to lose it...
+ addDocs(
+ 1,
+ start,
+ new LinkedList<
+ Long>()); // don't add this to the versions list because we are going to lose it...
h.close();
files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
- FSDataOutputStream dos = fs.create(new Path(new Path(logDir), files[files.length-1]), (short)1);
+ FSDataOutputStream dos =
+ fs.create(new Path(new Path(logDir), files[files.length - 1]), (short) 1);
dos.writeUTF("This is a trashed log file that really shouldn't work at all, but we'll see..");
dos.close();
ignoreException("Failure to open existing");
createCore();
// we should still be able to get the list of versions (not including the trashed log file)
- assertJQ(req("qt", "/get", "getVersions", "" + maxReq), "/versions==" + versions.subList(0, Math.min(maxReq, start)));
+ assertJQ(
+ req("qt", "/get", "getVersions", "" + maxReq),
+ "/versions==" + versions.subList(0, Math.min(maxReq, start)));
resetExceptionIgnores();
} finally {
@@ -760,8 +854,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
//
- // test that a partially written last tlog entry (that will cause problems for both reverse reading and for
- // log replay) doesn't stop us from coming up, and from recovering the documents that were not cut off.
+ // test that a partially written last tlog entry (that will cause problems for both reverse
+ // reading and for
+ // log replay) doesn't stop us from coming up, and from recovering the documents that were not cut
+ // off.
//
@Test
public void testTruncatedLog() throws Exception {
@@ -770,13 +866,14 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
- UpdateLog.testing_logReplayHook = () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
+ UpdateLog.testing_logReplayHook =
+ () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
@@ -785,42 +882,47 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
clearIndex();
assertU(commit());
- assertU(adoc("id","F1"));
- assertU(adoc("id","F2"));
- assertU(adoc("id","F3"));
+ assertU(adoc("id", "F1"));
+ assertU(adoc("id", "F2"));
+ assertU(adoc("id", "F3"));
h.close();
-
-
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
- FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1]));
+ FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length - 1]));
dos.writeLong(0xffffffffffffffffL);
- dos.writeChars("This should be appended to a good log file, representing a bad partially written record.");
+ dos.writeChars(
+ "This should be appended to a good log file, representing a bad partially written record.");
dos.close();
logReplay.release(1000);
logReplayFinish.drainPermits();
- ignoreException("OutOfBoundsException"); // this is what the corrupted log currently produces... subject to change.
+ ignoreException(
+ "OutOfBoundsException"); // this is what the corrupted log currently produces... subject
+ // to change.
createCore();
assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
resetExceptionIgnores();
- assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q", "*:*"), "/response/numFound==3");
//
// Now test that the bad log file doesn't mess up retrieving latest versions
//
- updateJ(jsonAdd(sdoc("id","F4", "_version_","104")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","F5", "_version_","105")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","F6", "_version_","106")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "F4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "F5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "F6", "_version_", "106")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- // This currently skips the bad log file and also returns the version of the clearIndex (del *:*)
+ // This currently skips the bad log file and also returns the version of the clearIndex (del
+ // *:*)
// assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
- assertJQ(req("qt","/get", "getVersions","3"), "/versions==[106,105,104]");
+ assertJQ(req("qt", "/get", "getVersions", "3"), "/versions==[106,105,104]");
} finally {
UpdateLog.testing_logReplayHook = null;
@@ -841,45 +943,51 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
clearIndex();
assertU(commit());
- assertU(adoc("id","G1"));
- assertU(adoc("id","G2"));
- assertU(adoc("id","G3"));
+ assertU(adoc("id", "G1"));
+ assertU(adoc("id", "G2"));
+ assertU(adoc("id", "G3"));
h.close();
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
- FSDataOutputStream dos = fs.create(new Path(logDir, files[files.length-1]), (short)1);
- dos.write(new byte[800]); // zero out file
+ FSDataOutputStream dos = fs.create(new Path(logDir, files[files.length - 1]), (short) 1);
+ dos.write(new byte[800]); // zero out file
dos.close();
-
- ignoreException("Failure to open existing log file"); // this is what the corrupted log currently produces... subject to change.
+ ignoreException(
+ "Failure to open existing log file"); // this is what the corrupted log currently
+ // produces... subject to change.
createCore();
resetExceptionIgnores();
// just make sure it responds
- assertJQ(req("q","*:*") ,"/response/numFound==0");
+ assertJQ(req("q", "*:*"), "/response/numFound==0");
//
// Now test that the bad log file doesn't mess up retrieving latest versions
//
- updateJ(jsonAdd(sdoc("id","G4", "_version_","104")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","G5", "_version_","105")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
- updateJ(jsonAdd(sdoc("id","G6", "_version_","106")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "G4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "G5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
+ updateJ(
+ jsonAdd(sdoc("id", "G6", "_version_", "106")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
- // This currently skips the bad log file and also returns the version of the clearIndex (del *:*)
+ // This currently skips the bad log file and also returns the version of the clearIndex (del
+ // *:*)
// assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
- assertJQ(req("qt","/get", "getVersions","3"), "/versions==[106,105,104]");
+ assertJQ(req("qt", "/get", "getVersions", "3"), "/versions==[106,105,104]");
assertU(commit());
- assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q", "*:*"), "/response/numFound==3");
// This messes up some other tests (on windows) if we don't remove the bad log.
- // This *should* hopefully just be because the tests are too fragile and not because of real bugs - but it should be investigated further.
+ // This *should* hopefully just be because the tests are too fragile and not because of real
+ // bugs - but it should be investigated further.
deleteLogs();
} finally {
@@ -888,7 +996,8 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
}
- // in rare circumstances, two logs can be left uncapped (lacking a commit at the end signifying that all the content in the log was committed)
+ // in rare circumstances, two logs can be left uncapped (lacking a commit at the end signifying
+ // that all the content in the log was committed)
@Test
public void testRecoveryMultipleLogs() throws Exception {
try {
@@ -896,13 +1005,14 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
- UpdateLog.testing_logReplayHook = () -> {
- try {
- assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
+ UpdateLog.testing_logReplayHook =
+ () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
@@ -911,48 +1021,60 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
clearIndex();
assertU(commit());
- assertU(adoc("id","AAAAAA"));
- assertU(adoc("id","BBBBBB"));
- assertU(adoc("id","CCCCCC"));
+ assertU(adoc("id", "AAAAAA"));
+ assertU(adoc("id", "BBBBBB"));
+ assertU(adoc("id", "CCCCCC"));
h.close();
String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
- String fname = files[files.length-1];
+ String fname = files[files.length - 1];
- try(FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1]))) {
+ try (FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length - 1]))) {
dos.writeLong(0xffffffffffffffffL);
- dos.writeChars("This should be appended to a good log file, representing a bad partially written record.");
+ dos.writeChars(
+ "This should be appended to a good log file, representing a bad partially written record.");
}
- try(FSDataInputStream dis = fs.open(new Path(logDir, files[files.length-1]))) {
+ try (FSDataInputStream dis = fs.open(new Path(logDir, files[files.length - 1]))) {
byte[] content = new byte[dis.available()];
dis.readFully(content);
- // Now make a newer log file with just the IDs changed. NOTE: this may not work if log format changes too much!
- findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content);
- findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content);
- findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content);
+ // Now make a newer log file with just the IDs changed. NOTE: this may not work if log
+ // format changes too much!
+ findReplace(
+ "AAAAAA".getBytes(StandardCharsets.UTF_8),
+ "aaaaaa".getBytes(StandardCharsets.UTF_8),
+ content);
+ findReplace(
+ "BBBBBB".getBytes(StandardCharsets.UTF_8),
+ "bbbbbb".getBytes(StandardCharsets.UTF_8),
+ content);
+ findReplace(
+ "CCCCCC".getBytes(StandardCharsets.UTF_8),
+ "cccccc".getBytes(StandardCharsets.UTF_8),
+ content);
// WARNING... assumes format of .00000n where n is less than 9
long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1));
- String fname2 = String.format(Locale.ROOT,
- UpdateLog.LOG_FILENAME_PATTERN,
- UpdateLog.TLOG_NAME,
- logNumber + 1);
+ String fname2 =
+ String.format(
+ Locale.ROOT, UpdateLog.LOG_FILENAME_PATTERN, UpdateLog.TLOG_NAME, logNumber + 1);
- try(FSDataOutputStream dos = fs.create(new Path(logDir, fname2), (short)1)) {
+ try (FSDataOutputStream dos = fs.create(new Path(logDir, fname2), (short) 1)) {
dos.write(content);
}
}
logReplay.release(1000);
logReplayFinish.drainPermits();
- ignoreException("OutOfBoundsException"); // this is what the corrupted log currently produces... subject to change.
+ ignoreException(
+ "OutOfBoundsException"); // this is what the corrupted log currently produces... subject
+ // to change.
createCore();
assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
resetExceptionIgnores();
- assertJQ(req("q","*:*") ,"/response/numFound==6");
+ assertJQ(req("q", "*:*"), "/response/numFound==6");
} finally {
UpdateLog.testing_logReplayHook = null;
@@ -960,21 +1082,21 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
}
-
// NOTE: replacement must currently be same size
private static void findReplace(byte[] from, byte[] to, byte[] data) {
int idx = -from.length;
- for(;;) {
- idx = indexOf(from, data, idx + from.length); // skip over previous match
+ for (; ; ) {
+ idx = indexOf(from, data, idx + from.length); // skip over previous match
if (idx < 0) break;
System.arraycopy(to, 0, data, idx, to.length);
}
}
private static int indexOf(byte[] target, byte[] data, int start) {
- outer: for (int i=start; i<data.length - target.length; i++) {
- for (int j=0; j<target.length; j++) {
... 1147 lines suppressed ...