You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2018/08/22 14:18:32 UTC
[accumulo] branch master updated: Pass ServerContext to RFile.
Closes #584 (#609)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new d1a7fb8 Pass ServerContext to RFile. Closes #584 (#609)
d1a7fb8 is described below
commit d1a7fb8be2c3b0ffa7ec43aa779cbc5de43d20f4
Author: Mike Miller <mm...@apache.org>
AuthorDate: Wed Aug 22 10:18:30 2018 -0400
Pass ServerContext to RFile. Closes #584 (#609)
* This allows init of crypto in tserver, eliminating static instance in
CryptoServiceFactory
---
.../accumulo/core/client/rfile/RFileScanner.java | 2 +-
.../core/client/rfile/RFileSummariesRetriever.java | 2 +-
.../apache/accumulo/core/file/FileOperations.java | 28 ++++++---
.../apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../accumulo/core/file/rfile/RFileOperations.java | 8 ++-
.../accumulo/core/file/rfile/SplitLarge.java | 8 ++-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 4 +-
.../accumulo/core/file/rfile/bcfile/PrintInfo.java | 2 +-
.../core/security/crypto/CryptoServiceFactory.java | 51 ++--------------
.../org/apache/accumulo/core/summary/Gatherer.java | 14 ++---
.../accumulo/core/client/rfile/RFileTest.java | 4 +-
.../core/file/rfile/CreateCompatTestFile.java | 2 +-
.../core/file/rfile/MultiLevelIndexTest.java | 4 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 4 +-
.../apache/accumulo/core/file/rfile/RFileTest.java | 69 +---------------------
.../accumulo/core/security/crypto/CryptoTest.java | 32 ++--------
.../org/apache/accumulo/server/ServerContext.java | 22 +++++++
.../tserver/src/main/findbugs/exclude-filter.xml | 2 +-
.../org/apache/accumulo/tserver/FileManager.java | 3 +-
.../org/apache/accumulo/tserver/InMemoryMap.java | 15 ++++-
.../org/apache/accumulo/tserver/TabletServer.java | 13 ++--
.../tserver/TabletServerResourceManager.java | 9 ++-
.../tserver/compaction/MajorCompactionRequest.java | 14 +++--
.../org/apache/accumulo/tserver/log/DfsLogger.java | 4 +-
.../apache/accumulo/tserver/tablet/Compactor.java | 6 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 6 +-
.../apache/accumulo/tserver/tablet/TabletData.java | 2 +-
.../accumulo/tserver/tablet/TabletMemory.java | 13 +++-
.../apache/accumulo/tserver/InMemoryMapTest.java | 26 +++++---
.../compaction/DefaultCompactionStrategyTest.java | 12 +++-
.../SizeLimitCompactionStrategyTest.java | 4 +-
.../compaction/TwoTierCompactionStrategyTest.java | 10 +++-
.../ConfigurableCompactionStrategyTest.java | 5 +-
.../org/apache/accumulo/test/InMemoryMapIT.java | 21 +++++--
.../util/memory/InMemoryMapMemoryUsageCheck.java | 3 +-
35 files changed, 208 insertions(+), 218 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
index b073ada..e9c06cc 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
@@ -218,7 +218,7 @@ class RFileScanner extends ScannerOptions implements Scanner {
if (null == this.dataCache) {
this.dataCache = new NoopCache();
}
- this.cryptoService = CryptoServiceFactory.getConfigured(tableConf);
+ this.cryptoService = CryptoServiceFactory.newInstance(tableConf);
}
@Override
diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java
index 7ae9b7c..07d7aaa 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java
@@ -90,7 +90,7 @@ class RFileSummariesRetriever implements SummaryInputArguments, SummaryFSOptions
for (RFileSource source : sources) {
SummaryReader fileSummary = SummaryReader.load(conf, acuconf, source.getInputStream(),
source.getLength(), summarySelector, factory,
- CryptoServiceFactory.getConfigured(acuconf));
+ CryptoServiceFactory.newInstance(acuconf));
SummaryCollection sc = fileSummary
.getSummaries(Collections.singletonList(new Gatherer.RowRange(startRow, endRow)));
all.merge(sc, factory);
diff --git a/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java b/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
index 051f774..25f9d73 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
@@ -269,6 +269,7 @@ public abstract class FileOperations {
protected FileSystem fs;
protected Configuration fsConf;
protected RateLimiter rateLimiter;
+ protected CryptoService cryptoService;
protected FileHelper fs(FileSystem fs) {
Objects.requireNonNull(fs);
@@ -299,14 +300,19 @@ public abstract class FileOperations {
return this;
}
+ protected FileHelper cryptoService(CryptoService cs) {
+ this.cryptoService = cs;
+ return this;
+ }
+
protected FileOptions toWriterBuilderOptions(String compression,
FSDataOutputStream outputStream, boolean startEnabled) {
return new FileOptions(tableConfiguration, filename, fs, fsConf, rateLimiter, compression,
- outputStream, startEnabled, null, null, null, false, null, null, null, true);
+ outputStream, startEnabled, null, null, null, false, cryptoService, null, null, true);
}
protected FileOptions toReaderBuilderOptions(BlockCache dataCache, BlockCache indexCache,
- Cache<String,Long> fileLenCache, boolean seekToBeginning, CryptoService cryptoService) {
+ Cache<String,Long> fileLenCache, boolean seekToBeginning) {
return new FileOptions(tableConfiguration, filename, fs, fsConf, rateLimiter, null, null,
false, dataCache, indexCache, fileLenCache, seekToBeginning, cryptoService, null, null,
true);
@@ -314,13 +320,13 @@ public abstract class FileOperations {
protected FileOptions toIndexReaderBuilderOptions() {
return new FileOptions(tableConfiguration, filename, fs, fsConf, rateLimiter, null, null,
- false, null, null, null, false, null, null, null, true);
+ false, null, null, null, false, cryptoService, null, null, true);
}
protected FileOptions toScanReaderBuilderOptions(Range range, Set<ByteSequence> columnFamilies,
boolean inclusive) {
return new FileOptions(tableConfiguration, filename, fs, fsConf, rateLimiter, null, null,
- false, null, null, null, false, null, range, columnFamilies, inclusive);
+ false, null, null, null, false, cryptoService, range, columnFamilies, inclusive);
}
}
@@ -364,6 +370,11 @@ public abstract class FileOperations {
return this;
}
+ public WriterBuilder withCryptoService(CryptoService cs) {
+ cryptoService(cs);
+ return this;
+ }
+
public FileSKVWriter build() throws IOException {
return openWriter(toWriterBuilderOptions(compression, outputStream, enableAccumuloStart));
}
@@ -381,7 +392,6 @@ public abstract class FileOperations {
private BlockCache indexCache;
private Cache<String,Long> fileLenCache;
private boolean seekToBeginning = false;
- private CryptoService cryptoService;
public ReaderTableConfiguration forFile(String filename, FileSystem fs, Configuration fsConf) {
filename(filename).fs(fs).fsConf(fsConf);
@@ -422,8 +432,8 @@ public abstract class FileOperations {
return this;
}
- public ReaderBuilder withCryptoService(CryptoService cryptoService) {
- this.cryptoService = cryptoService;
+ public ReaderBuilder withCryptoService(CryptoService cs) {
+ cryptoService(cs);
return this;
}
@@ -459,8 +469,8 @@ public abstract class FileOperations {
if (!tableConfiguration.getBoolean(Property.TABLE_BLOCKCACHE_ENABLED)) {
withDataCache(null);
}
- return openReader(toReaderBuilderOptions(dataCache, indexCache, fileLenCache, seekToBeginning,
- cryptoService));
+ return openReader(
+ toReaderBuilderOptions(dataCache, indexCache, fileLenCache, seekToBeginning));
}
}
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 51df3ee..3f75c2f 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -174,7 +174,7 @@ public class PrintInfo implements KeywordExecutable {
printCryptoParams(path, fs);
CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null,
- siteConfig, CryptoServiceFactory.getConfigured(siteConfig));
+ siteConfig, CryptoServiceFactory.newInstance(siteConfig));
Reader iter = new RFile.Reader(_rdr);
MetricsGatherer<Map<String,ArrayList<VisibilityMetric>>> vmg = new VisMetricsGatherer();
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
index abe78f0..a8b1dfb 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
@@ -130,7 +130,13 @@ public class RFileOperations extends FileOperations {
outputStream = fs.create(new Path(file), false, bufferSize, (short) rep, block);
}
- CryptoService cryptoService = CryptoServiceFactory.getConfigured(acuconf);
+
+ // calls to openWriter from the tserver will already have a crypto service initialized
+ // calls from clients will require a new crypto service
+ CryptoService cryptoService = options.cryptoService;
+ if (cryptoService == null) {
+ cryptoService = CryptoServiceFactory.newInstance(acuconf);
+ }
BCFile.Writer _cbw = new BCFile.Writer(outputStream, options.getRateLimiter(), compression,
conf, acuconf, cryptoService);
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java
index 148cf7a..d495357 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
@@ -30,7 +31,7 @@ import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
import org.apache.accumulo.core.file.rfile.RFile.Reader;
import org.apache.accumulo.core.file.rfile.RFile.Writer;
import org.apache.accumulo.core.file.rfile.bcfile.BCFile;
-import org.apache.accumulo.core.security.crypto.CryptoServiceFactory;
+import org.apache.accumulo.core.security.crypto.impl.NoCryptoService;
import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.hadoop.conf.Configuration;
@@ -48,6 +49,8 @@ public class SplitLarge {
@Parameter(names = "-m",
description = "the maximum size of the key/value pair to shunt to the small file")
long maxSize = 10 * 1024 * 1024;
+ @Parameter(names = "-crypto", description = "the class to perform encryption/decryption")
+ String cryptoClass = Property.INSTANCE_CRYPTO_SERVICE.getDefaultValue();
@Parameter(description = "<file.rf> { <file.rf> ... }")
List<String> files = new ArrayList<>();
}
@@ -60,7 +63,8 @@ public class SplitLarge {
for (String file : opts.files) {
AccumuloConfiguration aconf = DefaultConfiguration.getInstance();
- CryptoService cryptoService = CryptoServiceFactory.getConfigured(aconf);
+ CryptoService cryptoService = ConfigurationTypeHelper.getClassInstance(null, opts.cryptoClass,
+ CryptoService.class, new NoCryptoService());
Path path = new Path(file);
CachableBlockFile.Reader rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf,
cryptoService);
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 0898cfe..4b68600 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -644,7 +644,7 @@ public final class BCFile {
CryptoEnvironment cryptoEnvironment = null;
if (cryptoService == null) {
- cryptoService = CryptoServiceFactory.getConfigured(aconf);
+ cryptoService = CryptoServiceFactory.newInstance(aconf);
}
// backwards compatibility
@@ -682,7 +682,7 @@ public final class BCFile {
decryptionParams = CryptoUtils.readParams(dis);
CryptoEnvironmentImpl env = new CryptoEnvironmentImpl(Scope.RFILE, decryptionParams);
if (cryptoService == null) {
- cryptoService = CryptoServiceFactory.getConfigured(aconf);
+ cryptoService = CryptoServiceFactory.newInstance(aconf);
}
this.decrypter = cryptoService.getFileDecrypter(env);
}
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
index d854841..43b97db 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
@@ -36,7 +36,7 @@ public class PrintInfo {
BCFile.Reader bcfr = null;
try {
bcfr = new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf, siteConfig,
- CryptoServiceFactory.getConfigured(siteConfig));
+ CryptoServiceFactory.newInstance(siteConfig));
Set<Entry<String,MetaIndexEntry>> es = bcfr.metaIndex.index.entrySet();
diff --git a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoServiceFactory.java b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoServiceFactory.java
index 5931bdf..fab919a 100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoServiceFactory.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoServiceFactory.java
@@ -18,57 +18,18 @@ package org.apache.accumulo.core.security.crypto;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.security.crypto.impl.NoCryptoService;
import org.apache.accumulo.core.spi.crypto.CryptoService;
-import org.apache.accumulo.core.spi.crypto.CryptoService.CryptoException;
-import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
public class CryptoServiceFactory {
+
/**
- * Load and initialize the CryptoService only once, when this class is loaded.
+ * Create a new crypto service configured in {@link Property#INSTANCE_CRYPTO_SERVICE}.
*/
- private static CryptoService singleton = init();
-
- private static CryptoService init() {
- SiteConfiguration conf = SiteConfiguration.getInstance();
- String configuredClass = conf.get(Property.INSTANCE_CRYPTO_SERVICE.getKey());
- CryptoService newCryptoService = loadCryptoService(configuredClass);
+ public static CryptoService newInstance(AccumuloConfiguration conf) {
+ CryptoService newCryptoService = Property.createInstanceFromPropertyName(conf,
+ Property.INSTANCE_CRYPTO_SERVICE, CryptoService.class, new NoCryptoService());
newCryptoService.init(conf.getAllPropertiesWithPrefix(Property.INSTANCE_CRYPTO_PREFIX));
return newCryptoService;
}
-
- /**
- * Get the class configured in {@link Property#INSTANCE_CRYPTO_SERVICE}. This class should have
- * been loaded and initialized when CryptoServiceFactory is loaded.
- *
- * @throws CryptoException
- * if class configured differs from the original class loaded
- */
- public static CryptoService getConfigured(AccumuloConfiguration conf) {
- String currentClass = singleton.getClass().getName();
- String configuredClass = conf.get(Property.INSTANCE_CRYPTO_SERVICE.getKey());
- if (!currentClass.equals(configuredClass)) {
- String msg = String.format("Configured crypto class %s changed since initialization of %s.",
- configuredClass, currentClass);
- throw new CryptoService.CryptoException(msg);
- }
- return singleton;
- }
-
- private static CryptoService loadCryptoService(String className) {
- try {
- Class<? extends CryptoService> clazz = AccumuloVFSClassLoader.loadClass(className,
- CryptoService.class);
- return clazz.newInstance();
- } catch (Exception e) {
- throw new CryptoException(e);
- }
- }
-
- /**
- * This method is only for testing. Do not use.
- */
- public static void resetInstance() {
- singleton = init();
- }
}
diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
index 7fa7b38..07604bf 100644
--- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
+++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
@@ -57,8 +57,8 @@ import org.apache.accumulo.core.data.thrift.TSummaryRequest;
import org.apache.accumulo.core.metadata.schema.MetadataScanner;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.rpc.ThriftUtil;
-import org.apache.accumulo.core.security.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.spi.cache.BlockCache;
+import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
import org.apache.accumulo.core.trace.Tracer;
@@ -109,6 +109,7 @@ public class Gatherer {
private Text endRow = null;
private Range clipRange;
private Predicate<SummarizerConfiguration> summarySelector;
+ private CryptoService cryptoService;
private TSummaryRequest request;
@@ -116,8 +117,8 @@ public class Gatherer {
private Set<SummarizerConfiguration> summaries;
- public Gatherer(ClientContext context, TSummaryRequest request,
- AccumuloConfiguration tableConfig) {
+ public Gatherer(ClientContext context, TSummaryRequest request, AccumuloConfiguration tableConfig,
+ CryptoService cryptoService) {
this.ctx = context;
this.tableId = Table.ID.of(request.tableId);
this.startRow = ByteBufferUtil.toText(request.bounds.startRow);
@@ -126,6 +127,7 @@ public class Gatherer {
this.summaries = request.getSummarizers().stream().map(SummarizerConfigurationUtil::fromThrift)
.collect(Collectors.toSet());
this.request = request;
+ this.cryptoService = cryptoService;
this.summarizerPattern = request.getSummarizerPattern();
@@ -666,9 +668,7 @@ public class Gatherer {
List<RowRange> ranges, BlockCache summaryCache, BlockCache indexCache) {
Path path = new Path(file);
Configuration conf = CachedConfiguration.getInstance();
- return SummaryReader
- .load(volMgr.get(path), conf, ctx.getConfiguration(), factory, path, summarySelector,
- summaryCache, indexCache, CryptoServiceFactory.getConfigured(ctx.getConfiguration()))
- .getSummaries(ranges);
+ return SummaryReader.load(volMgr.get(path), conf, ctx.getConfiguration(), factory, path,
+ summarySelector, summaryCache, indexCache, cryptoService).getSummaries(ranges);
}
}
diff --git a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java
index b1d0d37..8fbdca3 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java
@@ -55,6 +55,7 @@ import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.file.rfile.RFile.Reader;
import org.apache.accumulo.core.iterators.user.RegExFilter;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.crypto.impl.NoCryptoService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -811,7 +812,8 @@ public class RFileTest {
private Reader getReader(LocalFileSystem localFs, String testFile) throws IOException {
return (Reader) FileOperations.getInstance().newReaderBuilder()
.forFile(testFile, localFs, localFs.getConf())
- .withTableConfiguration(DefaultConfiguration.getInstance()).build();
+ .withTableConfiguration(DefaultConfiguration.getInstance())
+ .withCryptoService(new NoCryptoService()).build();
}
@Test
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java
index 6088010..9c0e622 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java
@@ -60,7 +60,7 @@ public class CreateCompatTestFile {
FileSystem fs = FileSystem.get(conf);
AccumuloConfiguration aconf = DefaultConfiguration.getInstance();
BCFile.Writer _cbw = new BCFile.Writer(fs.create(new Path(args[0])), null, "gz", conf, aconf,
- CryptoServiceFactory.getConfigured(aconf));
+ CryptoServiceFactory.newInstance(aconf));
RFile.Writer writer = new RFile.Writer(_cbw, 1000);
writer.startNewLocalityGroup("lg1",
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java
index d1ff2a7..a53a78a 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java
@@ -57,7 +57,7 @@ public class MultiLevelIndexTest extends TestCase {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
FSDataOutputStream dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a"));
BCFile.Writer _cbw = new BCFile.Writer(dos, null, "gz", CachedConfiguration.getInstance(),
- aconf, CryptoServiceFactory.getConfigured(aconf));
+ aconf, CryptoServiceFactory.newInstance(aconf));
BufferedWriter mliw = new BufferedWriter(new Writer(_cbw, maxBlockSize));
@@ -78,7 +78,7 @@ public class MultiLevelIndexTest extends TestCase {
SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data);
FSDataInputStream in = new FSDataInputStream(bais);
CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in, data.length,
- CachedConfiguration.getInstance(), aconf, CryptoServiceFactory.getConfigured(aconf));
+ CachedConfiguration.getInstance(), aconf, CryptoServiceFactory.newInstance(aconf));
Reader reader = new Reader(_cbr, RFile.RINDEX_VER_8);
CachableBlockFile.CachedBlockRead rootIn = _cbr.getMetaBlock("root");
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 89ca96c..dcd1af1 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -149,7 +149,7 @@ public class MultiThreadedRFileTest {
Path path = new Path("file://" + rfile);
dos = fs.create(path, true);
BCFile.Writer _cbw = new BCFile.Writer(dos, null, "gz", conf, accumuloConfiguration,
- CryptoServiceFactory.getConfigured(accumuloConfiguration));
+ CryptoServiceFactory.newInstance(accumuloConfiguration));
SamplerConfigurationImpl samplerConfig = SamplerConfigurationImpl
.newSamplerConfig(accumuloConfiguration);
Sampler sampler = null;
@@ -183,7 +183,7 @@ public class MultiThreadedRFileTest {
// the caches used to obfuscate the multithreaded issues
CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null,
- defaultConf, CryptoServiceFactory.getConfigured(defaultConf));
+ defaultConf, CryptoServiceFactory.newInstance(defaultConf));
reader = new RFile.Reader(_cbr);
iter = new ColumnFamilySkippingIterator(reader);
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6055522..507090e 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -53,7 +53,6 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
@@ -78,8 +77,6 @@ import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.sample.impl.SamplerFactory;
import org.apache.accumulo.core.security.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.security.crypto.CryptoTest;
-import org.apache.accumulo.core.security.crypto.impl.AESCryptoService;
-import org.apache.accumulo.core.security.crypto.impl.NoCryptoService;
import org.apache.accumulo.core.spi.cache.BlockCacheManager;
import org.apache.accumulo.core.spi.cache.CacheType;
import org.apache.accumulo.core.util.CachedConfiguration;
@@ -250,7 +247,7 @@ public class RFileTest {
baos = new ByteArrayOutputStream();
dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a"));
BCFile.Writer _cbw = new BCFile.Writer(dos, null, "gz", conf, accumuloConfiguration,
- CryptoServiceFactory.getConfigured(accumuloConfiguration));
+ CryptoServiceFactory.newInstance(accumuloConfiguration));
SamplerConfigurationImpl samplerConfig = SamplerConfigurationImpl
.newSamplerConfig(accumuloConfiguration);
@@ -313,7 +310,7 @@ public class RFileTest {
CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader("source-1", in, fileLength, conf,
dataCache, indexCache, accumuloConfiguration,
- CryptoServiceFactory.getConfigured(accumuloConfiguration));
+ CryptoServiceFactory.newInstance(accumuloConfiguration));
reader = new RFile.Reader(_cbr);
if (cfsi)
iter = new ColumnFamilySkippingIterator(reader);
@@ -1723,13 +1720,11 @@ public class RFileTest {
@Test
public void testOldVersionsWithCrypto() throws Exception {
- turnCryptoOnInSiteConfig();
AccumuloConfiguration cryptoOnConf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
runVersionTest(3, cryptoOnConf);
runVersionTest(4, cryptoOnConf);
runVersionTest(6, cryptoOnConf);
runVersionTest(7, cryptoOnConf);
- turnCryptoOffInSiteConfig();
}
private void runVersionTest(int version, AccumuloConfiguration aconf) throws IOException {
@@ -1745,7 +1740,7 @@ public class RFileTest {
SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data);
FSDataInputStream in2 = new FSDataInputStream(bais);
CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in2, data.length,
- CachedConfiguration.getInstance(), aconf, CryptoServiceFactory.getConfigured(aconf));
+ CachedConfiguration.getInstance(), aconf, CryptoServiceFactory.newInstance(aconf));
Reader reader = new RFile.Reader(_cbr);
checkIndex(reader);
@@ -1801,33 +1796,15 @@ public class RFileTest {
return result;
}
- public void turnCryptoOnInSiteConfig() {
- SiteConfiguration.clearInstance();
- SiteConfiguration siteConfig = SiteConfiguration.create();
- siteConfig.set(Property.INSTANCE_CRYPTO_SERVICE, AESCryptoService.class.getName());
- siteConfig.set("instance.crypto.opts.kekId", "file:///tmp/testAESFile");
- siteConfig.set("instance.crypto.opts.keyManager", "uri");
- CryptoServiceFactory.resetInstance();
- }
-
- public static void turnCryptoOffInSiteConfig() {
- SiteConfiguration.getInstance().set(Property.INSTANCE_CRYPTO_SERVICE,
- NoCryptoService.class.getName());
- CryptoServiceFactory.resetInstance();
- }
-
@Test
public void testEncRFile1() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test1();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile2() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test2();
conf = null;
@@ -1835,149 +1812,116 @@ public class RFileTest {
@Test
public void testEncRFile3() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test3();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile4() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test4();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile5() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test5();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile6() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test6();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile7() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test7();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile8() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test8();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile9() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test9();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile10() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test10();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile11() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test11();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile12() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test12();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile13() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test13();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile14() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test14();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile16() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test16();
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile17() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test17();
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile18() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test18();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncRFile19() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test19();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
public void testEncryptedRFiles() throws Exception {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
test1();
test2();
@@ -1988,7 +1932,6 @@ public class RFileTest {
test7();
test8();
conf = null;
- turnCryptoOffInSiteConfig();
}
private Key newKey(int r, int c) {
@@ -2282,12 +2225,10 @@ public class RFileTest {
@Test
public void testEncSample() throws IOException {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
testSample();
testSampleLG();
conf = null;
- turnCryptoOffInSiteConfig();
}
@Test
@@ -2342,7 +2283,6 @@ public class RFileTest {
@Test
public void testCryptoDoesntLeakSensitive() throws IOException {
- turnCryptoOnInSiteConfig();
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
// test an empty file
@@ -2360,12 +2300,10 @@ public class RFileTest {
assertEquals(-1, Bytes.indexOf(rfBytes, toCheck));
}
}
- turnCryptoOffInSiteConfig();
}
@Test
public void testRootTabletEncryption() throws Exception {
- turnCryptoOnInSiteConfig();
// This tests that the normal set of operations used to populate a root tablet
conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
@@ -2449,6 +2387,5 @@ public class RFileTest {
testRfile.closeReader();
conf = null;
- turnCryptoOffInSiteConfig();
}
}
diff --git a/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java b/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java
index 7092a9c..ae284c2 100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java
@@ -48,7 +48,6 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.crypto.impl.AESCryptoService;
@@ -68,7 +67,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -104,26 +102,12 @@ public class CryptoTest {
fs.delete(aesPath, true);
}
- @Before
- public void turnCryptoOnInSiteConfig() {
- SiteConfiguration siteConfig = SiteConfiguration.getInstance();
- siteConfig.set(Property.INSTANCE_CRYPTO_SERVICE, AESCryptoService.class.getName());
- siteConfig.set("instance.crypto.opts.kekId", "file:///tmp/testAESFile");
- siteConfig.set("instance.crypto.opts.keyManager", "uri");
- CryptoServiceFactory.resetInstance();
- }
-
- public static void turnCryptoOffInSiteConfig() {
- SiteConfiguration siteConfig = SiteConfiguration.getInstance();
- siteConfig.set(Property.INSTANCE_CRYPTO_SERVICE, NoCryptoService.class.getName());
- CryptoServiceFactory.resetInstance();
- }
-
@Test
public void simpleGCMTest() throws Exception {
AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_CONF);
- CryptoService cryptoService = CryptoServiceFactory.getConfigured(conf);
+ CryptoService cryptoService = new AESCryptoService();
+ cryptoService.init(conf.getAllPropertiesWithPrefix(Property.INSTANCE_CRYPTO_PREFIX));
CryptoEnvironment encEnv = new CryptoEnvironmentImpl(Scope.RFILE, null);
FileEncrypter encrypter = cryptoService.getFileEncrypter(encEnv);
byte[] params = encrypter.getDecryptionParameters();
@@ -188,7 +172,6 @@ public class CryptoTest {
@Test
public void testNoEncryptionWAL() throws Exception {
NoCryptoService cs = new NoCryptoService();
- turnCryptoOffInSiteConfig();
byte[] encryptedBytes = encrypt(cs, Scope.WAL, CRYPTO_OFF_CONF);
String stringifiedBytes = Arrays.toString(encryptedBytes);
@@ -203,7 +186,6 @@ public class CryptoTest {
@Test
public void testNoEncryptionRFILE() throws Exception {
NoCryptoService cs = new NoCryptoService();
- turnCryptoOffInSiteConfig();
byte[] encryptedBytes = encrypt(cs, Scope.RFILE, CRYPTO_OFF_CONF);
String stringifiedBytes = Arrays.toString(encryptedBytes);
@@ -247,7 +229,6 @@ public class CryptoTest {
FileSystem fs = FileSystem.getLocal(CachedConfiguration.getInstance());
ArrayList<Key> keys = testData();
- turnCryptoOffInSiteConfig();
String file = "target/testFile2.rf";
fs.delete(new Path(file), true);
try (RFileWriter writer = RFile.newWriter().to(file).withFileSystem(fs)
@@ -259,8 +240,6 @@ public class CryptoTest {
}
}
- turnCryptoOnInSiteConfig();
- CryptoServiceFactory.resetInstance();
Scanner iter = RFile.newScanner().from(file).withFileSystem(fs)
.withTableProperties(cryptoOnConf).build();
ArrayList<Key> keysRead = new ArrayList<>();
@@ -338,13 +317,12 @@ public class CryptoTest {
private <C extends CryptoService> byte[] encrypt(C cs, Scope scope, String configFile)
throws Exception {
AccumuloConfiguration conf = setAndGetAccumuloConfig(configFile);
- CryptoService cryptoService = CryptoServiceFactory.getConfigured(conf);
+ cs.init(conf.getAllPropertiesWithPrefix(Property.INSTANCE_CRYPTO_PREFIX));
CryptoEnvironmentImpl env = new CryptoEnvironmentImpl(scope, null);
- FileEncrypter encrypter = cryptoService.getFileEncrypter(env);
+ FileEncrypter encrypter = cs.getFileEncrypter(env);
byte[] params = encrypter.getDecryptionParameters();
assertNotNull("CryptoService returned null FileEncrypter", encrypter);
- assertEquals(cryptoService.getClass(), cs.getClass());
ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(out);
@@ -367,7 +345,7 @@ public class CryptoTest {
byte[] params = CryptoUtils.readParams(dataIn);
AccumuloConfiguration conf = setAndGetAccumuloConfig(configFile);
- CryptoService cryptoService = CryptoServiceFactory.getConfigured(conf);
+ CryptoService cryptoService = CryptoServiceFactory.newInstance(conf);
CryptoEnvironment env = new CryptoEnvironmentImpl(scope, params);
FileDecrypter decrypter = cryptoService.getFileDecrypter(env);
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
index 4ae6b3e..dc71a4d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
@@ -33,6 +33,8 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.rpc.SslConnectionParams;
+import org.apache.accumulo.core.security.crypto.CryptoServiceFactory;
+import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.trace.DistributedTrace;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -63,6 +65,7 @@ public class ServerContext extends ClientContext {
private String applicationClassName = null;
private String hostname = null;
private AuthenticationTokenSecretManager secretManager;
+ private CryptoService cryptoService = null;
public ServerContext(SiteConfiguration siteConfig) {
this(new ServerInfo(siteConfig));
@@ -104,6 +107,18 @@ public class ServerContext extends ClientContext {
}
}
+ /**
+ * Should only be called by the Tablet server
+ */
+ public synchronized void setupCrypto() throws CryptoService.CryptoException {
+ if (cryptoService != null)
+ throw new CryptoService.CryptoException("Crypto Service " + cryptoService.getClass().getName()
+ + " already exists and cannot be setup again");
+
+ AccumuloConfiguration acuConf = getConfiguration();
+ cryptoService = CryptoServiceFactory.newInstance(acuConf);
+ }
+
public void teardownServer() {
DistributedTrace.disable();
}
@@ -241,4 +256,11 @@ public class ServerContext extends ClientContext {
}
return nameAllocator;
}
+
+ public CryptoService getCryptoService() {
+ if (cryptoService == null) {
+ throw new CryptoService.CryptoException("Crypto service not initialized.");
+ }
+ return cryptoService;
+ }
}
diff --git a/server/tserver/src/main/findbugs/exclude-filter.xml b/server/tserver/src/main/findbugs/exclude-filter.xml
index a334163..eb6386a 100644
--- a/server/tserver/src/main/findbugs/exclude-filter.xml
+++ b/server/tserver/src/main/findbugs/exclude-filter.xml
@@ -24,7 +24,7 @@
<Match>
<!-- false positive about forced garbage collection in resource manager -->
<Class name="org.apache.accumulo.tserver.TabletServerResourceManager" />
- <Method name="<init>" params="org.apache.accumulo.tserver.TabletServer,org.apache.accumulo.server.fs.VolumeManager" returns="void" />
+ <Method name="<init>" params="org.apache.accumulo.tserver.TabletServer,org.apache.accumulo.server.fs.VolumeManager,org.apache.accumulo.server.ServerContext" returns="void" />
<Bug code="DM" pattern="DM_GC" />
</Match>
</FindBugsFilter>
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
index 2091f09..313067e 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
@@ -326,7 +326,8 @@ public class FileManager {
.forFile(path.toString(), ns, ns.getConf())
.withTableConfiguration(
context.getServerConfFactory().getTableConfiguration(tablet.getTableId()))
- .withBlockCache(dataCache, indexCache).withFileLenCache(fileLenCache).build();
+ .withBlockCache(dataCache, indexCache).withFileLenCache(fileLenCache)
+ .withCryptoService(context.getCryptoService()).build();
readersReserved.put(reader, file);
} catch (Exception e) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java
index 6a693ba..2a09a15 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/InMemoryMap.java
@@ -71,6 +71,7 @@ import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfiguratio
import org.apache.accumulo.core.util.LocalityGroupUtil.Partitioner;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.PreAllocatedArray;
+import org.apache.accumulo.server.ServerContext;
import org.apache.commons.lang.mutable.MutableLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -85,6 +86,7 @@ public class InMemoryMap {
private static final Logger log = LoggerFactory.getLogger(InMemoryMap.class);
+ private ServerContext context;
private volatile String memDumpFile = null;
private final String memDumpDir;
private final String mapType;
@@ -131,7 +133,8 @@ public class InMemoryMap {
return pair.getSecond();
}
- public InMemoryMap(AccumuloConfiguration config) throws LocalityGroupConfigurationError {
+ public InMemoryMap(AccumuloConfiguration config, ServerContext serverContext)
+ throws LocalityGroupConfigurationError {
boolean useNativeMap = config.getBoolean(Property.TSERV_NATIVEMAP_ENABLED);
@@ -139,6 +142,7 @@ public class InMemoryMap {
this.lggroups = LocalityGroupUtil.getLocalityGroups(config);
this.config = config;
+ this.context = serverContext;
SimpleMap allMap;
SimpleMap sampleMap;
@@ -639,7 +643,8 @@ public class InMemoryMap {
FileSystem fs = FileSystem.getLocal(conf);
reader = new RFileOperations().newReaderBuilder().forFile(memDumpFile, fs, conf)
- .withTableConfiguration(SiteConfiguration.getInstance()).seekToBeginning().build();
+ .withTableConfiguration(SiteConfiguration.getInstance())
+ .withCryptoService(context.getCryptoService()).seekToBeginning().build();
if (iflag != null)
reader.setInterruptFlag(iflag);
@@ -810,7 +815,7 @@ public class InMemoryMap {
}
FileSKVWriter out = new RFileOperations().newWriterBuilder().forFile(tmpFile, fs, newConf)
- .withTableConfiguration(siteConf).build();
+ .withTableConfiguration(siteConf).withCryptoService(context.getCryptoService()).build();
InterruptibleIterator iter = map.skvIterator(null);
@@ -886,4 +891,8 @@ public class InMemoryMap {
iter.next();
}
}
+
+ public ServerContext getContext() {
+ return context;
+ }
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index f211a6e..e5c9eb0 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -413,7 +413,7 @@ public class TabletServer implements Runnable {
logger = new TabletServerLogger(this, walogMaxSize, syncCounter, flushCounter,
walCreationRetryFactory, walWritingRetryFactory, walogMaxAge);
- this.resourceManager = new TabletServerResourceManager(this, fs);
+ this.resourceManager = new TabletServerResourceManager(this, fs, context);
this.security = AuditedSecurityOperation.getInstance(context);
metricsFactory = new TabletServerMetricsFactory(aconf);
@@ -2078,7 +2078,7 @@ public class TabletServer implements Runnable {
ServerConfigurationFactory factory = context.getServerConfFactory();
ExecutorService es = resourceManager.getSummaryPartitionExecutor();
Future<SummaryCollection> future = new Gatherer(context, request,
- factory.getTableConfiguration(tableId)).gather(es);
+ factory.getTableConfiguration(tableId), context.getCryptoService()).gather(es);
return startSummaryOperation(credentials, future);
}
@@ -2096,8 +2096,8 @@ public class TabletServer implements Runnable {
ServerConfigurationFactory factory = context.getServerConfFactory();
ExecutorService spe = resourceManager.getSummaryRemoteExecutor();
Future<SummaryCollection> future = new Gatherer(context, request,
- factory.getTableConfiguration(Table.ID.of(request.getTableId()))).processPartition(spe,
- modulus, remainder);
+ factory.getTableConfiguration(Table.ID.of(request.getTableId())),
+ context.getCryptoService()).processPartition(spe, modulus, remainder);
return startSummaryOperation(credentials, future);
}
@@ -2118,8 +2118,8 @@ public class TabletServer implements Runnable {
BlockCache summaryCache = resourceManager.getSummaryCache();
BlockCache indexCache = resourceManager.getIndexCache();
FileSystemResolver volMgr = p -> fs.getVolumeByPath(p).getFileSystem();
- Future<SummaryCollection> future = new Gatherer(context, request, tableCfg)
- .processFiles(volMgr, files, summaryCache, indexCache, srp);
+ Future<SummaryCollection> future = new Gatherer(context, request, tableCfg,
+ context.getCryptoService()).processFiles(volMgr, files, summaryCache, indexCache, srp);
return startSummaryOperation(credentials, future);
}
@@ -3339,6 +3339,7 @@ public class TabletServer implements Runnable {
opts.parseArgs(app, args);
ServerContext context = new ServerContext(opts.getSiteConfiguration());
context.setupServer(app, TabletServer.class.getSimpleName(), opts.getAddress());
+ context.setupCrypto();
try {
final TabletServer server = new TabletServer(context);
if (UserGroupInformation.isSecurityEnabled()) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index 5e59025..8b2c60a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@ -64,6 +64,7 @@ import org.apache.accumulo.core.spi.scan.SimpleScanDispatcher;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.fate.util.LoggingRunnable;
+import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -130,6 +131,7 @@ public class TabletServerResourceManager {
private final BlockCache _sCache;
private final TabletServer tserver;
private final ServerConfigurationFactory conf;
+ private final ServerContext context;
private ExecutorService addEs(String name, ExecutorService tp) {
if (threadPools.containsKey(name)) {
@@ -307,9 +309,11 @@ public class TabletServerResourceManager {
return builder.build();
}
- public TabletServerResourceManager(TabletServer tserver, VolumeManager fs) {
+ public TabletServerResourceManager(TabletServer tserver, VolumeManager fs,
+ ServerContext context) {
this.tserver = tserver;
this.conf = tserver.getContext().getServerConfFactory();
+ this.context = context;
final AccumuloConfiguration acuConf = conf.getSystemConfiguration();
long maxMemory = acuConf.getAsBytes(Property.TSERV_MAXMEM);
@@ -861,7 +865,8 @@ public class TabletServerResourceManager {
Property.TABLE_COMPACTION_STRATEGY, CompactionStrategy.class,
new DefaultCompactionStrategy());
strategy.init(Property.getCompactionStrategyOptions(tableConf));
- MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, tableConf);
+ MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, tableConf,
+ context);
request.setFiles(tabletFiles);
try {
return strategy.shouldCompact(request);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
index 152c9eb..5886509 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
@@ -38,13 +38,13 @@ import org.apache.accumulo.core.data.impl.TabletIdImpl;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
-import org.apache.accumulo.core.security.crypto.CryptoServiceFactory;
import org.apache.accumulo.core.spi.cache.BlockCache;
import org.apache.accumulo.core.summary.Gatherer;
import org.apache.accumulo.core.summary.SummarizerFactory;
import org.apache.accumulo.core.summary.SummaryCollection;
import org.apache.accumulo.core.summary.SummaryReader;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.tserver.compaction.strategies.TooManyDeletesCompactionStrategy;
@@ -64,10 +64,11 @@ public class MajorCompactionRequest implements Cloneable {
final private BlockCache indexCache;
final private BlockCache summaryCache;
private Map<FileRef,DataFileValue> files;
+ final private ServerContext context;
public MajorCompactionRequest(KeyExtent extent, MajorCompactionReason reason,
VolumeManager manager, AccumuloConfiguration tabletConfig, BlockCache summaryCache,
- BlockCache indexCache) {
+ BlockCache indexCache, ServerContext context) {
this.extent = extent;
this.reason = reason;
this.volumeManager = manager;
@@ -75,16 +76,17 @@ public class MajorCompactionRequest implements Cloneable {
this.files = Collections.emptyMap();
this.summaryCache = summaryCache;
this.indexCache = indexCache;
+ this.context = context;
}
public MajorCompactionRequest(KeyExtent extent, MajorCompactionReason reason,
- AccumuloConfiguration tabletConfig) {
- this(extent, reason, null, tabletConfig, null, null);
+ AccumuloConfiguration tabletConfig, ServerContext context) {
+ this(extent, reason, null, tabletConfig, null, null, context);
}
public MajorCompactionRequest(MajorCompactionRequest mcr) {
this(mcr.extent, mcr.reason, mcr.volumeManager, mcr.tableConfig, mcr.summaryCache,
- mcr.indexCache);
+ mcr.indexCache, mcr.context);
// know this is already unmodifiable, no need to wrap again
this.files = mcr.files;
}
@@ -153,7 +155,7 @@ public class MajorCompactionRequest implements Cloneable {
Configuration conf = CachedConfiguration.getInstance();
SummaryCollection fsc = SummaryReader
.load(fs, conf, tableConfig, factory, file.path(), summarySelector, summaryCache,
- indexCache, CryptoServiceFactory.getConfigured(tableConfig))
+ indexCache, context.getCryptoService())
.getSummaries(Collections.singletonList(new Gatherer.RowRange(extent)));
sc.merge(fsc, factory);
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index 24ada5e..28b60dd 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -370,7 +370,7 @@ public class DfsLogger implements Comparable<DfsLogger> {
input.readFully(magicBuffer);
if (Arrays.equals(magicBuffer, magic)) {
byte[] params = CryptoUtils.readParams(input);
- CryptoService cryptoService = CryptoServiceFactory.getConfigured(conf);
+ CryptoService cryptoService = CryptoServiceFactory.newInstance(conf);
CryptoEnvironment env = new CryptoEnvironmentImpl(Scope.WAL, params);
FileDecrypter decrypter = cryptoService.getFileDecrypter(env);
@@ -429,7 +429,7 @@ public class DfsLogger implements Comparable<DfsLogger> {
flush = logFile.getClass().getMethod("hflush");
// Initialize the log file with a header and its encryption
- CryptoService cryptoService = CryptoServiceFactory.getConfigured(conf.getConfiguration());
+ CryptoService cryptoService = context.getCryptoService();
logFile.write(LOG_FILE_HEADER_V4.getBytes(UTF_8));
log.debug("Using {} for encrypting WAL {}", cryptoService.getClass().getSimpleName(),
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
index 1721dbe..8091c68 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
@@ -201,7 +201,8 @@ public class Compactor implements Callable<CompactionStats> {
FileOperations fileFactory = FileOperations.getInstance();
FileSystem ns = this.fs.getVolumeByPath(outputFilePath).getFileSystem();
mfw = fileFactory.newWriterBuilder().forFile(outputFilePathName, ns, ns.getConf())
- .withTableConfiguration(acuTableConf).withRateLimiter(env.getWriteLimiter()).build();
+ .withTableConfiguration(acuTableConf).withRateLimiter(env.getWriteLimiter())
+ .withCryptoService(context.getCryptoService()).build();
Map<String,Set<ByteSequence>> lGroups;
try {
@@ -290,7 +291,8 @@ public class Compactor implements Callable<CompactionStats> {
FileSKVIterator reader;
reader = fileFactory.newReaderBuilder().forFile(mapFile.path().toString(), fs, fs.getConf())
- .withTableConfiguration(acuTableConf).withRateLimiter(env.getReadLimiter()).build();
+ .withTableConfiguration(acuTableConf).withRateLimiter(env.getReadLimiter())
+ .withCryptoService(context.getCryptoService()).build();
readers.add(reader);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 5f18c36..a850a44 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1822,7 +1822,7 @@ public class Tablet implements TabletCommitter {
BlockCache sc = tabletResources.getTabletServerResourceManager().getSummaryCache();
BlockCache ic = tabletResources.getTabletServerResourceManager().getIndexCache();
MajorCompactionRequest request = new MajorCompactionRequest(extent, reason,
- getTabletServer().getFileSystem(), tableConfiguration, sc, ic);
+ getTabletServer().getFileSystem(), tableConfiguration, sc, ic, context);
request.setFiles(getDatafileManager().getDatafileSizes());
strategy.gatherInformation(request);
}
@@ -1867,7 +1867,7 @@ public class Tablet implements TabletCommitter {
inputFiles.addAll(findChopFiles(extent, firstAndLastKeys, allFiles.keySet()));
} else {
MajorCompactionRequest request = new MajorCompactionRequest(extent, reason,
- tableConfiguration);
+ tableConfiguration, context);
request.setFiles(allFiles);
plan = strategy.getCompactionPlan(request);
if (plan != null) {
@@ -2688,7 +2688,7 @@ public class Tablet implements TabletCommitter {
CompactionStrategy strategy = createCompactionStrategy(strategyConfig);
MajorCompactionRequest request = new MajorCompactionRequest(extent,
- MajorCompactionReason.USER, tableConfiguration);
+ MajorCompactionReason.USER, tableConfiguration, context);
request.setFiles(getDatafileManager().getDatafileSizes());
try {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
index 396648f..97cc43d 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
@@ -156,7 +156,7 @@ public class TabletData {
long maxTime = -1;
try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
.forFile(path.toString(), ns, ns.getConf()).withTableConfiguration(conf).seekToBeginning()
- .build()) {
+ .withCryptoService(context.getCryptoService()).build()) {
while (reader.hasTop()) {
maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp());
reader.next();
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletMemory.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletMemory.java
index 3c30d81..fcb8f83 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletMemory.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletMemory.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
+import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.tserver.InMemoryMap;
import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
import org.slf4j.Logger;
@@ -38,11 +39,13 @@ class TabletMemory implements Closeable {
private InMemoryMap deletingMemTable;
private long nextSeq = 1L;
private CommitSession commitSession;
+ private ServerContext context;
- TabletMemory(TabletCommitter tablet) {
+ TabletMemory(Tablet tablet) {
this.tablet = tablet;
+ this.context = tablet.getContext();
try {
- memTable = new InMemoryMap(tablet.getTableConfiguration());
+ memTable = new InMemoryMap(tablet.getTableConfiguration(), context);
} catch (LocalityGroupConfigurationError e) {
throw new RuntimeException(e);
}
@@ -72,7 +75,7 @@ class TabletMemory implements Closeable {
otherMemTable = memTable;
try {
- memTable = new InMemoryMap(tablet.getTableConfiguration());
+ memTable = new InMemoryMap(tablet.getTableConfiguration(), context);
} catch (LocalityGroupConfigurationError e) {
throw new RuntimeException(e);
}
@@ -182,6 +185,10 @@ class TabletMemory implements Closeable {
return commitSession;
}
+ public ServerContext getContext() {
+ return context;
+ }
+
@Override
public void close() throws IOException {
commitSession = null;
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
index 089b1c2..d9a8119 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
@@ -55,13 +55,16 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.sample.impl.SamplerFactory;
+import org.apache.accumulo.core.security.crypto.impl.NoCryptoService;
import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
+import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.ZooConfiguration;
import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -106,6 +109,13 @@ public class InMemoryMapTest {
Logger.getLogger(ZooConfiguration.class).setLevel(Level.OFF);
}
+ public static ServerContext getServerContext() {
+ ServerContext context = EasyMock.createMock(ServerContext.class);
+ EasyMock.expect(context.getCryptoService()).andReturn(new NoCryptoService()).anyTimes();
+ EasyMock.replay(context);
+ return context;
+ }
+
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(
new File(System.getProperty("user.dir") + "/target"));
@@ -176,7 +186,7 @@ public class InMemoryMapTest {
ConfigurationCopy config = new ConfigurationCopy(DefaultConfiguration.getInstance());
config.set(Property.TSERV_NATIVEMAP_ENABLED, "" + useNative);
config.set(Property.TSERV_MEMDUMP_DIR, memDumpDir);
- return new InMemoryMap(config);
+ return new InMemoryMap(config, getServerContext());
}
@Test
@@ -549,7 +559,7 @@ public class InMemoryMapTest {
LocalityGroupUtil.encodeColumnFamilies(toTextSet("cf3", "cf4")));
config.set(Property.TABLE_LOCALITY_GROUPS.getKey(), "lg1,lg2");
- InMemoryMap imm = new InMemoryMap(config);
+ InMemoryMap imm = new InMemoryMap(config, getServerContext());
Mutation m1 = new Mutation("r1");
m1.put("cf1", "x", 2, "1");
@@ -586,8 +596,8 @@ public class InMemoryMapTest {
seekLocalityGroups(iter1);
seekLocalityGroups(dc1);
- // TODO uncomment following when ACCUMULO-1628 is fixed
- // seekLocalityGroups(iter1.deepCopy(null));
+ // tests ACCUMULO-1628
+ seekLocalityGroups(iter1.deepCopy(null));
}
@Test
@@ -612,7 +622,7 @@ public class InMemoryMapTest {
for (ConfigurationCopy config : Arrays.asList(config1, config2)) {
- InMemoryMap imm = new InMemoryMap(config);
+ InMemoryMap imm = new InMemoryMap(config, getServerContext());
TreeMap<Key,Value> expectedSample = new TreeMap<>();
TreeMap<Key,Value> expectedAll = new TreeMap<>();
@@ -698,7 +708,7 @@ public class InMemoryMapTest {
config1.set(entry.getKey(), entry.getValue());
}
- InMemoryMap imm = new InMemoryMap(config1);
+ InMemoryMap imm = new InMemoryMap(config1, getServerContext());
TreeMap<Key,Value> expectedSample = new TreeMap<>();
TreeMap<Key,Value> expectedAll = new TreeMap<>();
@@ -758,7 +768,7 @@ public class InMemoryMapTest {
config1.set(entry.getKey(), entry.getValue());
}
- InMemoryMap imm = new InMemoryMap(config1);
+ InMemoryMap imm = new InMemoryMap(config1, getServerContext());
mutate(imm, "r", "cf:cq", 5, "b");
@@ -803,7 +813,7 @@ public class InMemoryMapTest {
config1.set(entry.getKey(), entry.getValue());
}
- InMemoryMap imm = new InMemoryMap(config1);
+ InMemoryMap imm = new InMemoryMap(config1, getServerContext());
// change sampler config after creating in mem map.
SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java
index 5e82f18..411f174 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java
@@ -42,9 +42,12 @@ import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.accumulo.core.security.crypto.impl.NoCryptoService;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
import org.junit.Test;
public class DefaultCompactionStrategyTest {
@@ -59,6 +62,13 @@ public class DefaultCompactionStrategyTest {
return new Pair<>(first, second);
}
+ public static ServerContext getServerContext() {
+ ServerContext context = EasyMock.createMock(ServerContext.class);
+ EasyMock.expect(context.getCryptoService()).andReturn(new NoCryptoService()).anyTimes();
+ EasyMock.replay(context);
+ return context;
+ }
+
static final Map<String,Pair<Key,Key>> fakeFiles = new HashMap<>();
static {
@@ -154,7 +164,7 @@ public class DefaultCompactionStrategyTest {
TestCompactionRequest(KeyExtent extent, MajorCompactionReason reason,
Map<FileRef,DataFileValue> files) {
- super(extent, reason, dfault);
+ super(extent, reason, dfault, getServerContext());
setFiles(files);
}
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java
index 6075108..ad4bdf5 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.accumulo.tserver.compaction;
+import static org.apache.accumulo.tserver.compaction.DefaultCompactionStrategyTest.getServerContext;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@@ -55,7 +57,7 @@ public class SizeLimitCompactionStrategyTest {
KeyExtent ke = new KeyExtent(Table.ID.of("0"), null, null);
MajorCompactionRequest mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL,
- DefaultConfiguration.getInstance());
+ DefaultConfiguration.getInstance(), getServerContext());
mcr.setFiles(nfl("f1", "2G", "f2", "2G", "f3", "2G", "f4", "2G"));
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java
index 6aed0f4..98c72d6 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.accumulo.tserver.compaction;
+import static org.apache.accumulo.tserver.compaction.DefaultCompactionStrategyTest.getServerContext;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@@ -29,6 +31,7 @@ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.tserver.InMemoryMapTest;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -66,7 +69,8 @@ public class TwoTierCompactionStrategyTest {
ttcs.init(opts);
conf = DefaultConfiguration.getInstance();
KeyExtent ke = new KeyExtent(Table.ID.of("0"), null, null);
- mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf);
+ mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf,
+ InMemoryMapTest.getServerContext());
Map<FileRef,DataFileValue> fileMap = createFileMap("f1", "10M", "f2", "10M", "f3", "10M", "f4",
"10M", "f5", "100M", "f6", "100M", "f7", "100M", "f8", "100M");
mcr.setFiles(fileMap);
@@ -85,7 +89,7 @@ public class TwoTierCompactionStrategyTest {
ttcs.init(opts);
conf = DefaultConfiguration.getInstance();
KeyExtent ke = new KeyExtent(Table.ID.of("0"), null, null);
- mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf);
+ mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf, getServerContext());
Map<FileRef,DataFileValue> fileMap = createFileMap("f1", "2G", "f2", "2G", "f3", "2G", "f4",
"2G");
mcr.setFiles(fileMap);
@@ -116,7 +120,7 @@ public class TwoTierCompactionStrategyTest {
ttcs.init(opts);
conf = DefaultConfiguration.getInstance();
KeyExtent ke = new KeyExtent(Table.ID.of("0"), null, null);
- mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf);
+ mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf, getServerContext());
Map<FileRef,DataFileValue> fileMap = createFileMap("f1", "1G", "f2", "10M", "f3", "10M", "f4",
"10M", "f5", "10M", "f6", "10M", "f7", "10M");
Map<FileRef,DataFileValue> filesToCompactMap = createFileMap("f2", "10M", "f3", "10M", "f4",
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java
index 67c5807..feaca76 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java
@@ -17,6 +17,8 @@
package org.apache.accumulo.tserver.compaction.strategies;
+import static org.apache.accumulo.tserver.compaction.DefaultCompactionStrategyTest.getServerContext;
+
import java.util.HashMap;
import java.util.Map;
@@ -39,7 +41,8 @@ public class ConfigurableCompactionStrategyTest {
@Test
public void testOutputOptions() throws Exception {
MajorCompactionRequest mcr = new MajorCompactionRequest(
- new KeyExtent(Table.ID.of("1"), null, null), MajorCompactionReason.USER, null);
+ new KeyExtent(Table.ID.of("1"), null, null), MajorCompactionReason.USER, null,
+ getServerContext());
Map<FileRef,DataFileValue> files = new HashMap<>();
files.put(new FileRef("hdfs://nn1/accumulo/tables/1/t-009/F00001.rf"),
diff --git a/test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java b/test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java
index 0444730..dac4442 100644
--- a/test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/InMemoryMapIT.java
@@ -41,12 +41,15 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.security.crypto.impl.NoCryptoService;
+import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.test.categories.SunnyDayTests;
import org.apache.accumulo.test.functional.NativeMapIT;
import org.apache.accumulo.tserver.InMemoryMap;
import org.apache.accumulo.tserver.MemKey;
import org.apache.accumulo.tserver.NativeMap;
import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -99,6 +102,13 @@ public class InMemoryMapIT {
}
}
+ public static ServerContext getServerContext() {
+ ServerContext context = EasyMock.createMock(ServerContext.class);
+ EasyMock.expect(context.getCryptoService()).andReturn(new NoCryptoService()).anyTimes();
+ EasyMock.replay(context);
+ return context;
+ }
+
@Test
public void testOneMutationOneKey() {
Mutation m = new Mutation("a");
@@ -233,12 +243,15 @@ public class InMemoryMapIT {
localityGroupNativeConfig.put(Property.TSERV_MEMDUMP_DIR.getKey(),
tempFolder.newFolder().getAbsolutePath());
- defaultMap = new InMemoryMap(new ConfigurationCopy(defaultMapConfig));
- nativeMapWrapper = new InMemoryMap(new ConfigurationCopy(nativeMapConfig));
+ defaultMap = new InMemoryMap(new ConfigurationCopy(defaultMapConfig), getServerContext());
+ nativeMapWrapper = new InMemoryMap(new ConfigurationCopy(nativeMapConfig),
+ getServerContext());
localityGroupMap = new InMemoryMap(
- updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupConfig)));
+ updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupConfig)),
+ getServerContext());
localityGroupMapWithNative = new InMemoryMap(
- updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupNativeConfig)));
+ updateConfigurationForLocalityGroups(new ConfigurationCopy(localityGroupNativeConfig)),
+ getServerContext());
} catch (Exception e) {
log.error("Error getting new InMemoryMap ", e);
fail(e.getMessage());
diff --git a/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java b/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java
index 83fe8f0..f2c1060 100644
--- a/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java
+++ b/test/src/main/java/org/apache/accumulo/test/util/memory/InMemoryMapMemoryUsageCheck.java
@@ -23,6 +23,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
+import org.apache.accumulo.test.InMemoryMapIT;
import org.apache.accumulo.tserver.InMemoryMap;
import org.apache.hadoop.io.Text;
@@ -55,7 +56,7 @@ class InMemoryMapMemoryUsageCheck extends MemoryUsageCheck {
@Override
void init() {
try {
- imm = new InMemoryMap(DefaultConfiguration.getInstance());
+ imm = new InMemoryMap(DefaultConfiguration.getInstance(), InMemoryMapIT.getServerContext());
} catch (LocalityGroupConfigurationError e) {
throw new RuntimeException(e);
}