You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2015/05/07 20:54:15 UTC
[11/11] accumulo git commit: Merge branch '1.7'
Merge branch '1.7'
This closes #35
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bc3dc3b8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bc3dc3b8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bc3dc3b8
Branch: refs/heads/master
Commit: bc3dc3b8de0bbc1f48760a413089b5ef04620c28
Parents: 6acfd41 b932755
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu May 7 14:48:54 2015 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu May 7 14:48:54 2015 -0400
----------------------------------------------------------------------
.../core/client/ClientConfiguration.java | 4 +-
.../core/client/ClientSideIteratorScanner.java | 4 +-
.../core/client/impl/ActiveCompactionImpl.java | 1 -
.../core/client/impl/ConditionalWriterImpl.java | 3 +-
.../core/client/impl/OfflineIterator.java | 2 +-
.../core/client/impl/ThriftScanner.java | 2 +-
.../core/client/impl/ThriftTransportPool.java | 13 +++--
.../core/client/lexicoder/PairLexicoder.java | 3 +-
.../lexicoder/impl/AbstractLexicoder.java | 3 +-
.../core/client/mapred/AbstractInputFormat.java | 11 ++---
.../client/mapred/AccumuloOutputFormat.java | 2 +-
.../core/client/mapred/InputFormatBase.java | 20 ++++----
.../client/mapred/impl/BatchInputSplit.java | 3 +-
.../client/mapreduce/AbstractInputFormat.java | 25 +++++-----
.../client/mapreduce/AccumuloInputFormat.java | 2 +-
.../core/client/mapreduce/InputFormatBase.java | 21 ++++----
.../core/client/mapreduce/RangeInputSplit.java | 2 +-
.../client/mapreduce/impl/BatchInputSplit.java | 19 ++++----
.../core/client/mapreduce/impl/SplitUtils.java | 8 ++--
.../core/client/mock/MockScannerBase.java | 7 +--
.../client/security/tokens/DelegationToken.java | 3 +-
.../accumulo/core/compaction/PatternType.java | 2 +-
.../accumulo/core/compaction/SizeType.java | 2 +-
.../accumulo/core/compaction/StringType.java | 2 +-
.../apache/accumulo/core/compaction/Type.java | 2 +-
.../accumulo/core/compaction/UIntType.java | 2 +-
.../accumulo/core/conf/ConfigurationDocGen.java | 8 ++--
.../org/apache/accumulo/core/conf/Property.java | 10 ++--
.../apache/accumulo/core/data/KeyExtent.java | 15 +++---
.../org/apache/accumulo/core/data/Mutation.java | 9 ++--
.../org/apache/accumulo/core/data/TabletId.java | 2 +
.../accumulo/core/data/impl/TabletIdImpl.java | 10 ++--
.../core/file/DispatchingFileFactory.java | 2 +-
.../core/file/blockfile/cache/HeapSize.java | 1 -
.../file/blockfile/cache/LruBlockCache.java | 17 +++----
.../apache/accumulo/core/file/rfile/RFile.java | 3 +-
.../core/iterators/AggregatingIterator.java | 1 -
.../accumulo/core/iterators/IteratorUtil.java | 3 +-
.../accumulo/core/iterators/LongCombiner.java | 4 +-
.../core/iterators/SortedKeyValueIterator.java | 10 ++--
.../core/iterators/system/MapFileIterator.java | 1 -
.../core/iterators/system/VisibilityFilter.java | 1 -
.../core/iterators/user/BigDecimalCombiner.java | 3 +-
.../core/iterators/user/RegExFilter.java | 1 -
.../iterators/user/RowEncodingIterator.java | 2 +-
.../iterators/user/TransformingIterator.java | 1 -
.../core/metadata/schema/MetadataSchema.java | 2 +-
.../accumulo/core/rpc/SaslConnectionParams.java | 7 ++-
.../core/rpc/UGIAssumingTransportFactory.java | 2 +-
.../CachingHDFSSecretKeyEncryptionStrategy.java | 12 ++---
.../security/crypto/CryptoModuleFactory.java | 8 ++--
.../NonCachingSecretKeyEncryptionStrategy.java | 14 +++---
.../org/apache/accumulo/core/util/Help.java | 2 +-
.../java/org/apache/accumulo/core/util/Jar.java | 2 +-
.../accumulo/core/util/UtilWaitThread.java | 1 -
.../core/util/format/FormatterFactory.java | 1 -
.../apache/accumulo/core/volume/VolumeImpl.java | 1 -
.../apache/accumulo/core/zookeeper/ZooUtil.java | 1 -
.../core/client/ClientConfigurationTest.java | 4 +-
.../client/lexicoder/BytesLexicoderTest.java | 2 +-
.../client/lexicoder/DateLexicoderTest.java | 2 +-
.../client/lexicoder/StringLexicoderTest.java | 2 +-
.../client/lexicoder/TextLexicoderTest.java | 2 +-
.../lexicoder/impl/AbstractLexicoderTest.java | 7 +--
.../client/lexicoder/impl/ByteUtilsTest.java | 4 +-
.../core/client/mock/MockNamespacesTest.java | 7 +--
.../apache/accumulo/core/data/MutationTest.java | 16 +++----
.../iterators/FirstEntryInRowIteratorTest.java | 3 +-
.../iterators/aggregation/NumSummationTest.java | 3 +-
.../iterators/user/RowDeletingIteratorTest.java | 2 +-
examples/simple/pom.xml | 9 ++++
.../client/CountingVerifyingReceiver.java | 2 +-
.../examples/simple/client/RowOperations.java | 1 -
.../simple/mapreduce/TokenFileWordCount.java | 1 -
.../examples/simple/reservations/ARS.java | 1 -
.../examples/simple/dirlist/CountTest.java | 1 -
.../org/apache/accumulo/fate/AdminUtil.java | 2 +-
.../org/apache/accumulo/fate/AgeOffStore.java | 1 -
.../java/org/apache/accumulo/fate/Fate.java | 1 -
.../apache/accumulo/fate/util/AddressUtil.java | 1 -
.../accumulo/fate/util/UtilWaitThread.java | 1 -
.../zookeeper/DistributedReadWriteLock.java | 1 -
.../apache/accumulo/fate/zookeeper/Retry.java | 1 -
.../fate/zookeeper/TransactionWatcher.java | 1 -
.../accumulo/fate/zookeeper/ZooCache.java | 13 ++---
.../accumulo/fate/util/AddressUtilTest.java | 1 -
maven-plugin/src/it/plugin-test/pom.xml | 7 +++
pom.xml | 50 ++++++++++++++++++++
.../org/apache/accumulo/proxy/ProxyServer.java | 3 +-
server/base/pom.xml | 9 ++++
.../org/apache/accumulo/server/TabletLevel.java | 4 +-
.../server/client/ClientServiceHandler.java | 4 +-
.../server/master/state/MetaDataStateStore.java | 2 +-
.../server/master/state/TabletStateStore.java | 7 +--
.../master/state/ZooTabletStateStore.java | 5 +-
.../server/problems/ProblemReports.java | 2 +-
.../AuthenticationTokenKeyManager.java | 3 +-
.../ZooAuthenticationKeyDistributor.java | 1 -
.../server/util/MasterMetadataUtil.java | 6 +--
.../accumulo/server/util/MetadataTableUtil.java | 8 ++--
.../accumulo/server/util/TabletIterator.java | 9 ----
.../iterators/MetadataBulkLoadFilterTest.java | 2 +-
.../security/handler/ZKAuthenticatorTest.java | 2 +-
server/gc/pom.xml | 11 +++++
.../org/apache/accumulo/gc/GCExecutable.java | 2 +-
.../gc/GarbageCollectWriteAheadLogs.java | 40 ++++++++--------
.../CloseWriteAheadLogReferences.java | 18 ++++---
server/master/pom.xml | 11 +++++
.../accumulo/master/TabletGroupWatcher.java | 13 ++---
.../accumulo/master/tableOps/ChooseDir.java | 2 +-
.../accumulo/master/tableOps/CleanUp.java | 9 ++--
.../master/tableOps/CleanUpBulkImport.java | 2 +-
.../accumulo/master/tableOps/CloneInfo.java | 2 +-
.../accumulo/master/tableOps/CloneMetadata.java | 2 +-
.../master/tableOps/CloneZookeeper.java | 2 +-
.../master/tableOps/CompactionDriver.java | 2 +-
.../master/tableOps/CompleteBulkImport.java | 2 +-
.../accumulo/master/tableOps/CreateDir.java | 2 +-
.../master/tableOps/CreateImportDir.java | 2 +-
.../accumulo/master/tableOps/ExportInfo.java | 2 +-
.../master/tableOps/FinishCancelCompaction.java | 2 +-
.../master/tableOps/FinishCloneTable.java | 2 +-
.../master/tableOps/FinishCreateNamespace.java | 2 +-
.../master/tableOps/FinishCreateTable.java | 2 +-
.../master/tableOps/FinishImportTable.java | 2 +-
.../tableOps/ImportPopulateZookeeper.java | 2 +-
.../master/tableOps/ImportSetupPermissions.java | 2 +-
.../master/tableOps/ImportedTableInfo.java | 2 +-
.../accumulo/master/tableOps/NamespaceInfo.java | 2 +-
.../master/tableOps/PopulateMetadata.java | 2 +-
.../master/tableOps/PopulateZookeeper.java | 2 +-
.../PopulateZookeeperWithNamespace.java | 2 +-
.../accumulo/master/tableOps/TableInfo.java | 2 +-
.../master/tableOps/WriteExportFiles.java | 2 +-
server/monitor/pom.xml | 9 ++++
.../monitor/servlets/trace/NullScanner.java | 3 +-
.../accumulo/monitor/ShowTraceLinkTypeTest.java | 3 +-
server/native/pom.xml | 7 +++
server/tracer/pom.xml | 11 +++++
.../accumulo/tracer/AsyncSpanReceiver.java | 4 +-
.../apache/accumulo/tracer/TraceTableStats.java | 30 +++++-------
server/tserver/pom.xml | 9 ++++
.../server/GarbageCollectionLogger.java | 3 +-
.../accumulo/tserver/MemKeyComparator.java | 2 +-
.../tserver/MemKeyConversionIterator.java | 2 +-
.../PartialMutationSkippingIterator.java | 2 +-
.../tserver/TabletIteratorEnvironment.java | 2 +-
.../apache/accumulo/tserver/TabletServer.java | 27 ++++-------
.../EverythingCompactionStrategy.java | 2 +-
.../apache/accumulo/tserver/log/DfsLogger.java | 8 ++--
.../tserver/log/TabletServerLogger.java | 4 +-
.../accumulo/tserver/tablet/MinorCompactor.java | 2 +-
.../accumulo/tserver/tablet/SplitInfo.java | 7 +--
.../apache/accumulo/tserver/tablet/Tablet.java | 44 ++++++-----------
.../org/apache/accumulo/shell/ShellOptions.java | 1 -
.../start/classloader/AccumuloClassLoader.java | 4 +-
.../accumulo/test/EstimateInMemMapOverhead.java | 1 -
.../test/InMemoryMapMemoryUsageTest.java | 2 +-
.../accumulo/test/IntObjectMemoryUsageTest.java | 2 +-
.../apache/accumulo/test/MemoryUsageTest.java | 2 +-
.../accumulo/test/MutationMemoryUsageTest.java | 2 +-
.../accumulo/test/TextMemoryUsageTest.java | 2 +-
.../test/continuous/ContinuousBatchWalker.java | 10 ----
.../accumulo/test/continuous/HistData.java | 2 +-
.../test/continuous/PrintScanTimeHistogram.java | 2 +-
.../accumulo/test/functional/AuthsIterator.java | 2 +-
.../accumulo/test/randomwalk/Environment.java | 2 +-
.../test/randomwalk/concurrent/Replication.java | 14 +++---
.../accumulo/harness/AccumuloClusterIT.java | 3 +-
.../apache/accumulo/proxy/SimpleProxyBase.java | 2 +-
.../test/BalanceWithOfflineTableIT.java | 8 ++--
.../accumulo/test/GarbageCollectWALIT.java | 2 +-
.../apache/accumulo/test/TableOperationsIT.java | 4 +-
.../java/org/apache/accumulo/test/VolumeIT.java | 2 +-
.../test/functional/AccumuloInputFormatIT.java | 14 +++---
.../accumulo/test/functional/ConcurrencyIT.java | 17 ++++---
.../accumulo/test/functional/KerberosIT.java | 6 +--
.../test/functional/KerberosProxyIT.java | 8 ++--
.../accumulo/test/functional/WALSunnyDayIT.java | 16 +++----
.../test/performance/RollWALPerformanceIT.java | 14 ++----
.../performance/metadata/FastBulkImportIT.java | 5 +-
.../test/replication/ReplicationIT.java | 4 +-
182 files changed, 529 insertions(+), 498 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/examples/simple/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/base/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
index 0a30314,29745d5..da2948d
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
@@@ -70,7 -68,7 +70,7 @@@ public class MasterMetadataUtil
private static final Logger log = LoggerFactory.getLogger(MasterMetadataUtil.class);
public static void addNewTablet(ClientContext context, KeyExtent extent, String path, TServerInstance location, Map<FileRef,DataFileValue> datafileSizes,
- Multimap<Long, FileRef> bulkLoadedFiles, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
- Map<FileRef,Long> bulkLoadedFiles, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
++ Multimap<Long,FileRef> bulkLoadedFiles, String time, long lastFlushID, long lastCompactID, ZooLock zooLock) {
Mutation m = extent.getPrevRowUpdateMutation();
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
@@@ -89,9 -87,9 +89,9 @@@
m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
}
- for (Entry<Long, FileRef> entry : bulkLoadedFiles.entries()) {
- for (Entry<FileRef,Long> entry : bulkLoadedFiles.entrySet()) {
- byte[] tidBytes = Long.toString(entry.getValue()).getBytes();
- m.put(TabletsSection.BulkFileColumnFamily.NAME, entry.getKey().meta(), new Value(tidBytes));
++ for (Entry<Long,FileRef> entry : bulkLoadedFiles.entries()) {
+ byte[] tidBytes = Long.toString(entry.getKey()).getBytes();
+ m.put(TabletsSection.BulkFileColumnFamily.NAME, entry.getValue().meta(), new Value(tidBytes));
}
MetadataTableUtil.update(context, zooLock, m, extent);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 58ca528,f53f6a5..5d5415d
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -926,9 -924,9 +926,9 @@@ public class MetadataTableUtil
}
}
- public static Multimap<Long, FileRef> getBulkFilesLoaded(ClientContext context, KeyExtent extent) throws IOException {
- public static Map<FileRef,Long> getBulkFilesLoaded(ClientContext context, KeyExtent extent) throws IOException {
++ public static Multimap<Long,FileRef> getBulkFilesLoaded(ClientContext context, KeyExtent extent) throws IOException {
Text metadataRow = extent.getMetadataEntry();
- Multimap<Long, FileRef> ret = HashMultimap.create();
- Map<FileRef,Long> ret = new HashMap<FileRef,Long>();
++ Multimap<Long,FileRef> ret = HashMultimap.create();
VolumeManager fs = VolumeManagerImpl.get();
Scanner scanner = new ScannerImpl(context, extent.isMeta() ? RootTable.ID : MetadataTable.ID, Authorizations.EMPTY);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/gc/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/master/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/monitor/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/native/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tracer/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tserver/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
index d866d99,f8f2183..f2111c7
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
@@@ -41,9 -39,8 +41,10 @@@ final public class SplitInfo
private final long initFlushID;
private final long initCompactID;
private final TServerInstance lastLocation;
- private final Multimap<Long, FileRef> bulkImported;
++ private final Multimap<Long,FileRef> bulkImported;
- SplitInfo(String d, SortedMap<FileRef,DataFileValue> dfv, String time, long initFlushID, long initCompactID, TServerInstance lastLocation, Multimap<Long, FileRef> bulkImported) {
- SplitInfo(String d, SortedMap<FileRef,DataFileValue> dfv, String time, long initFlushID, long initCompactID, TServerInstance lastLocation) {
++ SplitInfo(String d, SortedMap<FileRef,DataFileValue> dfv, String time, long initFlushID, long initCompactID, TServerInstance lastLocation,
++ Multimap<Long,FileRef> bulkImported) {
this.dir = d;
this.datafiles = dfv;
this.time = time;
@@@ -77,8 -73,4 +78,8 @@@
return lastLocation;
}
- public Multimap<Long, FileRef> getBulkImported() {
++ public Multimap<Long,FileRef> getBulkImported() {
+ return bulkImported;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 0b2d5e3,17864be..5de3236
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -253,9 -243,6 +253,8 @@@ public class Tablet implements TabletCo
private final ConfigurationObserver configObserver;
- private final Cache<Long, List<FileRef> > bulkImported = CacheBuilder.newBuilder().expireAfterAccess(4, TimeUnit.HOURS).build();
-
++ private final Cache<Long,List<FileRef>> bulkImported = CacheBuilder.newBuilder().expireAfterAccess(4, TimeUnit.HOURS).build();
+
private final int logId;
@Override
@@@ -324,8 -311,8 +323,8 @@@
}
private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager trm, SortedMap<FileRef,DataFileValue> datafiles,
- String time, long initFlushID, long initCompactID, TServerInstance lastLocation, Multimap<Long, FileRef> bulkImported) throws IOException {
- String time, long initFlushID, long initCompactID, TServerInstance lastLocation) throws IOException {
- this(tabletServer, extent, location, trm, NO_LOG_ENTRIES, datafiles, time, lastLocation, new HashSet<FileRef>(), initFlushID, initCompactID);
++ String time, long initFlushID, long initCompactID, TServerInstance lastLocation, Multimap<Long,FileRef> bulkImported) throws IOException {
+ this(tabletServer, extent, location, trm, NO_LOG_ENTRIES, datafiles, time, lastLocation, new HashSet<FileRef>(), initFlushID, initCompactID, bulkImported);
}
private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) {
@@@ -447,45 -445,20 +446,30 @@@
return null;
}
- private static Multimap<Long, FileRef> lookupBulkImported(SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) {
- Multimap<Long, FileRef> result = HashMultimap.create();
++ private static Multimap<Long,FileRef> lookupBulkImported(SortedMap<Key,Value> tabletsKeyValues, VolumeManager fs) {
++ Multimap<Long,FileRef> result = HashMultimap.create();
+ for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+ if (entry.getKey().getColumnFamily().compareTo(BulkFileColumnFamily.NAME) == 0) {
+ result.put(Long.decode(entry.getValue().toString()), new FileRef(fs, entry.getKey()));
+ }
+ }
+ return result;
+ }
+
public Tablet(TabletServer tabletServer, KeyExtent extent, Text location, TabletResourceManager trm, SortedMap<Key,Value> tabletsKeyValues)
throws IOException {
- this(tabletServer, extent, location, trm,
- lookupLogEntries(tabletsKeyValues, tabletServer, extent),
- lookupDatafiles(tabletServer, tabletServer.getFileSystem(), extent, tabletsKeyValues),
- lookupTime(tabletServer.getConfiguration(), extent, tabletsKeyValues),
- lookupLastServer(tabletsKeyValues),
- lookupScanFiles(tabletsKeyValues, tabletServer.getFileSystem()),
- lookupFlushID(tabletsKeyValues),
- lookupCompactID(tabletsKeyValues),
- lookupBulkImported(tabletsKeyValues, tabletServer.getFileSystem()));
- this(tabletServer, extent, location, trm, lookupLogEntries(tabletServer, extent, tabletsKeyValues), lookupDatafiles(tabletServer,
- tabletServer.getFileSystem(), extent, tabletsKeyValues), lookupTime(tabletServer.getConfiguration(), extent, tabletsKeyValues), lookupLastServer(
- extent, tabletsKeyValues), lookupScanFiles(extent, tabletsKeyValues, tabletServer.getFileSystem()), lookupFlushID(extent, tabletsKeyValues),
- lookupCompactID(extent, tabletsKeyValues));
++ this(tabletServer, extent, location, trm, lookupLogEntries(tabletsKeyValues, tabletServer, extent), lookupDatafiles(tabletServer,
++ tabletServer.getFileSystem(), extent, tabletsKeyValues), lookupTime(tabletServer.getConfiguration(), extent, tabletsKeyValues),
++ lookupLastServer(tabletsKeyValues), lookupScanFiles(tabletsKeyValues, tabletServer.getFileSystem()), lookupFlushID(tabletsKeyValues),
++ lookupCompactID(tabletsKeyValues), lookupBulkImported(tabletsKeyValues, tabletServer.getFileSystem()));
}
/**
* yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time
*/
- private Tablet(
- final TabletServer tabletServer,
- final KeyExtent extent,
- final Text location,
- final TabletResourceManager trm,
- final List<LogEntry> rawLogEntries,
- final SortedMap<FileRef,DataFileValue> rawDatafiles,
- String time,
- final TServerInstance lastLocation,
- final Set<FileRef> scanFiles,
- final long initFlushID,
- final long initCompactID,
- final Multimap<Long, FileRef> bulkImported) throws IOException {
+ private Tablet(final TabletServer tabletServer, final KeyExtent extent, final Text location, final TabletResourceManager trm,
+ final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue> rawDatafiles, String time, final TServerInstance lastLocation,
- Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException {
++ final Set<FileRef> scanFiles, final long initFlushID, final long initCompactID, final Multimap<Long,FileRef> bulkImported) throws IOException {
TableConfiguration tblConf = tabletServer.getTableConfiguration(extent);
if (null == tblConf) {
@@@ -675,8 -646,8 +659,8 @@@
currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), logEntry.filename, logEntry.getColumnQualifier().toString()));
}
- log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + entriesUsedOnTablet.get() + " mutations applied, " + getTabletMemory().getNumEntries()
- log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + getTabletMemory().getNumEntries()
-- + " entries created)");
++ log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + entriesUsedOnTablet.get() + " mutations applied, "
++ + getTabletMemory().getNumEntries() + " entries created)");
}
String contextName = tableConfiguration.get(Property.TABLE_CLASSPATH);
@@@ -2295,7 -2266,7 +2279,7 @@@
// it is possible that some of the bulk loading flags will be deleted after being read below because the bulk load
// finishes.... therefore split could propagate load flags for a finished bulk load... there is a special iterator
// on the metadata table to clean up this type of garbage
- Multimap<Long, FileRef> bulkLoadedFiles = MetadataTableUtil.getBulkFilesLoaded(getTabletServer(), extent);
- Map<FileRef,Long> bulkLoadedFiles = MetadataTableUtil.getBulkFilesLoaded(getTabletServer(), extent);
++ Multimap<Long,FileRef> bulkLoadedFiles = MetadataTableUtil.getBulkFilesLoaded(getTabletServer(), extent);
MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, getTabletServer(), getTabletServer().getLock());
MasterMetadataUtil.addNewTablet(getTabletServer(), low, lowDirectory, getTabletServer().getTabletSession(), lowDatafileSizes, bulkLoadedFiles, time,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
index d1ecfe9,0000000..05c907c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
@@@ -1,104 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.performance.metadata;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+// ACCUMULO-3327
+public class FastBulkImportIT extends ConfigurableMacIT {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
-
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(3);
+ cfg.setProperty(Property.TSERV_BULK_ASSIGNMENT_THREADS, "5");
+ cfg.setProperty(Property.TSERV_BULK_PROCESS_THREADS, "5");
+ cfg.setProperty(Property.TABLE_MAJC_RATIO, "9999");
+ cfg.setProperty(Property.TABLE_FILE_MAX, "9999");
+ }
+
-
+ @Test
+ public void test() throws Exception {
+ log.info("Creating table");
+ final String tableName = getUniqueNames(1)[0];
+ final Connector c = getConnector();
+ c.tableOperations().create(tableName);
+ log.info("Adding splits");
+ SortedSet<Text> splits = new TreeSet<>();
+ for (int i = 1; i < 0xfff; i += 7) {
+ splits.add(new Text(Integer.toHexString(i)));
+ }
+ c.tableOperations().addSplits(tableName, splits);
+
+ log.info("Creating bulk import files");
+ FileSystem fs = getCluster().getFileSystem();
+ Path basePath = getCluster().getTemporaryPath();
+ CachedConfiguration.setInstance(fs.getConf());
+
+ Path base = new Path(basePath, "testBulkFail_" + tableName);
+ fs.delete(base, true);
+ fs.mkdirs(base);
+ Path bulkFailures = new Path(base, "failures");
+ Path files = new Path(base, "files");
+ fs.mkdirs(bulkFailures);
+ fs.mkdirs(files);
+ for (int i = 0; i < 100; i++) {
- FileSKVWriter writer = FileOperations.getInstance().openWriter(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration());
++ FileSKVWriter writer = FileOperations.getInstance().openWriter(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf(),
++ AccumuloConfiguration.getDefaultConfiguration());
+ writer.startDefaultLocalityGroup();
+ for (int j = 0x100; j < 0xfff; j += 3) {
+ writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
+ }
+ writer.close();
+ }
+ log.info("Waiting for balance");
+ c.instanceOperations().waitForBalance();
+
+ log.info("Bulk importing files");
+ long now = System.currentTimeMillis();
+ c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), true);
+ double diffSeconds = (System.currentTimeMillis() - now) / 1000.;
+ log.info(String.format("Import took %.2f seconds", diffSeconds));
+ assertTrue(diffSeconds < 30);
+ }
+
+}