You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2015/04/25 01:20:54 UTC
[08/34] accumulo git commit: ACCUMULO-3638 merge master branch
ACCUMULO-3638 merge master branch
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4635de86
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4635de86
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4635de86
Branch: refs/heads/master
Commit: 4635de8671a62f0a412da775db05519f6831daaa
Parents: 98c3cef 9f108c0
Author: Eric C. Newton <er...@gmail.com>
Authored: Tue Mar 10 16:16:53 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Tue Mar 10 16:16:53 2015 -0400
----------------------------------------------------------------------
core/pom.xml | 7 --
core/src/main/findbugs/exclude-filter.xml | 67 +++++++++++++-----
.../client/admin/CompactionStrategyConfig.java | 5 ++
.../client/impl/MultiTableBatchWriterImpl.java | 13 ++--
.../core/client/impl/TabletLocatorImpl.java | 41 +++++++----
.../client/impl/TabletServerBatchWriter.java | 45 ++++++++----
.../core/client/mock/MockNamespace.java | 2 +-
.../client/security/tokens/DelegationToken.java | 16 ++---
.../client/security/tokens/PasswordToken.java | 13 ++--
.../apache/accumulo/core/conf/PropertyType.java | 2 +-
.../org/apache/accumulo/core/data/Value.java | 1 +
.../apache/accumulo/core/volume/VolumeImpl.java | 6 ++
.../accumulo/core/cli/TestClientOpts.java | 2 +-
.../core/file/rfile/RFileMetricsTest.java | 73 +++++++++-----------
.../iterators/user/ColumnSliceFilterTest.java | 2 +-
.../core/iterators/user/RowFilterTest.java | 2 +-
.../simple/src/main/findbugs/exclude-filter.xml | 18 +++++
.../examples/simple/mapreduce/RowHash.java | 2 +-
fate/src/main/findbugs/exclude-filter.xml | 18 +++++
.../accumulo/fate/zookeeper/ZooReader.java | 14 ++--
.../zookeeper/DistributedReadWriteLockTest.java | 17 +++--
.../src/main/findbugs/exclude-filter.xml | 18 +++++
.../src/main/findbugs/exclude-filter.xml | 30 ++++++++
pom.xml | 3 +-
proxy/pom.xml | 13 ----
proxy/src/main/findbugs/exclude-filter.xml | 7 +-
.../base/src/main/findbugs/exclude-filter.xml | 26 +++++++
.../server/master/balancer/GroupBalancer.java | 6 ++
.../master/state/TabletLocationState.java | 3 +-
.../server/metrics/MetricsConfiguration.java | 7 +-
.../TCredentialsUpdatingInvocationHandler.java | 5 +-
.../server/security/SecurityOperation.java | 7 --
.../security/delegation/AuthenticationKey.java | 11 +--
.../security/handler/KerberosAuthenticator.java | 4 +-
.../server/watcher/Log4jConfiguration.java | 5 +-
.../accumulo/server/ServerConstantsTest.java | 12 +++-
.../AuthenticationTokenKeyManagerTest.java | 34 +++++----
server/gc/src/main/findbugs/exclude-filter.xml | 18 +++++
.../gc/GarbageCollectWriteAheadLogs.java | 2 +-
.../master/src/main/findbugs/exclude-filter.xml | 23 ++++++
.../java/org/apache/accumulo/master/Master.java | 24 ++++---
.../RemoveCompleteReplicationRecords.java | 3 +
.../src/main/findbugs/exclude-filter.xml | 18 +++++
.../monitor/servlets/OperationServlet.java | 53 +++++++++-----
server/tracer/pom.xml | 13 ----
.../tracer/src/main/findbugs/exclude-filter.xml | 7 +-
.../src/main/findbugs/exclude-filter.xml | 24 +++++++
.../accumulo/server/logger/LogFileKey.java | 25 -------
.../accumulo/server/logger/LogFileValue.java | 25 -------
.../accumulo/tserver/log/LocalWALRecovery.java | 4 +-
.../replication/AccumuloReplicaSystem.java | 19 ++++-
.../apache/accumulo/tserver/scan/ScanTask.java | 30 ++++++--
.../apache/accumulo/tserver/tablet/Tablet.java | 14 ++--
.../tserver/tablet/TabletCommitter.java | 3 +
shell/src/main/findbugs/exclude-filter.xml | 18 +++++
.../apache/accumulo/shell/ShellConfigTest.java | 29 ++++++--
.../org/apache/accumulo/shell/ShellTest.java | 11 ++-
.../shell/commands/FormatterCommandTest.java | 6 +-
start/src/main/findbugs/exclude-filter.xml | 18 +++++
.../start/classloader/vfs/MiniDFSUtil.java | 4 ++
test/src/main/findbugs/exclude-filter.xml | 30 ++++++++
.../server/security/SystemCredentialsIT.java | 2 +
.../test/MasterRepairsDualAssignmentIT.java | 2 +-
.../accumulo/test/MultiTableRecoveryIT.java | 3 +-
.../java/org/apache/accumulo/test/UsersIT.java | 60 ++++++++++++++++
.../apache/accumulo/test/functional/BulkIT.java | 13 ++--
.../test/functional/ConfigurableMacIT.java | 4 +-
.../accumulo/test/functional/MonitorSslIT.java | 40 ++++++-----
.../accumulo/test/functional/PermissionsIT.java | 43 ++++++++----
.../apache/accumulo/test/functional/SslIT.java | 5 +-
trace/src/main/findbugs/exclude-filter.xml | 26 +++++++
71 files changed, 810 insertions(+), 366 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
index 3ece3c9,b24b562..ebad2c8
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
@@@ -33,8 -32,6 +32,8 @@@ import org.apache.hadoop.io.Text
*/
public class TabletLocationState {
- private static final Logger log = Logger.getLogger(TabletLocationState.class);
++ // private static final Logger log = Logger.getLogger(TabletLocationState.class);
+
static public class BadLocationStateException extends Exception {
private static final long serialVersionUID = 1L;
private Text metadataTableEntry;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 444789b,35c60d6..f44a9d1
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -214,39 -268,76 +214,39 @@@ public class GarbageCollectWriteAheadLo
}
}
}
-
- return 0;
+ return status.currentLog.deleted;
}
- /**
- * Converts a list of paths to their corresponding strings.
- *
- * @param paths
- * list of paths
- * @return string forms of paths
- */
- static List<String> paths2strings(List<Path> paths) {
- List<String> result = new ArrayList<String>(paths.size());
- for (Path path : paths)
- result.add(path.toString());
- return result;
- }
+ private long removeMetadataEntries(Map<TServerInstance, Set<Path> > candidates, GCStatus status, Set<TServerInstance> liveServers) throws IOException, KeeperException,
+ InterruptedException {
- /**
- * Reverses the given mapping of file paths to servers. The returned map provides a list of file paths for each server. Any path whose name is not in the
- * mapping of file names to paths is skipped.
- *
- * @param fileToServerMap
- * map of file paths to servers
- * @param nameToFileMap
- * map of file names to paths
- * @return map of servers to lists of file paths
- */
- static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) {
- Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>();
- for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) {
- if (!nameToFileMap.containsKey(fileServer.getKey().getName()))
- continue;
- ArrayList<Path> files = result.get(fileServer.getValue());
- if (files == null) {
- files = new ArrayList<Path>();
- result.put(fileServer.getValue(), files);
+ // remove any entries if there's a log reference, or a tablet is still assigned to the dead server
+
+ Map<Path, TServerInstance> walToDeadServer = new HashMap<>();
+ for (Entry<TServerInstance,Set<Path>> entry : candidates.entrySet()) {
+ for (Path file : entry.getValue()) {
+ walToDeadServer.put(file, entry.getKey());
}
- files.add(fileServer.getKey());
}
- return result;
- }
-
- protected int removeMetadataEntries(Map<String,Path> nameToFileMap, Map<String,Path> sortedWALogs, GCStatus status) throws IOException, KeeperException,
- InterruptedException {
- int count = 0;
- Iterator<LogEntry> iterator = MetadataTableUtil.getLogEntries(context);
-
- // For each WAL reference in the metadata table
- while (iterator.hasNext()) {
- // Each metadata reference has at least one WAL file
- for (String entry : iterator.next().logSet) {
- // old style WALs will have the IP:Port of their logger and new style will either be a Path either absolute or relative, in all cases
- // the last "/" will mark a UUID file name.
- String uuid = entry.substring(entry.lastIndexOf("/") + 1);
- if (!isUUID(uuid)) {
- // fully expect this to be a uuid, if its not then something is wrong and walog GC should not proceed!
- throw new IllegalArgumentException("Expected uuid, but got " + uuid + " from " + entry);
- }
-
- Path pathFromNN = nameToFileMap.remove(uuid);
- if (pathFromNN != null) {
- status.currentLog.inUse++;
- sortedWALogs.remove(uuid);
+ long count = 0;
+ RootTabletStateStore root = new RootTabletStateStore(context);
+ MetaDataStateStore meta = new MetaDataStateStore(context);
+ Iterator<TabletLocationState> states = Iterators.concat(root.iterator(), meta.iterator());
+ while (states.hasNext()) {
+ count++;
+ TabletLocationState state = states.next();
+ if (state.getState(liveServers) == TabletState.ASSIGNED_TO_DEAD_SERVER) {
+ candidates.remove(state.current);
+ }
+ for (Collection<String> wals : state.walogs) {
+ for (String wal : wals) {
- TServerInstance dead = walToDeadServer.get(wal);
++ TServerInstance dead = walToDeadServer.get(new Path(wal));
+ if (dead != null) {
+ candidates.get(dead).remove(wal);
+ }
}
-
- count++;
}
}
-
return count;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/findbugs/exclude-filter.xml
----------------------------------------------------------------------
diff --cc server/tserver/src/main/findbugs/exclude-filter.xml
index 0000000,aec413a..45f6a78
mode 000000,100644..100644
--- a/server/tserver/src/main/findbugs/exclude-filter.xml
+++ b/server/tserver/src/main/findbugs/exclude-filter.xml
@@@ -1,0 -1,24 +1,24 @@@
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <FindBugsFilter>
+ <Match>
+ <!-- locking is confusing, but probably correct -->
+ <Class name="org.apache.accumulo.tserver.tablet.Tablet" />
- <Method name="beginUpdatingLogsUsed" params="org.apache.accumulo.tserver.InMemoryMap,java.util.Collection,boolean" returns="boolean" />
++ <Method name="beginUpdatingLogsUsed" params="org.apache.accumulo.tserver.InMemoryMap,org.apache.accumulo.tserver.log.DfsLogger,boolean" returns="boolean" />
+ <Bug code="UL" pattern="UL_UNRELEASED_LOCK" />
+ </Match>
+ </FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 0c1edfa,95fe24a..e964719
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -2442,8 -2441,9 +2440,9 @@@ public class Tablet implements TabletCo
return currentLogs.size();
}
+ /* don't release the lock if this method returns true for success; instead, the caller should clean up by calling finishUpdatingLogsUsed() */
@Override
- public boolean beginUpdatingLogsUsed(InMemoryMap memTable, Collection<DfsLogger> more, boolean mincFinish) {
+ public boolean beginUpdatingLogsUsed(InMemoryMap memTable, DfsLogger more, boolean mincFinish) {
boolean releaseLock = true;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
index 39bde5c,4bc05a6..91a955e
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
@@@ -34,7 -35,10 +34,10 @@@ public interface TabletCommitter
void commit(CommitSession commitSession, List<Mutation> mutations);
+ /**
+ * If this method returns true, the caller must call {@link #finishUpdatingLogsUsed()} to clean up
+ */
- boolean beginUpdatingLogsUsed(InMemoryMap memTable, Collection<DfsLogger> copy, boolean mincFinish);
+ boolean beginUpdatingLogsUsed(InMemoryMap memTable, DfsLogger copy, boolean mincFinish);
void finishUpdatingLogsUsed();