You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/04/14 08:18:19 UTC
[01/21] hadoop git commit: YARN-4794. Deadlock in NMClientImpl.
Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-1312 3a490d6fc -> 648cc93f6
YARN-4794. Deadlock in NMClientImpl. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff722bbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff722bbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff722bbb
Branch: refs/heads/HDFS-1312
Commit: ff722bbbdfaf9715643507628e94ece26d904b9d
Parents: 44bbc50
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Apr 12 09:59:33 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Apr 12 09:59:33 2016 +0530
----------------------------------------------------------------------
.../yarn/client/api/impl/NMClientImpl.java | 36 ++++----------------
1 file changed, 7 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff722bbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
index e047368..dc92cda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
@@ -171,8 +171,6 @@ public class NMClientImpl extends NMClient {
throw RPCUtil.getRemoteException("Container "
+ startedContainer.containerId.toString() + " is already started");
}
- startedContainers
- .put(startedContainer.getContainerId(), startedContainer);
}
@Override
@@ -182,7 +180,8 @@ public class NMClientImpl extends NMClient {
// Do synchronization on StartedContainer to prevent race condition
// between startContainer and stopContainer only when startContainer is
// in progress for a given container.
- StartedContainer startingContainer = createStartedContainer(container);
+ StartedContainer startingContainer =
+ new StartedContainer(container.getId(), container.getNodeId());
synchronized (startingContainer) {
addStartingContainer(startingContainer);
@@ -210,18 +209,14 @@ public class NMClientImpl extends NMClient {
}
allServiceResponse = response.getAllServicesMetaData();
startingContainer.state = ContainerState.RUNNING;
- } catch (YarnException e) {
+ } catch (YarnException | IOException e) {
startingContainer.state = ContainerState.COMPLETE;
// Remove the started container if it failed to start
- removeStartedContainer(startingContainer);
- throw e;
- } catch (IOException e) {
- startingContainer.state = ContainerState.COMPLETE;
- removeStartedContainer(startingContainer);
+ startedContainers.remove(startingContainer.containerId);
throw e;
} catch (Throwable t) {
startingContainer.state = ContainerState.COMPLETE;
- removeStartedContainer(startingContainer);
+ startedContainers.remove(startingContainer.containerId);
throw RPCUtil.getRemoteException(t);
} finally {
if (proxy != null) {
@@ -263,7 +258,7 @@ public class NMClientImpl extends NMClient {
@Override
public void stopContainer(ContainerId containerId, NodeId nodeId)
throws YarnException, IOException {
- StartedContainer startedContainer = getStartedContainer(containerId);
+ StartedContainer startedContainer = startedContainers.get(containerId);
// Only allow one request of stopping the container to move forward
// When entering the block, check whether the precursor has already stopped
@@ -276,7 +271,7 @@ public class NMClientImpl extends NMClient {
stopContainerInternal(containerId, nodeId);
// Only after successful
startedContainer.state = ContainerState.COMPLETE;
- removeStartedContainer(startedContainer);
+ startedContainers.remove(startedContainer.containerId);
}
} else {
stopContainerInternal(containerId, nodeId);
@@ -333,23 +328,6 @@ public class NMClientImpl extends NMClient {
}
}
}
-
- protected synchronized StartedContainer createStartedContainer(
- Container container) throws YarnException, IOException {
- StartedContainer startedContainer = new StartedContainer(container.getId(),
- container.getNodeId());
- return startedContainer;
- }
-
- protected synchronized void
- removeStartedContainer(StartedContainer container) {
- startedContainers.remove(container.containerId);
- }
-
- protected synchronized StartedContainer getStartedContainer(
- ContainerId containerId) {
- return startedContainers.get(containerId);
- }
public AtomicBoolean getCleanupRunningContainers() {
return cleanupRunningContainers;
[13/21] hadoop git commit: YARN-4886. Add HDFS caller context for
EntityGroupFSTimelineStore. Contributed by Li Lu
Posted by ae...@apache.org.
YARN-4886. Add HDFS caller context for EntityGroupFSTimelineStore. Contributed by Li Lu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0cb4267
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0cb4267
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0cb4267
Branch: refs/heads/HDFS-1312
Commit: e0cb426758b3d716ff143f723fc16ef2f1e4971b
Parents: d2f3bbc
Author: Xuan <xg...@apache.org>
Authored: Wed Apr 13 10:38:17 2016 -0700
Committer: Xuan <xg...@apache.org>
Committed: Wed Apr 13 10:38:17 2016 -0700
----------------------------------------------------------------------
.../yarn/server/timeline/EntityGroupFSTimelineStore.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cb4267/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
index 34a2072..18b8951 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -86,6 +87,8 @@ public class EntityGroupFSTimelineStore extends CompositeService
static final String SUMMARY_LOG_PREFIX = "summarylog-";
static final String ENTITY_LOG_PREFIX = "entitylog-";
+ static final String ATS_V15_SERVER_DFS_CALLER_CTXT = "yarn_ats_server_v1_5";
+
private static final Logger LOG = LoggerFactory.getLogger(
EntityGroupFSTimelineStore.class);
private static final FsPermission ACTIVE_DIR_PERMISSION =
@@ -187,6 +190,8 @@ public class EntityGroupFSTimelineStore extends CompositeService
YarnConfiguration
.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT));
fs = activeRootPath.getFileSystem(conf);
+ CallerContext.setCurrent(
+ new CallerContext.Builder(ATS_V15_SERVER_DFS_CALLER_CTXT).build());
super.serviceInit(conf);
}
@@ -304,6 +309,7 @@ public class EntityGroupFSTimelineStore extends CompositeService
ServiceOperations.stopQuietly(cacheItem.getStore());
}
}
+ CallerContext.setCurrent(null);
super.serviceStop();
}
[05/21] hadoop git commit: HDFS-8356. Document missing properties in
hdfs-default.xml. Contributed by Ray Chiang.
Posted by ae...@apache.org.
HDFS-8356. Document missing properties in hdfs-default.xml. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/209303be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/209303be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/209303be
Branch: refs/heads/HDFS-1312
Commit: 209303be3a4d038b420ae3c11230d419bba07575
Parents: a9a607f
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Apr 12 15:14:34 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Apr 12 15:26:50 2016 +0900
----------------------------------------------------------------------
.../src/site/markdown/DeprecatedProperties.md | 1 +
.../src/main/resources/hdfs-default.xml | 1033 +++++++++++++++++-
.../hadoop/tools/TestHdfsConfigFields.java | 87 +-
3 files changed, 1108 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209303be/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index f8bc95b..62e1791 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -51,6 +51,7 @@ The following table lists the configuration property names that are deprecated i
| dfs.secondary.http.address | dfs.namenode.secondary.http-address |
| dfs.socket.timeout | dfs.client.socket-timeout |
| dfs.umaskmode | fs.permissions.umask-mode |
+| dfs.web.ugi | hadoop.http.staticuser.user |
| dfs.write.packet.size | dfs.client-write-packet-size |
| fs.checkpoint.dir | dfs.namenode.checkpoint.dir |
| fs.checkpoint.edits.dir | dfs.namenode.checkpoint.edits.dir |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209303be/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 35dce0e..897ec63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -264,7 +264,7 @@
dfs.datanode.dns.interface.
</description>
</property>
-
+
<property>
<name>dfs.datanode.dns.nameserver</name>
<value>default</value>
@@ -276,7 +276,7 @@
dfs.datanode.dns.nameserver.
</description>
</property>
-
+
<property>
<name>dfs.namenode.backup.address</name>
<value>0.0.0.0:50100</value>
@@ -285,7 +285,7 @@
If the port is 0 then the server will start on a free port.
</description>
</property>
-
+
<property>
<name>dfs.namenode.backup.http-address</name>
<value>0.0.0.0:50105</value>
@@ -1441,6 +1441,13 @@
<description>
The prefix for a given nameservice, contains a comma-separated
list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE).
+
+ Unique identifiers for each NameNode in the nameservice, delimited by
+ commas. This will be used by DataNodes to determine all the NameNodes
+ in the cluster. For example, if you used “mycluster” as the nameservice
+ ID previously, and you wanted to use “nn1” and “nn2” as the individual
+ IDs of the NameNodes, you would configure a property
+ dfs.ha.namenodes.mycluster, and its value "nn1,nn2".
</description>
</property>
@@ -3036,4 +3043,1024 @@
refreshes the configuration files used by the class.
</description>
</property>
+
+<property>
+ <name>datanode.https.port</name>
+ <value>50475</value>
+ <description>
+ HTTPS port for DataNode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.balancer.dispatcherThreads</name>
+ <value>200</value>
+ <description>
+ Size of the thread pool for the HDFS balancer block mover.
+ dispatchExecutor
+ </description>
+</property>
+
+<property>
+ <name>dfs.balancer.movedWinWidth</name>
+ <value>5400000</value>
+ <description>
+ Window of time in ms for the HDFS balancer tracking blocks and its
+ locations.
+ </description>
+</property>
+
+<property>
+ <name>dfs.balancer.moverThreads</name>
+ <value>1000</value>
+ <description>
+ Thread pool size for executing block moves.
+ moverThreadAllocator
+ </description>
+</property>
+
+<property>
+ <name>dfs.balancer.max-size-to-move</name>
+ <value>10737418240</value>
+ <description>
+ Maximum number of bytes that can be moved by the balancer in a single
+ thread.
+ </description>
+</property>
+
+<property>
+ <name>dfs.balancer.getBlocks.min-block-size</name>
+ <value>10485760</value>
+ <description>
+ Minimum block threshold size in bytes to ignore when fetching a source's
+ block list.
+ </description>
+</property>
+
+<property>
+ <name>dfs.balancer.getBlocks.size</name>
+ <value>2147483648</value>
+ <description>
+ Total size in bytes of Datanode blocks to get when fetching a source's
+ block list.
+ </description>
+</property>
+
+<property>
+ <name>dfs.block.invalidate.limit</name>
+ <value>1000</value>
+ <description>
+ Limit on the list of invalidated block list kept by the Namenode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.block.misreplication.processing.limit</name>
+ <value>10000</value>
+ <description>
+ Maximum number of blocks to process for initializing replication queues.
+ </description>
+</property>
+
+<property>
+ <name>dfs.block.placement.ec.classname</name>
+ <value>org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant</value>
+ <description>
+ Placement policy class for striped files.
+ Defaults to BlockPlacementPolicyRackFaultTolerant.class
+ </description>
+</property>
+
+<property>
+ <name>dfs.block.replicator.classname</name>
+ <value>org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault</value>
+ <description>
+ Class representing block placement policy for non-striped files.
+ </description>
+</property>
+
+<property>
+ <name>dfs.blockreport.incremental.intervalMsec</name>
+ <value>0</value>
+ <description>
+ If set to a positive integer, the value in ms to wait between sending
+ incremental block reports from the Datanode to the Namenode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.checksum.type</name>
+ <value>CRC32C</value>
+ <description>
+ Checksum type
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.block.write.locateFollowingBlock.retries</name>
+ <value>5</value>
+ <description>
+ Number of retries to use when finding the next block during HDFS writes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.failover.proxy.provider</name>
+ <value></value>
+ <description>
+ The prefix (plus a required nameservice ID) for the class name of the
+ configured Failover proxy provider for the host. For more detailed
+ information, please consult the "Configuration Details" section of
+ the HDFS High Availability documentation.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.key.provider.cache.expiry</name>
+ <value>864000000</value>
+ <description>
+ DFS client security key cache expiration in milliseconds.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.max.block.acquire.failures</name>
+ <value>3</value>
+ <description>
+ Maximum failures allowed when trying to get block information from a specific datanode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.read.prefetch.size</name>
+ <value></value>
+ <description>
+ The number of bytes for the DFSClient will fetch from the Namenode
+ during a read operation. Defaults to 10 * ${dfs.blocksize}.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.read.short.circuit.replica.stale.threshold.ms</name>
+ <value>1800000</value>
+ <description>
+ Threshold in milliseconds for read entries during short-circuit local reads.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.read.shortcircuit.buffer.size</name>
+ <value>1048576</value>
+ <description>
+ Buffer size in bytes for short-circuit local reads.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.replica.accessor.builder.classes</name>
+ <value></value>
+ <description>
+ Comma-separated classes for building ReplicaAccessor. If the classes
+ are specified, client will use external BlockReader that uses the
+ ReplicaAccessor built by the builder.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.retry.interval-ms.get-last-block-length</name>
+ <value>4000</value>
+ <description>
+ Retry interval in milliseconds to wait between retries in getting
+ block lengths from the datanodes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.retry.max.attempts</name>
+ <value>10</value>
+ <description>
+ Max retry attempts for DFSClient talking to namenodes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.retry.policy.enabled</name>
+ <value>false</value>
+ <description>
+ If true, turns on DFSClient retry policy.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.retry.policy.spec</name>
+ <value>10000,6,60000,10</value>
+ <description>
+ Set to pairs of timeouts and retries for DFSClient.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.retry.times.get-last-block-length</name>
+ <value>3</value>
+ <description>
+ Number of retries for calls to fetchLocatedBlocksAndGetLastBlockLength().
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.retry.window.base</name>
+ <value>3000</value>
+ <description>
+ Base time window in ms for DFSClient retries. For each retry attempt,
+ this value is extended linearly (e.g. 3000 ms for first attempt and
+ first retry, 6000 ms for second retry, 9000 ms for third retry, etc.).
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.socket-timeout</name>
+ <value>60000</value>
+ <description>
+ Default timeout value in milliseconds for all sockets.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.socketcache.capacity</name>
+ <value>16</value>
+ <description>
+ Socket cache capacity (in entries) for short-circuit reads.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.socketcache.expiryMsec</name>
+ <value>3000</value>
+ <description>
+ Socket cache expiration for short-circuit reads in msec.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.test.drop.namenode.response.number</name>
+ <value>0</value>
+ <description>
+ The number of Namenode responses dropped by DFSClient for each RPC call. Used
+ for testing the NN retry cache.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.hedged.read.threadpool.size</name>
+ <value>0</value>
+ <description>
+ Support 'hedged' reads in DFSClient. To enable this feature, set the parameter
+ to a positive number. The threadpool size is how many threads to dedicate
+ to the running of these 'hedged', concurrent reads in your client.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.hedged.read.threshold.millis</name>
+ <value>500</value>
+ <description>
+ Configure 'hedged' reads in DFSClient. This is the number of milliseconds
+ to wait before starting up a 'hedged' read.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.use.legacy.blockreader</name>
+ <value>false</value>
+ <description>
+ If true, use the RemoteBlockReader class for local read short circuit. If false, use
+ the newer RemoteBlockReader2 class.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.write.byte-array-manager.count-limit</name>
+ <value>2048</value>
+ <description>
+ The maximum number of arrays allowed for each array length.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.write.byte-array-manager.count-reset-time-period-ms</name>
+ <value>10000</value>
+ <description>
+ The time period in milliseconds that the allocation count for each array length is
+ reset to zero if there is no increment.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.write.byte-array-manager.count-threshold</name>
+ <value>128</value>
+ <description>
+ The count threshold for each array length so that a manager is created only after the
+ allocation count exceeds the threshold. In other words, the particular array length
+ is not managed until the allocation count exceeds the threshold.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.write.byte-array-manager.enabled</name>
+ <value>false</value>
+ <description>
+ If true, enables byte array manager used by DFSOutputStream.
+ </description>
+</property>
+
+<property>
+ <name>dfs.client.write.max-packets-in-flight</name>
+ <value>80</value>
+ <description>
+ The maximum number of DFSPackets allowed in flight.
+ </description>
+</property>
+
+<property>
+ <name>dfs.content-summary.limit</name>
+ <value>5000</value>
+ <description>
+ The maximum content summary counts allowed in one locking period. 0 or a negative number
+ means no limit (i.e. no yielding).
+ </description>
+</property>
+
+<property>
+ <name>dfs.content-summary.sleep-microsec</name>
+ <value>500</value>
+ <description>
+ The length of time in microseconds to put the thread to sleep, between reaquiring the locks
+ in content summary computation.
+ </description>
+</property>
+
+<property>
+ <name>dfs.data.transfer.client.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ If true, set TCP_NODELAY to sockets for transferring data from DFS client.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.balance.max.concurrent.moves</name>
+ <value>5</value>
+ <description>
+ Maximum number of threads for Datanode balancer pending moves. This
+ value is reconfigurable via the "dfsadmin -reconfig" command.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.fsdataset.factory</name>
+ <value></value>
+ <description>
+ The class name for the underlying storage that stores replicas for a
+ Datanode. Defaults to
+ org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.fsdataset.volume.choosing.policy</name>
+ <value></value>
+ <description>
+ The class name of the policy for choosing volumes in the list of
+ directories. Defaults to
+ org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy.
+ If you would like to take into account available disk space, set the
+ value to
+ "org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy".
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.hostname</name>
+ <value></value>
+ <description>
+ Optional. The hostname for the Datanode containing this
+ configuration file. Will be different for each machine.
+ Defaults to current hostname.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.lazywriter.interval.sec</name>
+ <value>60</value>
+ <description>
+ Interval in seconds for Datanodes for lazy persist writes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.network.counts.cache.max.size</name>
+ <value>2147483647</value>
+ <description>
+ The maximum number of entries the datanode per-host network error
+ count cache may contain.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.oob.timeout-ms</name>
+ <value>1500,0,0,0</value>
+ <description>
+ Timeout value when sending OOB response for each OOB type, which are
+ OOB_RESTART, OOB_RESERVED1, OOB_RESERVED2, and OOB_RESERVED3,
+ respectively. Currently, only OOB_RESTART is used.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.parallel.volumes.load.threads.num</name>
+ <value></value>
+ <description>
+ Maximum number of threads to use for upgrading data directories.
+ The default value is the number of storage directories in the
+ DataNode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.ram.disk.replica.tracker</name>
+ <value></value>
+ <description>
+ Name of the class implementing the RamDiskReplicaTracker interface.
+ Defaults to
+ org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.restart.replica.expiration</name>
+ <value>50</value>
+ <description>
+ During shutdown for restart, the amount of time in seconds budgeted for
+ datanode restart.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.socket.reuse.keepalive</name>
+ <value>4000</value>
+ <description>
+ The window of time in ms before the DataXceiver closes a socket for a
+ single request. If a second request occurs within that window, the
+ socket can be reused.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.socket.write.timeout</name>
+ <value>480000</value>
+ <description>
+ Timeout in ms for clients socket writes to DataNodes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.sync.behind.writes.in.background</name>
+ <value>false</value>
+ <description>
+ If set to true, then sync_file_range() system call will occur
+ asynchronously. This property is only valid when the property
+ dfs.datanode.sync.behind.writes is true.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.transferTo.allowed</name>
+ <value>true</value>
+ <description>
+ If false, break block tranfers on 32-bit machines greater than
+ or equal to 2GB into smaller chunks.
+ </description>
+</property>
+
+<property>
+ <name>dfs.ha.fencing.methods</name>
+ <value></value>
+ <description>
+ A list of scripts or Java classes which will be used to fence
+ the Active NameNode during a failover. See the HDFS High
+ Availability documentation for details on automatic HA
+ configuration.
+ </description>
+</property>
+
+<property>
+ <name>dfs.ha.standby.checkpoints</name>
+ <value>true</value>
+ <description>
+ If true, a NameNode in Standby state periodically takes a checkpoint
+ of the namespace, saves it to its local storage and then upload to
+ the remote NameNode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.ha.zkfc.port</name>
+ <value>8019</value>
+ <description>
+ The port number that the zookeeper failover controller RPC
+ server binds to.
+ </description>
+</property>
+
+<property>
+ <name>dfs.journalnode.edits.dir</name>
+ <value>/tmp/hadoop/dfs/journalnode/</value>
+ <description>
+ The directory where the journal edit files are stored.
+ </description>
+</property>
+
+<property>
+ <name>dfs.journalnode.kerberos.internal.spnego.principal</name>
+ <value></value>
+ <description>
+ Kerberos SPNEGO principal name used by the journal node.
+ </description>
+</property>
+
+<property>
+ <name>dfs.journalnode.kerberos.principal</name>
+ <value></value>
+ <description>
+ Kerberos principal name for the journal node.
+ </description>
+</property>
+
+<property>
+ <name>dfs.journalnode.keytab.file</name>
+ <value></value>
+ <description>
+ Kerberos keytab file for the journal node.
+ </description>
+</property>
+
+<property>
+ <name>dfs.ls.limit</name>
+ <value>1000</value>
+ <description>
+ Limit the number of files printed by ls. If less or equal to
+ zero, at most DFS_LIST_LIMIT_DEFAULT (= 1000) will be printed.
+ </description>
+</property>
+
+<property>
+ <name>dfs.mover.movedWinWidth</name>
+ <value>5400000</value>
+ <description>
+ The minimum time interval, in milliseconds, that a block can be
+ moved to another location again.
+ </description>
+</property>
+
+<property>
+ <name>dfs.mover.moverThreads</name>
+ <value>1000</value>
+ <description>
+ Configure the balancer's mover thread pool size.
+ </description>
+</property>
+
+<property>
+ <name>dfs.mover.retry.max.attempts</name>
+ <value>10</value>
+ <description>
+ The maximum number of retries before the mover consider the
+ move failed.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.audit.log.async</name>
+ <value>false</value>
+ <description>
+ If true, enables asynchronous audit log.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.audit.log.token.tracking.id</name>
+ <value>false</value>
+ <description>
+ If true, adds a tracking ID for all audit log events.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction</name>
+ <value>0.6</value>
+ <description>
+ Only used when the dfs.block.replicator.classname is set to
+ org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy.
+ Special value between 0 and 1, noninclusive. Increases chance of
+ placing blocks on Datanodes with less disk space used.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.backup.dnrpc-address</name>
+ <value></value>
+ <description>
+ Service RPC address for the backup Namenode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.delegation.token.always-use</name>
+ <value>false</value>
+ <description>
+ For testing. Setting to true always allows the DT secret manager
+ to be used, even if security is disabled.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.edits.asynclogging</name>
+ <value>false</value>
+ <description>
+ If set to true, enables asynchronous edit logs in the Namenode. If set
+ to false, the Namenode uses the traditional synchronous edit logs.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.edits.dir.minimum</name>
+ <value>1</value>
+ <description>
+ dfs.namenode.edits.dir includes both required directories
+ (specified by dfs.namenode.edits.dir.required) and optional directories.
+
+ The number of usable optional directories must be greater than or equal
+ to this property. If the number of usable optional directories falls
+ below dfs.namenode.edits.dir.minimum, HDFS will issue an error.
+
+ This property defaults to 1.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.edits.journal-plugin</name>
+ <value></value>
+ <description>
+ When FSEditLog is creating JournalManagers from dfs.namenode.edits.dir,
+ and it encounters a URI with a schema different to "file" it loads the
+ name of the implementing class from
+ "dfs.namenode.edits.journal-plugin.[schema]". This class must implement
+ JournalManager and have a constructor which takes (Configuration, URI).
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.file.close.num-committed-allowed</name>
+ <value>0</value>
+ <description>
+ Normally a file can only be closed with all its blocks are committed.
+ When this value is set to a positive integer N, a file can be closed
+ when N blocks are committed and the rest complete.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.inode.attributes.provider.class</name>
+ <value></value>
+ <description>
+ Name of class to use for delegating HDFS authorization.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.max-num-blocks-to-log</name>
+ <value>1000</value>
+ <description>
+ Puts a limit on the number of blocks printed to the log by the Namenode
+ after a block report.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.max.op.size</name>
+ <value>52428800</value>
+ <description>
+ Maximum opcode size in bytes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.missing.checkpoint.periods.before.shutdown</name>
+ <value>3</value>
+ <description>
+ The number of checkpoint period windows (as defined by the property
+ dfs.namenode.checkpoint.period) allowed by the Namenode to perform
+ saving the namespace before shutdown.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.name.cache.threshold</name>
+ <value>10</value>
+ <description>
+ Frequently accessed files that are accessed more times than this
+ threshold are cached in the FSDirectory nameCache.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.replication.max-streams</name>
+ <value>2</value>
+ <description>
+ Hard limit for the number of highest-priority replication streams.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.replication.max-streams-hard-limit</name>
+ <value>4</value>
+ <description>
+ Hard limit for all replication streams.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.replication.pending.timeout-sec</name>
+ <value>-1</value>
+ <description>
+ Timeout in seconds for block replication. If this value is 0 or less,
+ then it will default to 5 minutes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.stale.datanode.minimum.interval</name>
+ <value>3</value>
+ <description>
+ Minimum number of missed heartbeats intervals for a datanode to
+ be marked stale by the Namenode. The actual interval is calculated as
+ (dfs.namenode.stale.datanode.minimum.interval * dfs.heartbeat.interval)
+ in seconds. If this value is greater than the property
+ dfs.namenode.stale.datanode.interval, then the calculated value above
+ is used.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.storageinfo.defragment.timeout.ms</name>
+ <value>4</value>
+ <description>
+ Timeout value in ms for the StorageInfo compaction run.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.storageinfo.defragment.interval.ms</name>
+ <value>600000</value>
+ <description>
+ The thread for checking the StorageInfo for defragmentation will
+ run periodically. The time between runs is determined by this
+ property.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.storageinfo.defragment.ratio</name>
+ <value>0.75</value>
+ <description>
+ The defragmentation threshold for the StorageInfo.
+ </description>
+</property>
+
+<property>
+ <name>dfs.pipeline.ecn</name>
+ <value>false</value>
+ <description>
+ If true, allows ECN (explicit congestion notification) from the
+ Datanode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.accept-recovery.timeout.ms</name>
+ <value>120000</value>
+ <description>
+ Quorum timeout in milliseconds during accept phase of
+ recovery/synchronization for a specific segment.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.finalize-segment.timeout.ms</name>
+ <value>120000</value>
+ <description>
+ Quorum timeout in milliseconds during finalizing for a specific
+ segment.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.get-journal-state.timeout.ms</name>
+ <value>120000</value>
+ <description>
+ Timeout in milliseconds when calling getJournalState().
+ JournalNodes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.new-epoch.timeout.ms</name>
+ <value>120000</value>
+ <description>
+ Timeout in milliseconds when getting an epoch number for write
+ access to JournalNodes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.prepare-recovery.timeout.ms</name>
+ <value>120000</value>
+ <description>
+ Quorum timeout in milliseconds during preparation phase of
+ recovery/synchronization for a specific segment.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.queued-edits.limit.mb</name>
+ <value>10</value>
+ <description>
+ Queue size in MB for quorum journal edits.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.select-input-streams.timeout.ms</name>
+ <value>20000</value>
+ <description>
+ Timeout in milliseconds for accepting streams from JournalManagers.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.start-segment.timeout.ms</name>
+ <value>20000</value>
+ <description>
+ Quorum timeout in milliseconds for starting a log segment.
+ </description>
+</property>
+
+<property>
+ <name>dfs.qjournal.write-txns.timeout.ms</name>
+ <value>20000</value>
+ <description>
+ Write timeout in milliseconds when writing to a quorum of remote
+ journals.
+ </description>
+</property>
+
+<property>
+ <name>dfs.quota.by.storage.type.enabled</name>
+ <value>true</value>
+ <description>
+ If true, enables quotas based on storage type.
+ </description>
+</property>
+
+<property>
+ <name>dfs.secondary.namenode.kerberos.principal</name>
+ <value></value>
+ <description>
+ Kerberos principal name for the Secondary NameNode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.secondary.namenode.keytab.file</name>
+ <value></value>
+ <description>
+ Kerberos keytab file for the Secondary NameNode.
+ </description>
+</property>
+
+<property>
+ <name>dfs.web.authentication.filter</name>
+ <value>org.apache.hadoop.hdfs.web.AuthFilter</value>
+ <description>
+ Authentication filter class used for WebHDFS.
+ </description>
+</property>
+
+<property>
+ <name>dfs.web.authentication.simple.anonymous.allowed</name>
+ <value></value>
+ <description>
+ If true, allow anonymous user to access WebHDFS. Set to
+ false to disable anonymous authentication.
+ </description>
+</property>
+
+<property>
+ <name>dfs.web.ugi</name>
+ <value></value>
+ <description>
+ dfs.web.ugi is deprecated. Use hadoop.http.staticuser.user instead.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.netty.high.watermark</name>
+ <value>65535</value>
+ <description>
+ High watermark configuration to Netty for Datanode WebHdfs.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.netty.low.watermark</name>
+ <value>32768</value>
+ <description>
+ Low watermark configuration to Netty for Datanode WebHdfs.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.oauth2.access.token.provider</name>
+ <value></value>
+ <description>
+ Access token provider class for WebHDFS using OAuth2.
+ Defaults to org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.oauth2.client.id</name>
+ <value></value>
+ <description>
+ Client id used to obtain access token with either credential or
+ refresh token.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.oauth2.enabled</name>
+ <value>false</value>
+ <description>
+ If true, enables OAuth2 in WebHDFS
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.oauth2.refresh.url</name>
+ <value></value>
+ <description>
+ URL against which to post for obtaining bearer token with
+ either credential or refresh token.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value></value>
+ <description>
+ Keystore key password for HTTPS SSL configuration
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value></value>
+ <description>
+ Keystore location for HTTPS SSL configuration
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value></value>
+ <description>
+ Keystore password for HTTPS SSL configuration
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.location</name>
+ <value></value>
+ <description>
+ Truststore location for HTTPS SSL configuration
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.password</name>
+ <value></value>
+ <description>
+ Truststore password for HTTPS SSL configuration
+ </description>
+</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/209303be/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 9637f59..46420f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -45,7 +45,67 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
// Set error modes
errorIfMissingConfigProps = true;
- errorIfMissingXmlProps = false;
+ errorIfMissingXmlProps = true;
+
+ // Initialize used variables
+ configurationPropsToSkipCompare = new HashSet<String>();
+
+ // Ignore testing based parameter
+ configurationPropsToSkipCompare.add("ignore.secure.ports.for.testing");
+
+ // Remove deprecated properties listed in Configuration#DeprecationDelta
+ configurationPropsToSkipCompare.add(DFSConfigKeys.DFS_DF_INTERVAL_KEY);
+
+ // Remove default properties
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT);
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_DEFAULT);
+
+ // Remove support property
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY);
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY);
+
+ // Purposely hidden, based on comments in DFSConfigKeys
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY);
+
+ // Fully deprecated properties?
+ configurationPropsToSkipCompare
+ .add("dfs.corruptfilesreturned.max");
+ configurationPropsToSkipCompare
+ .add("dfs.datanode.hdfs-blocks-metadata.enabled");
+ configurationPropsToSkipCompare
+ .add("dfs.metrics.session-id");
+ configurationPropsToSkipCompare
+ .add("dfs.datanode.synconclose");
+ configurationPropsToSkipCompare
+ .add("dfs.datanode.non.local.lazy.persist");
+ configurationPropsToSkipCompare
+ .add("dfs.namenode.tolerate.heartbeat.multiplier");
+ configurationPropsToSkipCompare
+ .add("dfs.namenode.stripe.min");
+ configurationPropsToSkipCompare
+ .add("dfs.namenode.replqueue.threshold-pct");
+
+ // Removed by HDFS-6440
+ configurationPropsToSkipCompare
+ .add("dfs.ha.log-roll.rpc.timeout");
+
+ // Example (not real) property in hdfs-default.xml
+ configurationPropsToSkipCompare.add("dfs.ha.namenodes");
+
+ // Property used for internal testing only
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION);
+
+ // Property not intended for users
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
+ configurationPropsToSkipCompare
+ .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
// Allocate
xmlPropsToSkipCompare = new HashSet<String>();
@@ -58,21 +118,12 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
// Used dynamically as part of DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX
xmlPropsToSkipCompare.add("dfs.namenode.edits.journal-plugin.qjournal");
- // Example (not real) property in hdfs-default.xml
- xmlPropsToSkipCompare.add("dfs.ha.namenodes.EXAMPLENAMESERVICE");
-
// Defined in org.apache.hadoop.fs.CommonConfigurationKeys
xmlPropsToSkipCompare.add("hadoop.user.group.metrics.percentiles.intervals");
// Used oddly by DataNode to create new config String
xmlPropsToSkipCompare.add("hadoop.hdfs.configuration.version");
- // Kept in the NfsConfiguration class in the hadoop-hdfs-nfs module
- xmlPrefixToSkipCompare.add("nfs");
-
- // Not a hardcoded property. Used by SaslRpcClient
- xmlPrefixToSkipCompare.add("dfs.namenode.kerberos.principal.pattern");
-
// Skip comparing in branch-2. Removed in trunk with HDFS-7985.
xmlPropsToSkipCompare.add("dfs.webhdfs.enabled");
@@ -82,5 +133,21 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
// Ignore HTrace properties
xmlPropsToSkipCompare.add("fs.client.htrace");
xmlPropsToSkipCompare.add("hadoop.htrace");
+
+ // Ignore SpanReceiveHost properties
+ xmlPropsToSkipCompare.add("dfs.htrace.spanreceiver.classes");
+ xmlPropsToSkipCompare.add("dfs.client.htrace.spanreceiver.classes");
+
+ // Remove deprecated properties listed in Configuration#DeprecationDelta
+ xmlPropsToSkipCompare.add(DFSConfigKeys.DFS_DF_INTERVAL_KEY);
+
+ // Kept in the NfsConfiguration class in the hadoop-hdfs-nfs module
+ xmlPrefixToSkipCompare.add("nfs");
+
+ // Not a hardcoded property. Used by SaslRpcClient
+ xmlPrefixToSkipCompare.add("dfs.namenode.kerberos.principal.pattern");
+
+ // Skip over example property
+ xmlPrefixToSkipCompare.add("dfs.ha.namenodes");
}
}
[12/21] hadoop git commit: HDFS-10270. TestJMXGet:testNameNode() fails. Contributed by Gergely Novák
Posted by ae...@apache.org.
HDFS-10270. TestJMXGet:testNameNode() fails. Contributed by Gergely Novák
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2f3bbc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2f3bbc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2f3bbc2
Branch: refs/heads/HDFS-1312
Commit: d2f3bbc29046435904ad9418073795439c71b441
Parents: 903428b
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Apr 13 11:22:36 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed Apr 13 11:22:36 2016 -0500
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/tools/TestJMXGet.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2f3bbc2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
index a9e41ec..f83e7d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
@@ -111,9 +111,6 @@ public class TestJMXGet {
jmx.getValue("NumLiveDataNodes")));
assertGauge("CorruptBlocks", Long.parseLong(jmx.getValue("CorruptBlocks")),
getMetrics("FSNamesystem"));
- DFSTestUtil.waitForMetric(jmx, "NumOpenConnections", numDatanodes);
- assertEquals(numDatanodes, Integer.parseInt(
- jmx.getValue("NumOpenConnections")));
cluster.shutdown();
MBeanServerConnection mbsc = ManagementFactory.getPlatformMBeanServer();
[21/21] hadoop git commit: Merge branch 'trunk' into HDFS-1312
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/648cc93f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/648cc93f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/648cc93f
Branch: refs/heads/HDFS-1312
Commit: 648cc93f6e70f844f1324a7ede83c127158dcd38
Parents: 3a490d6 27b131e
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Apr 13 23:16:15 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Apr 13 23:16:15 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/CachingGetSpaceUsed.java | 168 +++
.../src/main/java/org/apache/hadoop/fs/DU.java | 240 +---
.../java/org/apache/hadoop/fs/GetSpaceUsed.java | 147 +++
.../apache/hadoop/fs/WindowsGetSpaceUsed.java | 46 +
.../org/apache/hadoop/http/HttpServer2.java | 29 +-
.../org/apache/hadoop/metrics2/lib/Interns.java | 16 +-
.../apache/hadoop/util/ShutdownHookManager.java | 2 +-
.../src/site/markdown/DeprecatedProperties.md | 1 +
.../markdown/filesystem/fsdatainputstream.md | 35 +-
.../test/java/org/apache/hadoop/fs/TestDU.java | 46 +-
.../org/apache/hadoop/fs/TestGetSpaceUsed.java | 133 +++
.../org/apache/hadoop/http/TestHttpServer.java | 10 +
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +
.../server/blockmanagement/DatanodeManager.java | 115 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 18 +
.../hadoop/hdfs/server/datanode/DataNode.java | 9 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 93 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 20 +-
.../hdfs/server/namenode/FSNamesystem.java | 78 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 24 +-
.../src/main/resources/hdfs-default.xml | 1033 +++++++++++++++++-
.../hdfs/TestDecommissionWithStriped.java | 76 +-
.../TestSortLocatedStripedBlock.java | 557 ++++++++++
.../server/datanode/TestBlockReplacement.java | 3 +-
.../TestDataNodeVolumeFailureToleration.java | 19 +-
.../fsdataset/impl/TestSpaceReservation.java | 60 +
.../namenode/TestNameNodeReconfigure.java | 51 +-
.../hadoop/tools/TestHdfsConfigFields.java | 87 +-
.../org/apache/hadoop/tools/TestJMXGet.java | 3 -
.../yarn/client/api/impl/NMClientImpl.java | 36 +-
.../webapps/static/dt-1.9.4/css/jui-dt.css | 4 +-
.../localizer/LocalizedResource.java | 8 +-
.../nodemanager/webapp/ApplicationPage.java | 18 +-
.../nodemanager/webapp/TestNMAppsPage.java | 86 ++
.../timeline/EntityGroupFSTimelineStore.java | 6 +
35 files changed, 2885 insertions(+), 396 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/648cc93f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/648cc93f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
[14/21] hadoop git commit: HADOOP-12993. Change ShutdownHookManger
complete shutdown log from INFO to DEBUG. Contributed by Xiaoyu Yao.
Posted by ae...@apache.org.
HADOOP-12993. Change ShutdownHookManger complete shutdown log from INFO to DEBUG. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ced42da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ced42da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ced42da
Branch: refs/heads/HDFS-1312
Commit: 8ced42daff5cd0cb11d26042ae8c8255ef629a40
Parents: e0cb426
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Mon Apr 11 16:36:18 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Apr 13 15:34:17 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/util/ShutdownHookManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ced42da/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
index 33f942f..81983f0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
@@ -81,7 +81,7 @@ public class ShutdownHookManager {
LOG.error("ShutdownHookManger shutdown forcefully.");
EXECUTOR.shutdownNow();
}
- LOG.info("ShutdownHookManger complete shutdown.");
+ LOG.debug("ShutdownHookManger complete shutdown.");
} catch (InterruptedException ex) {
LOG.error("ShutdownHookManger interrupted while waiting for " +
"termination.", ex);
[16/21] hadoop git commit: HDFS-10279. Improve validation of the
configured number of tolerated failed volumes. Contributed by Lin Yiqun.
Posted by ae...@apache.org.
HDFS-10279. Improve validation of the configured number of tolerated failed volumes. Contributed by Lin Yiqun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/314aa21a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/314aa21a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/314aa21a
Branch: refs/heads/HDFS-1312
Commit: 314aa21a89134fac68ac3cb95efdeb56bd3d7b05
Parents: 192112d
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Apr 13 16:39:50 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Apr 13 16:39:50 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/DNConf.java | 18 ++++++++++++++++++
.../hadoop/hdfs/server/datanode/DataNode.java | 9 +++++++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 13 ++-----------
.../TestDataNodeVolumeFailureToleration.java | 19 +++++++++++++++++--
4 files changed, 46 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/314aa21a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 5cff2d3..b616414 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -114,6 +114,9 @@ public class DNConf {
// Allow LAZY_PERSIST writes from non-local clients?
private final boolean allowNonLocalLazyPersist;
+ private final int volFailuresTolerated;
+ private final int volsConfigured;
+
public DNConf(Configuration conf) {
this.conf = conf;
socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
@@ -238,6 +241,13 @@ public class DNConf {
this.bpReadyTimeout = conf.getLong(
DFS_DATANODE_BP_READY_TIMEOUT_KEY,
DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT);
+
+ this.volFailuresTolerated =
+ conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
+ DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT);
+ String[] dataDirs =
+ conf.getTrimmedStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
+ this.volsConfigured = (dataDirs == null) ? 0 : dataDirs.length;
}
// We get minimumNameNodeVersion via a method so it can be mocked out in tests.
@@ -363,4 +373,12 @@ public class DNConf {
public long getLifelineIntervalMs() {
return lifelineIntervalMs;
}
+
+ public int getVolFailuresTolerated() {
+ return volFailuresTolerated;
+ }
+
+ public int getVolsConfigured() {
+ return volsConfigured;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/314aa21a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 625eb3f..288fc3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1268,6 +1268,15 @@ public class DataNode extends ReconfigurableBase
LOG.info("Starting DataNode with maxLockedMemory = " +
dnConf.maxLockedMemory);
+ int volFailuresTolerated = dnConf.getVolFailuresTolerated();
+ int volsConfigured = dnConf.getVolsConfigured();
+ if (volFailuresTolerated < 0 || volFailuresTolerated >= volsConfigured) {
+ throw new DiskErrorException("Invalid value configured for "
+ + "dfs.datanode.failed.volumes.tolerated - " + volFailuresTolerated
+ + ". Value configured is either less than 0 or >= "
+ + "to the number of configured volumes (" + volsConfigured + ").");
+ }
+
storage = new DataStorage();
// global DN settings
http://git-wip-us.apache.org/repos/asf/hadoop/blob/314aa21a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 381c799..f7e0aae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -268,24 +268,15 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf);
// The number of volumes required for operation is the total number
// of volumes minus the number of failed volumes we can tolerate.
- volFailuresTolerated =
- conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
- DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT);
+ volFailuresTolerated = datanode.getDnConf().getVolFailuresTolerated();
- String[] dataDirs = conf.getTrimmedStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
Collection<StorageLocation> dataLocations = DataNode.getStorageLocations(conf);
List<VolumeFailureInfo> volumeFailureInfos = getInitialVolumeFailureInfos(
dataLocations, storage);
- int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length;
+ int volsConfigured = datanode.getDnConf().getVolsConfigured();
int volsFailed = volumeFailureInfos.size();
- if (volFailuresTolerated < 0 || volFailuresTolerated >= volsConfigured) {
- throw new DiskErrorException("Invalid value configured for "
- + "dfs.datanode.failed.volumes.tolerated - " + volFailuresTolerated
- + ". Value configured is either less than 0 or >= "
- + "to the number of configured volumes (" + volsConfigured + ").");
- }
if (volsFailed > volFailuresTolerated) {
throw new DiskErrorException("Too many failed volumes - "
+ "current valid volumes: " + storage.getNumStorageDirs()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/314aa21a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
index 1eb8bca..2f8239e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureToleration.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -229,9 +231,22 @@ public class TestDataNodeVolumeFailureToleration {
prepareDirToFail(dirs[i]);
}
restartDatanodes(volumesTolerated, manageDfsDirs);
- assertEquals(expectedBPServiceState, cluster.getDataNodes().get(0)
- .isBPServiceAlive(cluster.getNamesystem().getBlockPoolId()));
+ } catch (DiskErrorException e) {
+ GenericTestUtils.assertExceptionContains("Invalid value configured for "
+ + "dfs.datanode.failed.volumes.tolerated", e);
} finally {
+ boolean bpServiceState;
+ // If the datanode not registered successfully,
+ // because the invalid value configured for tolerated volumes
+ if (cluster.getDataNodes().size() == 0) {
+ bpServiceState = false;
+ } else {
+ bpServiceState =
+ cluster.getDataNodes().get(0)
+ .isBPServiceAlive(cluster.getNamesystem().getBlockPoolId());
+ }
+ assertEquals(expectedBPServiceState, bpServiceState);
+
for (File dir : dirs) {
FileUtil.chmod(dir.toString(), "755");
}
[19/21] hadoop git commit: YARN-4541. Change log message in
LocalizedResource#handle() to DEBUG (rchiang via rkanter)
Posted by ae...@apache.org.
YARN-4541. Change log message in LocalizedResource#handle() to DEBUG (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d9194df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d9194df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d9194df
Branch: refs/heads/HDFS-1312
Commit: 0d9194df00fd68bfb7a8ba504b0cddd7d7c69b8a
Parents: 5566177
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Apr 13 17:45:18 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Apr 13 17:45:18 2016 -0700
----------------------------------------------------------------------
.../containermanager/localizer/LocalizedResource.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d9194df/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
index 04d95f1..464293a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
@@ -200,9 +200,11 @@ public class LocalizedResource implements EventHandler<ResourceEvent> {
LOG.warn("Can't handle this event at current state", e);
}
if (oldState != newState) {
- LOG.info("Resource " + resourcePath + (localPath != null ?
- "(->" + localPath + ")": "") + " transitioned from " + oldState
- + " to " + newState);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resource " + resourcePath + (localPath != null ?
+ "(->" + localPath + ")": "") + " transitioned from " + oldState
+ + " to " + newState);
+ }
}
} finally {
this.writeLock.unlock();
[15/21] hadoop git commit: HDFS-10209. Support enable caller context
in HDFS namenode audit log without restart namenode
Posted by ae...@apache.org.
HDFS-10209. Support enable caller context in HDFS namenode audit log without restart namenode
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/192112d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/192112d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/192112d5
Branch: refs/heads/HDFS-1312
Commit: 192112d5a2e7ce4ec8eb47e21ab744b34c848893
Parents: 8ced42d
Author: Xiaobing Zhou <xz...@hortonworks.com>
Authored: Tue Apr 5 11:54:42 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Apr 13 15:34:41 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystem.java | 52 +++++++++++++++++++-
.../hadoop/hdfs/server/namenode/NameNode.java | 24 ++++++++-
.../namenode/TestNameNodeReconfigure.java | 51 ++++++++++++++++++-
3 files changed, 123 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/192112d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 39b3598..471e6b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -902,6 +902,36 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return null;
}
+ /**
+ * Locate DefaultAuditLogger, if any, to enable/disable CallerContext.
+ *
+ * @param value
+ * true, enable CallerContext, otherwise false to disable it.
+ */
+ void setCallerContextEnabled(final boolean value) {
+ for (AuditLogger logger : auditLoggers) {
+ if (logger instanceof DefaultAuditLogger) {
+ ((DefaultAuditLogger) logger).setCallerContextEnabled(value);
+ break;
+ }
+ }
+ }
+
+ /**
+ * Get the value indicating if CallerContext is enabled.
+ *
+ * @return true, if CallerContext is enabled, otherwise false, if it's
+ * disabled.
+ */
+ boolean getCallerContextEnabled() {
+ for (AuditLogger logger : auditLoggers) {
+ if (logger instanceof DefaultAuditLogger) {
+ return ((DefaultAuditLogger) logger).getCallerContextEnabled();
+ }
+ }
+ return false;
+ }
+
private List<AuditLogger> initAuditLoggers(Configuration conf) {
// Initialize the custom access loggers if configured.
Collection<String> alClasses =
@@ -6848,13 +6878,33 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
};
- private boolean isCallerContextEnabled;
+ private volatile boolean isCallerContextEnabled;
private int callerContextMaxLen;
private int callerSignatureMaxLen;
private boolean logTokenTrackingId;
private Set<String> debugCmdSet = new HashSet<String>();
+ /**
+ * Enable or disable CallerContext.
+ *
+ * @param value
+ * true, enable CallerContext, otherwise false to disable it.
+ */
+ void setCallerContextEnabled(final boolean value) {
+ isCallerContextEnabled = value;
+ }
+
+ /**
+ * Get the value indicating if CallerContext is enabled.
+ *
+ * @return true, if CallerContext is enabled, otherwise false, if it's
+ * disabled.
+ */
+ boolean getCallerContextEnabled() {
+ return isCallerContextEnabled;
+ }
+
@Override
public void initialize(Configuration conf) {
isCallerContextEnabled = conf.getBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/192112d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 1301940..b39c842 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -107,6 +107,8 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY;
@@ -277,7 +279,8 @@ public class NameNode extends ReconfigurableBase implements
.unmodifiableList(Arrays
.asList(DFS_HEARTBEAT_INTERVAL_KEY,
DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
- FS_PROTECTED_DIRECTORIES));
+ FS_PROTECTED_DIRECTORIES,
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY));
private static final String USAGE = "Usage: hdfs namenode ["
+ StartupOption.BACKUP.getName() + "] | \n\t["
@@ -2008,7 +2011,9 @@ public class NameNode extends ReconfigurableBase implements
+ datanodeManager.getHeartbeatRecheckInterval());
}
case FS_PROTECTED_DIRECTORIES:
- return getNamesystem().getFSDirectory().setProtectedDirectories(newVal);
+ return reconfProtectedDirectories(newVal);
+ case HADOOP_CALLER_CONTEXT_ENABLED_KEY:
+ return reconfCallerContextEnabled(newVal);
default:
break;
}
@@ -2016,6 +2021,21 @@ public class NameNode extends ReconfigurableBase implements
.get(property));
}
+ private String reconfProtectedDirectories(String newVal) {
+ return getNamesystem().getFSDirectory().setProtectedDirectories(newVal);
+ }
+
+ private String reconfCallerContextEnabled(String newVal) {
+ Boolean callerContextEnabled;
+ if (newVal == null) {
+ callerContextEnabled = HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
+ } else {
+ callerContextEnabled = Boolean.parseBoolean(newVal);
+ }
+ namesystem.setCallerContextEnabled(callerContextEnabled);
+ return Boolean.toString(callerContextEnabled);
+ }
+
@Override // ReconfigurableBase
protected Configuration getNewConf() {
return new HdfsConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/192112d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
index abdb1ea..4fd7af6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
@@ -50,13 +52,60 @@ public class TestNameNodeReconfigure {
public void setUp() throws IOException {
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ }
+
+ @Test
+ public void testReconfigureCallerContextEnabled()
+ throws ReconfigurationException {
+ final NameNode nameNode = cluster.getNameNode();
+ final FSNamesystem nameSystem = nameNode.getNamesystem();
+
+ // try invalid values
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "text");
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
+ false,
+ nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
+ HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
+
+ // enable CallerContext
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "true");
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", true,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
+ true,
+ nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
+ HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
+
+ // disable CallerContext
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "false");
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
+ false,
+ nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
+ HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
+
+ // revert to default
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, null);
+
+ // verify default
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", null,
+ nameNode.getConf().get(HADOOP_CALLER_CONTEXT_ENABLED_KEY));
}
/**
* Test that we can modify configuration properties.
*/
@Test
- public void testReconfigure() throws ReconfigurationException {
+ public void testReconfigureHearbeatCheck1() throws ReconfigurationException {
final NameNode nameNode = cluster.getNameNode();
final DatanodeManager datanodeManager = nameNode.namesystem
.getBlockManager().getDatanodeManager();
[20/21] hadoop git commit: HADOOP-12822. Change "Metrics intern cache
overflow" log level from WARN to INFO. Contributed by Andras Bokor.
Posted by ae...@apache.org.
HADOOP-12822. Change "Metrics intern cache overflow" log level from WARN to INFO. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27b131e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27b131e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27b131e7
Branch: refs/heads/HDFS-1312
Commit: 27b131e79c5fa99de3ed4fb529d854dd5da55bde
Parents: 0d9194d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Apr 14 10:55:51 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Apr 14 10:55:51 2016 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/metrics2/lib/Interns.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27b131e7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/Interns.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/Interns.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/Interns.java
index 7ad6660..8e28367 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/Interns.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/Interns.java
@@ -18,15 +18,15 @@
package org.apache.hadoop.metrics2.lib;
-import java.util.Map;
-import java.util.LinkedHashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
/**
* Helpers to create interned metrics info
@@ -34,7 +34,7 @@ import org.apache.hadoop.metrics2.MetricsTag;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class Interns {
- private static final Log LOG = LogFactory.getLog(Interns.class);
+ private static final Logger LOG = LoggerFactory.getLogger(Interns.class);
// A simple intern cache with two keys
// (to avoid creating new (combined) key objects for lookup)
@@ -47,7 +47,7 @@ public class Interns {
protected boolean removeEldestEntry(Map.Entry<K1, Map<K2, V>> e) {
boolean overflow = expireKey1At(size());
if (overflow && !gotOverflow) {
- LOG.warn("Metrics intern cache overflow at "+ size() +" for "+ e);
+ LOG.info("Metrics intern cache overflow at {} for {}", size(), e);
gotOverflow = true;
}
return overflow;
@@ -67,7 +67,7 @@ public class Interns {
@Override protected boolean removeEldestEntry(Map.Entry<K2, V> e) {
boolean overflow = expireKey2At(size());
if (overflow && !gotOverflow) {
- LOG.warn("Metrics intern cache overflow at "+ size() +" for "+ e);
+ LOG.info("Metrics intern cache overflow at {} for {}", size(), e);
gotOverflow = true;
}
return overflow;
[17/21] hadoop git commit: Revert "HDFS-10209. Support enable caller
context in HDFS namenode audit log without restart namenode"
Posted by ae...@apache.org.
Revert "HDFS-10209. Support enable caller context in HDFS namenode audit log without restart namenode"
This reverts commit 192112d5a2e7ce4ec8eb47e21ab744b34c848893.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4895c73d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4895c73d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4895c73d
Branch: refs/heads/HDFS-1312
Commit: 4895c73dd493a53eab43f0d16e92c19af15c460b
Parents: 314aa21
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Apr 13 16:45:18 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Apr 13 16:45:18 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystem.java | 52 +-------------------
.../hadoop/hdfs/server/namenode/NameNode.java | 24 +--------
.../namenode/TestNameNodeReconfigure.java | 51 +------------------
3 files changed, 4 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4895c73d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 471e6b9..39b3598 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -902,36 +902,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return null;
}
- /**
- * Locate DefaultAuditLogger, if any, to enable/disable CallerContext.
- *
- * @param value
- * true, enable CallerContext, otherwise false to disable it.
- */
- void setCallerContextEnabled(final boolean value) {
- for (AuditLogger logger : auditLoggers) {
- if (logger instanceof DefaultAuditLogger) {
- ((DefaultAuditLogger) logger).setCallerContextEnabled(value);
- break;
- }
- }
- }
-
- /**
- * Get the value indicating if CallerContext is enabled.
- *
- * @return true, if CallerContext is enabled, otherwise false, if it's
- * disabled.
- */
- boolean getCallerContextEnabled() {
- for (AuditLogger logger : auditLoggers) {
- if (logger instanceof DefaultAuditLogger) {
- return ((DefaultAuditLogger) logger).getCallerContextEnabled();
- }
- }
- return false;
- }
-
private List<AuditLogger> initAuditLoggers(Configuration conf) {
// Initialize the custom access loggers if configured.
Collection<String> alClasses =
@@ -6878,33 +6848,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
};
- private volatile boolean isCallerContextEnabled;
+ private boolean isCallerContextEnabled;
private int callerContextMaxLen;
private int callerSignatureMaxLen;
private boolean logTokenTrackingId;
private Set<String> debugCmdSet = new HashSet<String>();
- /**
- * Enable or disable CallerContext.
- *
- * @param value
- * true, enable CallerContext, otherwise false to disable it.
- */
- void setCallerContextEnabled(final boolean value) {
- isCallerContextEnabled = value;
- }
-
- /**
- * Get the value indicating if CallerContext is enabled.
- *
- * @return true, if CallerContext is enabled, otherwise false, if it's
- * disabled.
- */
- boolean getCallerContextEnabled() {
- return isCallerContextEnabled;
- }
-
@Override
public void initialize(Configuration conf) {
isCallerContextEnabled = conf.getBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4895c73d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index b39c842..1301940 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -107,8 +107,6 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY;
@@ -279,8 +277,7 @@ public class NameNode extends ReconfigurableBase implements
.unmodifiableList(Arrays
.asList(DFS_HEARTBEAT_INTERVAL_KEY,
DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
- FS_PROTECTED_DIRECTORIES,
- HADOOP_CALLER_CONTEXT_ENABLED_KEY));
+ FS_PROTECTED_DIRECTORIES));
private static final String USAGE = "Usage: hdfs namenode ["
+ StartupOption.BACKUP.getName() + "] | \n\t["
@@ -2011,9 +2008,7 @@ public class NameNode extends ReconfigurableBase implements
+ datanodeManager.getHeartbeatRecheckInterval());
}
case FS_PROTECTED_DIRECTORIES:
- return reconfProtectedDirectories(newVal);
- case HADOOP_CALLER_CONTEXT_ENABLED_KEY:
- return reconfCallerContextEnabled(newVal);
+ return getNamesystem().getFSDirectory().setProtectedDirectories(newVal);
default:
break;
}
@@ -2021,21 +2016,6 @@ public class NameNode extends ReconfigurableBase implements
.get(property));
}
- private String reconfProtectedDirectories(String newVal) {
- return getNamesystem().getFSDirectory().setProtectedDirectories(newVal);
- }
-
- private String reconfCallerContextEnabled(String newVal) {
- Boolean callerContextEnabled;
- if (newVal == null) {
- callerContextEnabled = HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
- } else {
- callerContextEnabled = Boolean.parseBoolean(newVal);
- }
- namesystem.setCallerContextEnabled(callerContextEnabled);
- return Boolean.toString(callerContextEnabled);
- }
-
@Override // ReconfigurableBase
protected Configuration getNewConf() {
return new HdfsConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4895c73d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
index 4fd7af6..abdb1ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
@@ -34,8 +34,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
@@ -52,60 +50,13 @@ public class TestNameNodeReconfigure {
public void setUp() throws IOException {
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).build();
- cluster.waitActive();
- }
-
- @Test
- public void testReconfigureCallerContextEnabled()
- throws ReconfigurationException {
- final NameNode nameNode = cluster.getNameNode();
- final FSNamesystem nameSystem = nameNode.getNamesystem();
-
- // try invalid values
- nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "text");
- assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
- nameSystem.getCallerContextEnabled());
- assertEquals(
- HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
- false,
- nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
- HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
-
- // enable CallerContext
- nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "true");
- assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", true,
- nameSystem.getCallerContextEnabled());
- assertEquals(
- HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
- true,
- nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
- HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
-
- // disable CallerContext
- nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "false");
- assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
- nameSystem.getCallerContextEnabled());
- assertEquals(
- HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
- false,
- nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
- HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
-
- // revert to default
- nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, null);
-
- // verify default
- assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
- nameSystem.getCallerContextEnabled());
- assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", null,
- nameNode.getConf().get(HADOOP_CALLER_CONTEXT_ENABLED_KEY));
}
/**
* Test that we can modify configuration properties.
*/
@Test
- public void testReconfigureHearbeatCheck1() throws ReconfigurationException {
+ public void testReconfigure() throws ReconfigurationException {
final NameNode nameNode = cluster.getNameNode();
final DatanodeManager datanodeManager = nameNode.namesystem
.getBlockManager().getDatanodeManager();
[10/21] hadoop git commit: HADOOP-12973. Make DU pluggable. (Elliott
Clark via cmccabe)
Posted by ae...@apache.org.
HADOOP-12973. Make DU pluggable. (Elliott Clark via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35f07705
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35f07705
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35f07705
Branch: refs/heads/HDFS-1312
Commit: 35f07705552ef5636f8b8d2599a6af5ec1426203
Parents: 042a3ae
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Apr 12 16:20:30 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Apr 12 16:28:42 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/CachingGetSpaceUsed.java | 168 +++++++++++++
.../src/main/java/org/apache/hadoop/fs/DU.java | 240 ++++---------------
.../java/org/apache/hadoop/fs/GetSpaceUsed.java | 147 ++++++++++++
.../apache/hadoop/fs/WindowsGetSpaceUsed.java | 46 ++++
.../test/java/org/apache/hadoop/fs/TestDU.java | 46 ++--
.../org/apache/hadoop/fs/TestGetSpaceUsed.java | 133 ++++++++++
.../datanode/fsdataset/impl/BlockPoolSlice.java | 93 +++----
7 files changed, 617 insertions(+), 256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f07705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
new file mode 100644
index 0000000..6ef75d2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Interface for class that can tell estimate much space
+ * is used in a directory.
+ * <p>
+ * The implementor is fee to cache space used. As such there
+ * are methods to update the cached value with any known changes.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
+ static final Logger LOG = LoggerFactory.getLogger(CachingGetSpaceUsed.class);
+
+ protected final AtomicLong used = new AtomicLong();
+ private final AtomicBoolean running = new AtomicBoolean(true);
+ private final long refreshInterval;
+ private final String dirPath;
+ private Thread refreshUsed;
+
+ /**
+ * This is the constructor used by the builder.
+ * All overriding classes should implement this.
+ */
+ public CachingGetSpaceUsed(CachingGetSpaceUsed.Builder builder)
+ throws IOException {
+ this(builder.getPath(), builder.getInterval(), builder.getInitialUsed());
+ }
+
+ /**
+ * Keeps track of disk usage.
+ *
+ * @param path the path to check disk usage in
+ * @param interval refresh the disk usage at this interval
+ * @param initialUsed use this value until next refresh
+ * @throws IOException if we fail to refresh the disk usage
+ */
+ CachingGetSpaceUsed(File path,
+ long interval,
+ long initialUsed) throws IOException {
+ dirPath = path.getCanonicalPath();
+ refreshInterval = interval;
+ used.set(initialUsed);
+ }
+
+ void init() {
+ if (used.get() < 0) {
+ used.set(0);
+ refresh();
+ }
+
+ if (refreshInterval > 0) {
+ refreshUsed = new Thread(new RefreshThread(this),
+ "refreshUsed-" + dirPath);
+ refreshUsed.setDaemon(true);
+ refreshUsed.start();
+ } else {
+ running.set(false);
+ refreshUsed = null;
+ }
+ }
+
+ protected abstract void refresh();
+
+ /**
+ * @return an estimate of space used in the directory path.
+ */
+ @Override public long getUsed() throws IOException {
+ return Math.max(used.get(), 0);
+ }
+
+ /**
+ * @return The directory path being monitored.
+ */
+ public String getDirPath() {
+ return dirPath;
+ }
+
+ /**
+ * Increment the cached value of used space.
+ */
+ public void incDfsUsed(long value) {
+ used.addAndGet(value);
+ }
+
+ /**
+ * Is the background thread running.
+ */
+ boolean running() {
+ return running.get();
+ }
+
+ /**
+ * How long in between runs of the background refresh.
+ */
+ long getRefreshInterval() {
+ return refreshInterval;
+ }
+
+ /**
+ * Reset the current used data amount. This should be called
+ * when the cached value is re-computed.
+ *
+ * @param usedValue new value that should be the disk usage.
+ */
+ protected void setUsed(long usedValue) {
+ this.used.set(usedValue);
+ }
+
+ @Override
+ public void close() throws IOException {
+ running.set(false);
+ if (refreshUsed != null) {
+ refreshUsed.interrupt();
+ }
+ }
+
+ private static final class RefreshThread implements Runnable {
+
+ final CachingGetSpaceUsed spaceUsed;
+
+ RefreshThread(CachingGetSpaceUsed spaceUsed) {
+ this.spaceUsed = spaceUsed;
+ }
+
+ @Override
+ public void run() {
+ while (spaceUsed.running()) {
+ try {
+ Thread.sleep(spaceUsed.getRefreshInterval());
+ // update the used variable
+ spaceUsed.refresh();
+ } catch (InterruptedException e) {
+ LOG.warn("Thread Interrupted waiting to refresh disk information", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f07705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
index 5a4f526..f700e4f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
@@ -17,227 +17,73 @@
*/
package org.apache.hadoop.fs;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.Shell;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
-/** Filesystem disk space usage statistics. Uses the unix 'du' program*/
+/** Filesystem disk space usage statistics. Uses the unix 'du' program */
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
-public class DU extends Shell {
- private String dirPath;
+public class DU extends CachingGetSpaceUsed {
+ private DUShell duShell;
- private AtomicLong used = new AtomicLong();
- private volatile boolean shouldRun = true;
- private Thread refreshUsed;
- private IOException duException = null;
- private long refreshInterval;
-
- /**
- * Keeps track of disk usage.
- * @param path the path to check disk usage in
- * @param interval refresh the disk usage at this interval
- * @throws IOException if we fail to refresh the disk usage
- */
- public DU(File path, long interval) throws IOException {
- this(path, interval, -1L);
+ @VisibleForTesting
+ public DU(File path, long interval, long initialUsed) throws IOException {
+ super(path, interval, initialUsed);
}
-
- /**
- * Keeps track of disk usage.
- * @param path the path to check disk usage in
- * @param interval refresh the disk usage at this interval
- * @param initialUsed use this value until next refresh
- * @throws IOException if we fail to refresh the disk usage
- */
- public DU(File path, long interval, long initialUsed) throws IOException {
- super(0);
- //we set the Shell interval to 0 so it will always run our command
- //and use this one to set the thread sleep interval
- this.refreshInterval = interval;
- this.dirPath = path.getCanonicalPath();
-
- //populate the used variable if the initial value is not specified.
- if (initialUsed < 0) {
- run();
- } else {
- this.used.set(initialUsed);
- }
- }
-
- /**
- * Keeps track of disk usage.
- * @param path the path to check disk usage in
- * @param conf configuration object
- * @throws IOException if we fail to refresh the disk usage
- */
- public DU(File path, Configuration conf) throws IOException {
- this(path, conf, -1L);
+ public DU(CachingGetSpaceUsed.Builder builder) throws IOException {
+ this(builder.getPath(), builder.getInterval(), builder.getInitialUsed());
}
- /**
- * Keeps track of disk usage.
- * @param path the path to check disk usage in
- * @param conf configuration object
- * @param initialUsed use it until the next refresh.
- * @throws IOException if we fail to refresh the disk usage
- */
- public DU(File path, Configuration conf, long initialUsed)
- throws IOException {
- this(path, conf.getLong(CommonConfigurationKeys.FS_DU_INTERVAL_KEY,
- CommonConfigurationKeys.FS_DU_INTERVAL_DEFAULT), initialUsed);
+ @Override
+ protected synchronized void refresh() {
+ if (duShell == null) {
+ duShell = new DUShell();
+ }
+ try {
+ duShell.startRefresh();
+ } catch (IOException ioe) {
+ LOG.warn("Could not get disk usage information", ioe);
+ }
}
-
-
- /**
- * This thread refreshes the "used" variable.
- *
- * Future improvements could be to not permanently
- * run this thread, instead run when getUsed is called.
- **/
- class DURefreshThread implements Runnable {
-
+ private final class DUShell extends Shell {
+ void startRefresh() throws IOException {
+ run();
+ }
@Override
- public void run() {
-
- while(shouldRun) {
+ public String toString() {
+ return
+ "du -sk " + getDirPath() + "\n" + used.get() + "\t" + getDirPath();
+ }
- try {
- Thread.sleep(refreshInterval);
-
- try {
- //update the used variable
- DU.this.run();
- } catch (IOException e) {
- synchronized (DU.this) {
- //save the latest exception so we can return it in getUsed()
- duException = e;
- }
-
- LOG.warn("Could not get disk usage information", e);
- }
- } catch (InterruptedException e) {
- }
- }
+ @Override
+ protected String[] getExecString() {
+ return new String[]{"du", "-sk", getDirPath()};
}
- }
-
- /**
- * Decrease how much disk space we use.
- * @param value decrease by this value
- */
- public void decDfsUsed(long value) {
- used.addAndGet(-value);
- }
- /**
- * Increase how much disk space we use.
- * @param value increase by this value
- */
- public void incDfsUsed(long value) {
- used.addAndGet(value);
- }
-
- /**
- * @return disk space used
- * @throws IOException if the shell command fails
- */
- public long getUsed() throws IOException {
- //if the updating thread isn't started, update on demand
- if(refreshUsed == null) {
- run();
- } else {
- synchronized (DU.this) {
- //if an exception was thrown in the last run, rethrow
- if(duException != null) {
- IOException tmp = duException;
- duException = null;
- throw tmp;
- }
+ @Override
+ protected void parseExecResult(BufferedReader lines) throws IOException {
+ String line = lines.readLine();
+ if (line == null) {
+ throw new IOException("Expecting a line not the end of stream");
+ }
+ String[] tokens = line.split("\t");
+ if (tokens.length == 0) {
+ throw new IOException("Illegal du output");
}
+ setUsed(Long.parseLong(tokens[0]) * 1024);
}
-
- return Math.max(used.longValue(), 0L);
- }
-
- /**
- * @return the path of which we're keeping track of disk usage
- */
- public String getDirPath() {
- return dirPath;
- }
-
- /**
- * Override to hook in DUHelper class. Maybe this can be used more
- * generally as well on Unix/Linux based systems
- */
- @Override
- protected void run() throws IOException {
- if (WINDOWS) {
- used.set(DUHelper.getFolderUsage(dirPath));
- return;
- }
- super.run();
- }
-
- /**
- * Start the disk usage checking thread.
- */
- public void start() {
- //only start the thread if the interval is sane
- if(refreshInterval > 0) {
- refreshUsed = new Thread(new DURefreshThread(),
- "refreshUsed-"+dirPath);
- refreshUsed.setDaemon(true);
- refreshUsed.start();
- }
- }
-
- /**
- * Shut down the refreshing thread.
- */
- public void shutdown() {
- this.shouldRun = false;
-
- if(this.refreshUsed != null) {
- this.refreshUsed.interrupt();
- }
- }
-
- @Override
- public String toString() {
- return
- "du -sk " + dirPath +"\n" +
- used + "\t" + dirPath;
}
- @Override
- protected String[] getExecString() {
- return new String[] {"du", "-sk", dirPath};
- }
-
- @Override
- protected void parseExecResult(BufferedReader lines) throws IOException {
- String line = lines.readLine();
- if (line == null) {
- throw new IOException("Expecting a line not the end of stream");
- }
- String[] tokens = line.split("\t");
- if(tokens.length == 0) {
- throw new IOException("Illegal du output");
- }
- this.used.set(Long.parseLong(tokens[0])*1024);
- }
public static void main(String[] args) throws Exception {
String path = ".";
@@ -245,6 +91,10 @@ public class DU extends Shell {
path = args[0];
}
- System.out.println(new DU(new File(path), new Configuration()).toString());
+ GetSpaceUsed du = new GetSpaceUsed.Builder().setPath(new File(path))
+ .setConf(new Configuration())
+ .build();
+ String duResult = du.toString();
+ System.out.println(duResult);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f07705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GetSpaceUsed.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GetSpaceUsed.java
new file mode 100644
index 0000000..aebc3f7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GetSpaceUsed.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+public interface GetSpaceUsed {
+ long getUsed() throws IOException;
+
+ /**
+ * The builder class
+ */
+ final class Builder {
+ static final Logger LOG = LoggerFactory.getLogger(Builder.class);
+
+ static final String CLASSNAME_KEY = "fs.getspaceused.classname";
+
+ private Configuration conf;
+ private Class<? extends GetSpaceUsed> klass = null;
+ private File path = null;
+ private Long interval = null;
+ private Long initialUsed = null;
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public Builder setConf(Configuration conf) {
+ this.conf = conf;
+ return this;
+ }
+
+ public long getInterval() {
+ if (interval != null) {
+ return interval;
+ }
+ long result = CommonConfigurationKeys.FS_DU_INTERVAL_DEFAULT;
+ if (conf == null) {
+ return result;
+ }
+ return conf.getLong(CommonConfigurationKeys.FS_DU_INTERVAL_KEY, result);
+ }
+
+ public Builder setInterval(long interval) {
+ this.interval = interval;
+ return this;
+ }
+
+ public Class<? extends GetSpaceUsed> getKlass() {
+ if (klass != null) {
+ return klass;
+ }
+ Class<? extends GetSpaceUsed> result = null;
+ if (Shell.WINDOWS) {
+ result = WindowsGetSpaceUsed.class;
+ } else {
+ result = DU.class;
+ }
+ if (conf == null) {
+ return result;
+ }
+ return conf.getClass(CLASSNAME_KEY, result, GetSpaceUsed.class);
+ }
+
+ public Builder setKlass(Class<? extends GetSpaceUsed> klass) {
+ this.klass = klass;
+ return this;
+ }
+
+ public File getPath() {
+ return path;
+ }
+
+ public Builder setPath(File path) {
+ this.path = path;
+ return this;
+ }
+
+ public long getInitialUsed() {
+ if (initialUsed == null) {
+ return -1;
+ }
+ return initialUsed;
+ }
+
+ public Builder setInitialUsed(long initialUsed) {
+ this.initialUsed = initialUsed;
+ return this;
+ }
+
+ public GetSpaceUsed build() throws IOException {
+ GetSpaceUsed getSpaceUsed = null;
+ try {
+ Constructor<? extends GetSpaceUsed> cons =
+ getKlass().getConstructor(Builder.class);
+ getSpaceUsed = cons.newInstance(this);
+ } catch (InstantiationException e) {
+ LOG.warn("Error trying to create an instance of " + getKlass(), e);
+ } catch (IllegalAccessException e) {
+ LOG.warn("Error trying to create " + getKlass(), e);
+ } catch (InvocationTargetException e) {
+ LOG.warn("Error trying to create " + getKlass(), e);
+ } catch (NoSuchMethodException e) {
+ LOG.warn("Doesn't look like the class " + getKlass() +
+ " have the needed constructor", e);
+ }
+ // If there were any exceptions then du will be null.
+ // Construct our best guess fallback.
+ if (getSpaceUsed == null) {
+ if (Shell.WINDOWS) {
+ getSpaceUsed = new WindowsGetSpaceUsed(this);
+ } else {
+ getSpaceUsed = new DU(this);
+ }
+ }
+ // Call init after classes constructors have finished.
+ if (getSpaceUsed instanceof CachingGetSpaceUsed) {
+ ((CachingGetSpaceUsed) getSpaceUsed).init();
+ }
+ return getSpaceUsed;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f07705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/WindowsGetSpaceUsed.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/WindowsGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/WindowsGetSpaceUsed.java
new file mode 100644
index 0000000..deb1343
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/WindowsGetSpaceUsed.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.IOException;
+
+/**
+ * Class to tell the size of a path on windows.
+ * Rather than shelling out, on windows this uses DUHelper.getFolderUsage
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class WindowsGetSpaceUsed extends CachingGetSpaceUsed {
+
+
+ WindowsGetSpaceUsed(CachingGetSpaceUsed.Builder builder) throws IOException {
+ super(builder.getPath(), builder.getInterval(), builder.getInitialUsed());
+ }
+
+ /**
+ * Override to hook in DUHelper class.
+ */
+ @Override
+ protected void refresh() {
+ used.set(DUHelper.getFolderUsage(getDirPath()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f07705/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
index dded9fb..739263f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDU.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.test.GenericTestUtils;
-/** This test makes sure that "DU" does not get to run on each call to getUsed */
+/** This test makes sure that "DU" does not get to run on each call to getUsed */
public class TestDU extends TestCase {
final static private File DU_DIR = GenericTestUtils.getTestDir("dutmp");
@@ -42,7 +42,7 @@ public class TestDU extends TestCase {
public void tearDown() throws IOException {
FileUtil.fullyDelete(DU_DIR);
}
-
+
private void createFile(File newFile, int size) throws IOException {
// write random data so that filesystems with compression enabled (e.g., ZFS)
// can't compress the file
@@ -54,18 +54,18 @@ public class TestDU extends TestCase {
RandomAccessFile file = new RandomAccessFile(newFile, "rws");
file.write(data);
-
+
file.getFD().sync();
file.close();
}
/**
* Verify that du returns expected used space for a file.
- * We assume here that if a file system crates a file of size
+ * We assume here that if a file system crates a file of size
* that is a multiple of the block size in this file system,
* then the used size for the file will be exactly that size.
* This is true for most file systems.
- *
+ *
* @throws IOException
* @throws InterruptedException
*/
@@ -78,28 +78,29 @@ public class TestDU extends TestCase {
createFile(file, writtenSize);
Thread.sleep(5000); // let the metadata updater catch up
-
- DU du = new DU(file, 10000);
- du.start();
+
+ DU du = new DU(file, 10000, -1);
+ du.init();
long duSize = du.getUsed();
- du.shutdown();
+ du.close();
assertTrue("Invalid on-disk size",
duSize >= writtenSize &&
writtenSize <= (duSize + slack));
-
- //test with 0 interval, will not launch thread
- du = new DU(file, 0);
- du.start();
+
+ //test with 0 interval, will not launch thread
+ du = new DU(file, 0, -1);
+ du.init();
duSize = du.getUsed();
- du.shutdown();
-
+ du.close();
+
assertTrue("Invalid on-disk size",
duSize >= writtenSize &&
writtenSize <= (duSize + slack));
-
- //test without launching thread
- du = new DU(file, 10000);
+
+ //test without launching thread
+ du = new DU(file, 10000, -1);
+ du.init();
duSize = du.getUsed();
assertTrue("Invalid on-disk size",
@@ -111,8 +112,8 @@ public class TestDU extends TestCase {
assertTrue(file.createNewFile());
Configuration conf = new Configuration();
conf.setLong(CommonConfigurationKeys.FS_DU_INTERVAL_KEY, 10000L);
- DU du = new DU(file, conf);
- du.decDfsUsed(Long.MAX_VALUE);
+ DU du = new DU(file, 10000L, -1);
+ du.incDfsUsed(-Long.MAX_VALUE);
long duSize = du.getUsed();
assertTrue(String.valueOf(duSize), duSize >= 0L);
}
@@ -121,7 +122,7 @@ public class TestDU extends TestCase {
File file = new File(DU_DIR, "dataX");
createFile(file, 8192);
DU du = new DU(file, 3000, 1024);
- du.start();
+ du.init();
assertTrue("Initial usage setting not honored", du.getUsed() == 1024);
// wait until the first du runs.
@@ -131,4 +132,7 @@ public class TestDU extends TestCase {
assertTrue("Usage didn't get updated", du.getUsed() == 8192);
}
+
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f07705/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetSpaceUsed.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetSpaceUsed.java
new file mode 100644
index 0000000..f436713
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetSpaceUsed.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+
+public class TestGetSpaceUsed {
+ final static private File DIR = new File(
+ System.getProperty("test.build.data", "/tmp"), "TestGetSpaceUsed");
+
+ @Before
+ public void setUp() {
+ FileUtil.fullyDelete(DIR);
+ assertTrue(DIR.mkdirs());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtil.fullyDelete(DIR);
+ }
+
+ /**
+ * Test that the builder can create a class specified through the class.
+ */
+ @Test
+ public void testBuilderConf() throws Exception {
+ File file = new File(DIR, "testBuilderConf");
+ assertTrue(file.createNewFile());
+ Configuration conf = new Configuration();
+ conf.set("fs.getspaceused.classname", DummyDU.class.getName());
+ CachingGetSpaceUsed instance =
+ (CachingGetSpaceUsed) new CachingGetSpaceUsed.Builder()
+ .setPath(file)
+ .setInterval(0)
+ .setConf(conf)
+ .build();
+ assertNotNull(instance);
+ assertTrue(instance instanceof DummyDU);
+ assertFalse(instance.running());
+ instance.close();
+ }
+
+ @Test
+ public void testBuildInitial() throws Exception {
+ File file = new File(DIR, "testBuildInitial");
+ assertTrue(file.createNewFile());
+ CachingGetSpaceUsed instance =
+ (CachingGetSpaceUsed) new CachingGetSpaceUsed.Builder()
+ .setPath(file)
+ .setInitialUsed(90210)
+ .setKlass(DummyDU.class)
+ .build();
+ assertEquals(90210, instance.getUsed());
+ instance.close();
+ }
+
+ @Test
+ public void testBuildInterval() throws Exception {
+ File file = new File(DIR, "testBuildInitial");
+ assertTrue(file.createNewFile());
+ CachingGetSpaceUsed instance =
+ (CachingGetSpaceUsed) new CachingGetSpaceUsed.Builder()
+ .setPath(file)
+ .setInitialUsed(90210)
+ .setInterval(50060)
+ .setKlass(DummyDU.class)
+ .build();
+ assertEquals(50060, instance.getRefreshInterval());
+ instance.close();
+ }
+
+ @Test
+ public void testBuildNonCaching() throws Exception {
+ File file = new File(DIR, "testBuildNonCaching");
+ assertTrue(file.createNewFile());
+ GetSpaceUsed instance = new CachingGetSpaceUsed.Builder()
+ .setPath(file)
+ .setInitialUsed(90210)
+ .setInterval(50060)
+ .setKlass(DummyGetSpaceUsed.class)
+ .build();
+ assertEquals(300, instance.getUsed());
+ assertTrue(instance instanceof DummyGetSpaceUsed);
+ }
+
+ private static class DummyDU extends CachingGetSpaceUsed {
+
+ public DummyDU(Builder builder) throws IOException {
+ // Push to the base class.
+ // Most times that's all that will need to be done.
+ super(builder);
+ }
+
+ @Override
+ protected void refresh() {
+ // This is a test so don't du anything.
+ }
+ }
+
+ private static class DummyGetSpaceUsed implements GetSpaceUsed {
+
+ public DummyGetSpaceUsed(GetSpaceUsed.Builder builder) {
+
+ }
+
+ @Override public long getUsed() throws IOException {
+ return 300;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f07705/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index aa33851..cb8e483 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -36,8 +36,9 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DU;
+import org.apache.hadoop.fs.CachingGetSpaceUsed;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GetSpaceUsed;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -62,10 +63,10 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.io.Files;
/**
- * A block pool slice represents a portion of a block pool stored on a volume.
- * Taken together, all BlockPoolSlices sharing a block pool ID across a
+ * A block pool slice represents a portion of a block pool stored on a volume.
+ * Taken together, all BlockPoolSlices sharing a block pool ID across a
* cluster represent a single block pool.
- *
+ *
* This class is synchronized by {@link FsVolumeImpl}.
*/
class BlockPoolSlice {
@@ -92,10 +93,10 @@ class BlockPoolSlice {
private final Timer timer;
// TODO:FEDERATION scalability issue - a thread per DU is needed
- private final DU dfsUsage;
+ private final GetSpaceUsed dfsUsage;
/**
- * Create a blook pool slice
+ * Create a blook pool slice
* @param bpid Block pool Id
* @param volume {@link FsVolumeImpl} to which this BlockPool belongs to
* @param bpDir directory corresponding to the BlockPool
@@ -107,7 +108,7 @@ class BlockPoolSlice {
Configuration conf, Timer timer) throws IOException {
this.bpid = bpid;
this.volume = volume;
- this.currentDir = new File(bpDir, DataStorage.STORAGE_DIR_CURRENT);
+ this.currentDir = new File(bpDir, DataStorage.STORAGE_DIR_CURRENT);
this.finalizedDir = new File(
currentDir, DataStorage.STORAGE_DIR_FINALIZED);
this.lazypersistDir = new File(currentDir, DataStorage.STORAGE_DIR_LAZY_PERSIST);
@@ -151,8 +152,10 @@ class BlockPoolSlice {
}
// Use cached value initially if available. Or the following call will
// block until the initial du command completes.
- this.dfsUsage = new DU(bpDir, conf, loadDfsUsed());
- this.dfsUsage.start();
+ this.dfsUsage = new CachingGetSpaceUsed.Builder().setPath(bpDir)
+ .setConf(conf)
+ .setInitialUsed(loadDfsUsed())
+ .build();
// Make the dfs usage to be saved during shutdown.
ShutdownHookManager.get().addShutdownHook(
@@ -173,7 +176,7 @@ class BlockPoolSlice {
File getFinalizedDir() {
return finalizedDir;
}
-
+
File getLazypersistDir() {
return lazypersistDir;
}
@@ -188,17 +191,21 @@ class BlockPoolSlice {
/** Run DU on local drives. It must be synchronized from caller. */
void decDfsUsed(long value) {
- dfsUsage.decDfsUsed(value);
+ if (dfsUsage instanceof CachingGetSpaceUsed) {
+ ((CachingGetSpaceUsed)dfsUsage).incDfsUsed(-value);
+ }
}
-
+
long getDfsUsed() throws IOException {
return dfsUsage.getUsed();
}
void incDfsUsed(long value) {
- dfsUsage.incDfsUsed(value);
+ if (dfsUsage instanceof CachingGetSpaceUsed) {
+ ((CachingGetSpaceUsed)dfsUsage).incDfsUsed(value);
+ }
}
-
+
/**
* Read in the cached DU value and return it if it is less than
* cachedDfsUsedCheckTime which is set by
@@ -304,7 +311,10 @@ class BlockPoolSlice {
}
File blockFile = FsDatasetImpl.moveBlockFiles(b, f, blockDir);
File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp());
- dfsUsage.incDfsUsed(b.getNumBytes()+metaFile.length());
+ if (dfsUsage instanceof CachingGetSpaceUsed) {
+ ((CachingGetSpaceUsed) dfsUsage).incDfsUsed(
+ b.getNumBytes() + metaFile.length());
+ }
return blockFile;
}
@@ -331,7 +341,7 @@ class BlockPoolSlice {
}
-
+
void getVolumeMap(ReplicaMap volumeMap,
final RamDiskReplicaTracker lazyWriteReplicaMap)
throws IOException {
@@ -342,7 +352,7 @@ class BlockPoolSlice {
FsDatasetImpl.LOG.info(
"Recovered " + numRecovered + " replicas from " + lazypersistDir);
}
-
+
boolean success = readReplicasFromCache(volumeMap, lazyWriteReplicaMap);
if (!success) {
// add finalized replicas
@@ -436,7 +446,7 @@ class BlockPoolSlice {
FileUtil.fullyDelete(source);
return numRecovered;
}
-
+
private void addReplicaToReplicasMap(Block block, ReplicaMap volumeMap,
final RamDiskReplicaTracker lazyWriteReplicaMap,boolean isFinalized)
throws IOException {
@@ -444,7 +454,7 @@ class BlockPoolSlice {
long blockId = block.getBlockId();
long genStamp = block.getGenerationStamp();
if (isFinalized) {
- newReplica = new FinalizedReplica(blockId,
+ newReplica = new FinalizedReplica(blockId,
block.getNumBytes(), genStamp, volume, DatanodeUtil
.idToBlockDir(finalizedDir, blockId));
} else {
@@ -461,7 +471,7 @@ class BlockPoolSlice {
// We don't know the expected block length, so just use 0
// and don't reserve any more space for writes.
newReplica = new ReplicaBeingWritten(blockId,
- validateIntegrityAndSetLength(file, genStamp),
+ validateIntegrityAndSetLength(file, genStamp),
genStamp, volume, file.getParentFile(), null, 0);
loadRwr = false;
}
@@ -507,7 +517,7 @@ class BlockPoolSlice {
incrNumBlocks();
}
}
-
+
/**
* Add replicas under the given directory to the volume map
@@ -537,12 +547,12 @@ class BlockPoolSlice {
}
if (!Block.isBlockFilename(file))
continue;
-
+
long genStamp = FsDatasetUtil.getGenerationStampFromFile(
files, file);
long blockId = Block.filename2id(file.getName());
- Block block = new Block(blockId, file.length(), genStamp);
- addReplicaToReplicasMap(block, volumeMap, lazyWriteReplicaMap,
+ Block block = new Block(blockId, file.length(), genStamp);
+ addReplicaToReplicasMap(block, volumeMap, lazyWriteReplicaMap,
isFinalized);
}
}
@@ -636,11 +646,11 @@ class BlockPoolSlice {
/**
* Find out the number of bytes in the block that match its crc.
- *
- * This algorithm assumes that data corruption caused by unexpected
+ *
+ * This algorithm assumes that data corruption caused by unexpected
* datanode shutdown occurs only in the last crc chunk. So it checks
* only the last chunk.
- *
+ *
* @param blockFile the block file
* @param genStamp generation stamp of the block
* @return the number of valid bytes
@@ -667,7 +677,7 @@ class BlockPoolSlice {
int bytesPerChecksum = checksum.getBytesPerChecksum();
int checksumSize = checksum.getChecksumSize();
long numChunks = Math.min(
- (blockFileLen + bytesPerChecksum - 1)/bytesPerChecksum,
+ (blockFileLen + bytesPerChecksum - 1)/bytesPerChecksum,
(metaFileLen - crcHeaderLen)/checksumSize);
if (numChunks == 0) {
return 0;
@@ -710,17 +720,20 @@ class BlockPoolSlice {
IOUtils.closeStream(blockIn);
}
}
-
+
@Override
public String toString() {
return currentDir.getAbsolutePath();
}
-
+
void shutdown(BlockListAsLongs blocksListToPersist) {
saveReplicas(blocksListToPersist);
saveDfsUsed();
dfsUsedSaved = true;
- dfsUsage.shutdown();
+
+ if (dfsUsage instanceof CachingGetSpaceUsed) {
+ IOUtils.cleanup(LOG, ((CachingGetSpaceUsed) dfsUsage));
+ }
}
private boolean readReplicasFromCache(ReplicaMap volumeMap,
@@ -729,17 +742,17 @@ class BlockPoolSlice {
File replicaFile = new File(currentDir, REPLICA_CACHE_FILE);
// Check whether the file exists or not.
if (!replicaFile.exists()) {
- LOG.info("Replica Cache file: "+ replicaFile.getPath() +
+ LOG.info("Replica Cache file: "+ replicaFile.getPath() +
" doesn't exist ");
return false;
}
long fileLastModifiedTime = replicaFile.lastModified();
if (System.currentTimeMillis() > fileLastModifiedTime + replicaCacheExpiry) {
- LOG.info("Replica Cache file: " + replicaFile.getPath() +
+ LOG.info("Replica Cache file: " + replicaFile.getPath() +
" has gone stale");
// Just to make findbugs happy
if (!replicaFile.delete()) {
- LOG.info("Replica Cache file: " + replicaFile.getPath() +
+ LOG.info("Replica Cache file: " + replicaFile.getPath() +
" cannot be deleted");
}
return false;
@@ -776,7 +789,7 @@ class BlockPoolSlice {
iter.remove();
volumeMap.add(bpid, info);
}
- LOG.info("Successfully read replica from cache file : "
+ LOG.info("Successfully read replica from cache file : "
+ replicaFile.getPath());
return true;
} catch (Exception e) {
@@ -794,10 +807,10 @@ class BlockPoolSlice {
// close the inputStream
IOUtils.closeStream(inputStream);
}
- }
-
+ }
+
private void saveReplicas(BlockListAsLongs blocksListToPersist) {
- if (blocksListToPersist == null ||
+ if (blocksListToPersist == null ||
blocksListToPersist.getNumberOfBlocks()== 0) {
return;
}
@@ -813,7 +826,7 @@ class BlockPoolSlice {
replicaCacheFile.getPath());
return;
}
-
+
FileOutputStream out = null;
try {
out = new FileOutputStream(tmpFile);
@@ -827,7 +840,7 @@ class BlockPoolSlice {
// and continue.
LOG.warn("Failed to write replicas to cache ", e);
if (replicaCacheFile.exists() && !replicaCacheFile.delete()) {
- LOG.warn("Failed to delete replicas file: " +
+ LOG.warn("Failed to delete replicas file: " +
replicaCacheFile.getPath());
}
} finally {
[06/21] hadoop git commit: YARN-4810. NM applicationpage cause
internal error 500. Contributed by Bibin A Chundatt.
Posted by ae...@apache.org.
YARN-4810. NM applicationpage cause internal error 500. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/437e9d64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/437e9d64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/437e9d64
Branch: refs/heads/HDFS-1312
Commit: 437e9d6475a91cafc4c993b206312912b5f13ad9
Parents: 209303b
Author: Naganarasimha <na...@apache.org>
Authored: Tue Apr 12 17:59:46 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Tue Apr 12 17:59:46 2016 +0530
----------------------------------------------------------------------
.../nodemanager/webapp/ApplicationPage.java | 18 +++-
.../nodemanager/webapp/TestNMAppsPage.java | 86 ++++++++++++++++++++
2 files changed, 101 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/437e9d64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java
index 1a92491..bc90d8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
@@ -75,10 +76,21 @@ public class ApplicationPage extends NMView implements YarnWebParams {
@Override
protected void render(Block html) {
- ApplicationId applicationID =
- ConverterUtils.toApplicationId(this.recordFactory,
- $(APPLICATION_ID));
+ ApplicationId applicationID = null;
+ try {
+ applicationID = ConverterUtils.toApplicationId(this.recordFactory,
+ $(APPLICATION_ID));
+ } catch (IllegalArgumentException e) {
+ html.p()._("Invalid Application Id " + $(APPLICATION_ID))._();
+ return;
+ }
+ DIV<Hamlet> div = html.div("#content");
Application app = this.nmContext.getApplications().get(applicationID);
+ if (app == null) {
+ div.h1("Unknown application with id " + applicationID
+ + ". Application might have been completed")._();
+ return;
+ }
AppInfo info = new AppInfo(app);
info("Application's information")
._("ApplicationId", info.getId())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/437e9d64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMAppsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMAppsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMAppsPage.java
new file mode 100644
index 0000000..e64d43c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMAppsPage.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.webapp;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.ApplicationPage.ApplicationBlock;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.google.inject.Binder;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+@RunWith(Parameterized.class)
+public class TestNMAppsPage {
+
+ String applicationid;
+
+ public TestNMAppsPage(String appid) {
+ this.applicationid = appid;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> getAppIds() {
+ return Arrays.asList(new Object[][] { { "appid" },
+ { "application_123123213_0001" }, { "" } });
+ }
+
+ @Test
+ public void testNMAppsPage() {
+ Configuration conf = new Configuration();
+ final NMContext nmcontext = new NMContext(
+ new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(),
+ null, new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ Injector injector = WebAppTests.createMockInjector(NMContext.class,
+ nmcontext, new Module() {
+ @Override
+ public void configure(Binder binder) {
+ NodeManager nm = TestNMAppsPage.mocknm(nmcontext);
+ binder.bind(NodeManager.class).toInstance(nm);
+ binder.bind(Context.class).toInstance(nmcontext);
+ }
+ });
+ ApplicationBlock instance = injector.getInstance(ApplicationBlock.class);
+ instance.set(YarnWebParams.APPLICATION_ID, applicationid);
+ instance.render();
+ }
+
+ protected static NodeManager mocknm(NMContext nmcontext) {
+ NodeManager rm = mock(NodeManager.class);
+ when(rm.getNMContext()).thenReturn(nmcontext);
+ return rm;
+ }
+
+}
[08/21] hadoop git commit: HDFS-9918. Erasure Coding: Sort located
striped blocks based on decommissioned states. Contributed by Rakesh R.
Posted by ae...@apache.org.
HDFS-9918. Erasure Coding: Sort located striped blocks based on decommissioned states. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ef42873
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ef42873
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ef42873
Branch: refs/heads/HDFS-1312
Commit: 6ef42873a02bfcbff5521869f4d6f66539d1db41
Parents: 600d129
Author: Zhe Zhang <zh...@apache.org>
Authored: Tue Apr 12 13:38:58 2016 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Tue Apr 12 13:38:58 2016 -0700
----------------------------------------------------------------------
.../server/blockmanagement/DatanodeManager.java | 115 +++-
.../hdfs/server/namenode/FSNamesystem.java | 22 +-
.../hdfs/TestDecommissionWithStriped.java | 76 ++-
.../TestSortLocatedStripedBlock.java | 557 +++++++++++++++++++
4 files changed, 731 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ef42873/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index cd1bdab..da02a90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringStr
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.*;
import org.apache.hadoop.net.NetworkTopology.InvalidTopologyException;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ReflectionUtils;
import java.io.IOException;
@@ -368,49 +370,110 @@ public class DatanodeManager {
}
- /** Sort the located blocks by the distance to the target host. */
- public void sortLocatedBlocks(final String targethost,
- final List<LocatedBlock> locatedblocks) {
- //sort the blocks
+ /**
+ * Sort the non-striped located blocks by the distance to the target host.
+ *
+ * For striped blocks, it will only move decommissioned/stale nodes to the
+ * bottom. For example, assume we have storage list:
+ * d0, d1, d2, d3, d4, d5, d6, d7, d8, d9
+ * mapping to block indices:
+ * 0, 1, 2, 3, 4, 5, 6, 7, 8, 2
+ *
+ * Here the internal block b2 is duplicated, locating in d2 and d9. If d2 is
+ * a decommissioning node then should switch d2 and d9 in the storage list.
+ * After sorting locations, will update corresponding block indices
+ * and block tokens.
+ */
+ public void sortLocatedBlocks(final String targetHost,
+ final List<LocatedBlock> locatedBlocks) {
+ Comparator<DatanodeInfo> comparator = avoidStaleDataNodesForRead ?
+ new DFSUtil.DecomStaleComparator(staleInterval) :
+ DFSUtil.DECOM_COMPARATOR;
+ // sort located block
+ for (LocatedBlock lb : locatedBlocks) {
+ if (lb.isStriped()) {
+ sortLocatedStripedBlock(lb, comparator);
+ } else {
+ sortLocatedBlock(lb, targetHost, comparator);
+ }
+ }
+ }
+
+ /**
+ * Move decommissioned/stale datanodes to the bottom. After sorting it will
+ * update block indices and block tokens respectively.
+ *
+ * @param lb located striped block
+ * @param comparator dn comparator
+ */
+ private void sortLocatedStripedBlock(final LocatedBlock lb,
+ Comparator<DatanodeInfo> comparator) {
+ DatanodeInfo[] di = lb.getLocations();
+ HashMap<DatanodeInfo, Byte> locToIndex = new HashMap<>();
+ HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken =
+ new HashMap<>();
+ LocatedStripedBlock lsb = (LocatedStripedBlock) lb;
+ for (int i = 0; i < di.length; i++) {
+ locToIndex.put(di[i], lsb.getBlockIndices()[i]);
+ locToToken.put(di[i], lsb.getBlockTokens()[i]);
+ }
+ // Move decommissioned/stale datanodes to the bottom
+ Arrays.sort(di, comparator);
+
+ // must update cache since we modified locations array
+ lb.updateCachedStorageInfo();
+
+ // must update block indices and block tokens respectively
+ for (int i = 0; i < di.length; i++) {
+ lsb.getBlockIndices()[i] = locToIndex.get(di[i]);
+ lsb.getBlockTokens()[i] = locToToken.get(di[i]);
+ }
+ }
+
+ /**
+ * Move decommissioned/stale datanodes to the bottom. Also, sort nodes by
+ * network distance.
+ *
+ * @param lb located block
+ * @param targetHost target host
+ * @param comparator dn comparator
+ */
+ private void sortLocatedBlock(final LocatedBlock lb, String targetHost,
+ Comparator<DatanodeInfo> comparator) {
// As it is possible for the separation of node manager and datanode,
// here we should get node but not datanode only .
- Node client = getDatanodeByHost(targethost);
+ Node client = getDatanodeByHost(targetHost);
if (client == null) {
List<String> hosts = new ArrayList<> (1);
- hosts.add(targethost);
+ hosts.add(targetHost);
List<String> resolvedHosts = dnsToSwitchMapping.resolve(hosts);
if (resolvedHosts != null && !resolvedHosts.isEmpty()) {
String rName = resolvedHosts.get(0);
if (rName != null) {
client = new NodeBase(rName + NodeBase.PATH_SEPARATOR_STR +
- targethost);
+ targetHost);
}
} else {
LOG.error("Node Resolution failed. Please make sure that rack " +
"awareness scripts are functional.");
}
}
-
- Comparator<DatanodeInfo> comparator = avoidStaleDataNodesForRead ?
- new DFSUtil.DecomStaleComparator(staleInterval) :
- DFSUtil.DECOM_COMPARATOR;
-
- for (LocatedBlock b : locatedblocks) {
- DatanodeInfo[] di = b.getLocations();
- // Move decommissioned/stale datanodes to the bottom
- Arrays.sort(di, comparator);
-
- int lastActiveIndex = di.length - 1;
- while (lastActiveIndex > 0 && isInactive(di[lastActiveIndex])) {
- --lastActiveIndex;
- }
- int activeLen = lastActiveIndex + 1;
- networktopology.sortByDistance(client, b.getLocations(), activeLen);
- // must update cache since we modified locations array
- b.updateCachedStorageInfo();
+
+ DatanodeInfo[] di = lb.getLocations();
+ // Move decommissioned/stale datanodes to the bottom
+ Arrays.sort(di, comparator);
+
+ // Sort nodes by network distance only for located blocks
+ int lastActiveIndex = di.length - 1;
+ while (lastActiveIndex > 0 && isInactive(di[lastActiveIndex])) {
+ --lastActiveIndex;
}
+ int activeLen = lastActiveIndex + 1;
+ networktopology.sortByDistance(client, lb.getLocations(), activeLen);
+
+ // must update cache since we modified locations array
+ lb.updateCachedStorageInfo();
}
-
/** @return the datanode descriptor for the host. */
public DatanodeDescriptor getDatanodeByHost(final String host) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ef42873/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 00e1241..39b3598 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -184,7 +184,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeException;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@@ -1779,25 +1778,28 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
LocatedBlocks blocks = res.blocks;
+ sortLocatedBlocks(clientMachine, blocks);
+ return blocks;
+ }
+
+ private void sortLocatedBlocks(String clientMachine, LocatedBlocks blocks) {
if (blocks != null) {
List<LocatedBlock> blkList = blocks.getLocatedBlocks();
- if (blkList == null || blkList.size() == 0 ||
- blkList.get(0) instanceof LocatedStripedBlock) {
- // no need to sort locations for striped blocks
- return blocks;
+ if (blkList == null || blkList.size() == 0) {
+ // simply return, block list is empty
+ return;
}
- blockManager.getDatanodeManager().sortLocatedBlocks(
- clientMachine, blkList);
+ blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
+ blkList);
// lastBlock is not part of getLocatedBlocks(), might need to sort it too
LocatedBlock lastBlock = blocks.getLastLocatedBlock();
if (lastBlock != null) {
ArrayList<LocatedBlock> lastBlockList = Lists.newArrayList(lastBlock);
- blockManager.getDatanodeManager().sortLocatedBlocks(
- clientMachine, lastBlockList);
+ blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
+ lastBlockList);
}
}
- return blocks;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ef42873/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
index bde2ceb..c0d8268 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
@@ -27,6 +27,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CountDownLatch;
@@ -49,12 +50,15 @@ import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.PathUtils;
import org.junit.After;
import org.junit.Assert;
@@ -160,6 +164,13 @@ public class TestDecommissionWithStriped {
}
@Test(timeout = 120000)
+ public void testFileMultipleBlockGroups() throws Exception {
+ LOG.info("Starting test testFileMultipleBlockGroups");
+ int writeBytes = 2 * blockSize * dataBlocks;
+ testDecommission(writeBytes, 9, 1, "testFileMultipleBlockGroups");
+ }
+
+ @Test(timeout = 120000)
public void testFileSmallerThanOneCell() throws Exception {
LOG.info("Starting test testFileSmallerThanOneCell");
testDecommission(cellSize - 1, 4, 1, "testFileSmallerThanOneCell");
@@ -274,7 +285,15 @@ public class TestDecommissionWithStriped {
int deadDecomissioned = fsn.getNumDecomDeadDataNodes();
int liveDecomissioned = fsn.getNumDecomLiveDataNodes();
- ((HdfsDataInputStream) dfs.open(ecFile)).getAllBlocks();
+ List<LocatedBlock> lbs = ((HdfsDataInputStream) dfs.open(ecFile))
+ .getAllBlocks();
+
+ // prepare expected block index and token list.
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList = new ArrayList<>();
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList =
+ new ArrayList<>();
+ prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
+
// Decommission node. Verify that node is decommissioned.
decommissionNode(0, decommisionNodes, AdminStates.DECOMMISSIONED);
@@ -290,9 +309,55 @@ public class TestDecommissionWithStriped {
assertNull(checkFile(dfs, ecFile, storageCount, decommisionNodes, numDNs));
StripedFileTestUtil.checkData(dfs, ecFile, writeBytes, decommisionNodes,
null);
+
+ assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
+
cleanupFile(dfs, ecFile);
}
+ private void prepareBlockIndexAndTokenList(List<LocatedBlock> lbs,
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList,
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList) {
+ for (LocatedBlock lb : lbs) {
+ HashMap<DatanodeInfo, Byte> locToIndex = new HashMap<DatanodeInfo, Byte>();
+ locToIndexList.add(locToIndex);
+
+ HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken =
+ new HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>();
+ locToTokenList.add(locToToken);
+
+ DatanodeInfo[] di = lb.getLocations();
+ LocatedStripedBlock stripedBlk = (LocatedStripedBlock) lb;
+ for (int i = 0; i < di.length; i++) {
+ locToIndex.put(di[i], stripedBlk.getBlockIndices()[i]);
+ locToToken.put(di[i], stripedBlk.getBlockTokens()[i]);
+ }
+ }
+ }
+
+ /**
+ * Verify block index and token values. Must update block indices and block
+ * tokens after sorting.
+ */
+ private void assertBlockIndexAndTokenPosition(List<LocatedBlock> lbs,
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList,
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList) {
+ for (int i = 0; i < lbs.size(); i++) {
+ LocatedBlock lb = lbs.get(i);
+ LocatedStripedBlock stripedBlk = (LocatedStripedBlock) lb;
+ HashMap<DatanodeInfo, Byte> locToIndex = locToIndexList.get(i);
+ HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken =
+ locToTokenList.get(i);
+ DatanodeInfo[] di = lb.getLocations();
+ for (int j = 0; j < di.length; j++) {
+ Assert.assertEquals("Block index value mismatches after sorting",
+ (byte) locToIndex.get(di[j]), stripedBlk.getBlockIndices()[j]);
+ Assert.assertEquals("Block token value mismatches after sorting",
+ locToToken.get(di[j]), stripedBlk.getBlockTokens()[j]);
+ }
+ }
+ }
+
private List<DatanodeInfo> getDecommissionDatanode(DistributedFileSystem dfs,
Path ecFile, int writeBytes, int decomNodeCount) throws IOException {
ArrayList<DatanodeInfo> decommissionedNodes = new ArrayList<>();
@@ -447,7 +512,12 @@ public class TestDecommissionWithStriped {
return "For block " + blk.getBlock() + " replica on " + nodes[j]
+ " is given as downnode, " + "but is not decommissioned";
}
- // TODO: Add check to verify that the Decommissioned node (if any)
+ // Decommissioned node (if any) should only be last node in list.
+ if (j < repl) {
+ return "For block " + blk.getBlock() + " decommissioned node "
+ + nodes[j] + " was not last node in list: " + (j + 1) + " of "
+ + nodes.length;
+ }
// should only be last node in list.
LOG.info("Block " + blk.getBlock() + " replica on " + nodes[j]
+ " is decommissioned.");
@@ -470,4 +540,4 @@ public class TestDecommissionWithStriped {
}
return null;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ef42873/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java
new file mode 100644
index 0000000..1dd067d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java
@@ -0,0 +1,557 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class tests the sorting of located striped blocks based on
+ * decommissioned states.
+ */
+public class TestSortLocatedStripedBlock {
+ static final Logger LOG = LoggerFactory
+ .getLogger(TestSortLocatedStripedBlock.class);
+ static final int BLK_GROUP_WIDTH = StripedFileTestUtil.NUM_DATA_BLOCKS
+ + StripedFileTestUtil.NUM_PARITY_BLOCKS;
+ static final int NUM_DATA_BLOCKS = StripedFileTestUtil.NUM_DATA_BLOCKS;
+ static final int NUM_PARITY_BLOCKS = StripedFileTestUtil.NUM_PARITY_BLOCKS;
+ static DatanodeManager dm;
+ static final long STALE_INTERVAL = 30 * 1000 * 60;
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ dm = mockDatanodeManager();
+ }
+
+ /**
+ * Test to verify sorting with multiple decommissioned datanodes exists in
+ * storage lists.
+ *
+ * We have storage list, marked decommissioned internal blocks with a '
+ * d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12
+ * mapping to indices
+ * 0', 1', 2, 3, 4, 5, 6, 7', 8', 0, 1, 7, 8
+ *
+ * Decommissioned node indices: 0, 1, 7, 8
+ *
+ * So in the original list nodes d0, d1, d7, d8 are decommissioned state.
+ *
+ * After sorting the expected block indices list should be,
+ * 0, 1, 2, 3, 4, 5, 6, 7, 8, 0', 1', 7', 8'
+ *
+ * After sorting the expected storage list will be,
+ * d9, d10, d2, d3, d4, d5, d6, d11, d12, d0, d1, d7, d8.
+ *
+ * Note: after sorting block indices will not be in ascending order.
+ */
+ @Test(timeout = 10000)
+ public void testWithMultipleDecommnDatanodes() {
+ LOG.info("Starting test testSortWithMultipleDecommnDatanodes");
+ int lbsCount = 2; // two located block groups
+ List<Integer> decommnNodeIndices = new ArrayList<>();
+ decommnNodeIndices.add(0);
+ decommnNodeIndices.add(1);
+ decommnNodeIndices.add(7);
+ decommnNodeIndices.add(8);
+ List<Integer> targetNodeIndices = new ArrayList<>();
+ targetNodeIndices.addAll(decommnNodeIndices);
+ // map contains decommissioned node details in each located strip block
+ // which will be used for assertions
+ HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
+ lbsCount * decommnNodeIndices.size());
+ List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
+ NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ targetNodeIndices, decommissionedNodes);
+
+ // prepare expected block index and token list.
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList = new ArrayList<>();
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList =
+ new ArrayList<>();
+ prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
+
+ dm.sortLocatedBlocks(null, lbs);
+
+ assertDecommnNodePosition(BLK_GROUP_WIDTH, decommissionedNodes, lbs);
+ assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
+ }
+
+ /**
+ * Test to verify sorting with two decommissioned datanodes exists in
+ * storage lists for the same block index.
+ *
+ * We have storage list, marked decommissioned internal blocks with a '
+ * d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12, d13
+ * mapping to indices
+ * 0', 1', 2, 3, 4', 5', 6, 7, 8, 0, 1', 4, 5, 1
+ *
+ * Decommissioned node indices: 0', 1', 4', 5', 1'
+ *
+ * Here decommissioned has done twice to the datanode block index 1.
+ * So in the original list nodes d0, d1, d4, d5, d10 are decommissioned state.
+ *
+ * After sorting the expected block indices list will be,
+ * 0, 1, 2, 3, 4, 5, 6, 7, 8, 0', 1', 1', 4', 5'
+ *
+ * After sorting the expected storage list will be,
+ * d9, d13, d2, d3, d11, d12, d6, d7, d8, d0, d1, d10, d4, d5.
+ *
+ * Note: after sorting block indices will not be in ascending order.
+ */
+ @Test(timeout = 10000)
+ public void testTwoDatanodesWithSameBlockIndexAreDecommn() {
+ LOG.info("Starting test testTwoDatanodesWithSameBlockIndexAreDecommn");
+ int lbsCount = 2; // two located block groups
+ List<Integer> decommnNodeIndices = new ArrayList<>();
+ decommnNodeIndices.add(0);
+ decommnNodeIndices.add(1);
+ decommnNodeIndices.add(4);
+ decommnNodeIndices.add(5);
+ // representing blockIndex 1, later this also decommissioned
+ decommnNodeIndices.add(1);
+
+ List<Integer> targetNodeIndices = new ArrayList<>();
+ targetNodeIndices.addAll(decommnNodeIndices);
+ // map contains decommissioned node details in each located strip block
+ // which will be used for assertions
+ HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
+ lbsCount * decommnNodeIndices.size());
+ List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
+ NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ targetNodeIndices, decommissionedNodes);
+
+ // prepare expected block index and token list.
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList = new ArrayList<>();
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList =
+ new ArrayList<>();
+ prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
+
+ dm.sortLocatedBlocks(null, lbs);
+ assertDecommnNodePosition(BLK_GROUP_WIDTH, decommissionedNodes, lbs);
+ assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
+ }
+
+ /**
+ * Test to verify sorting with decommissioned datanodes exists in storage
+ * list which is smaller than stripe size.
+ *
+ * We have storage list, marked decommissioned internal blocks with a '
+ * d0, d1, d2, d3, d6, d7, d8, d9, d10, d11
+ * mapping to indices
+ * 0', 1, 2', 3, 6, 7, 8, 0, 2', 2
+ *
+ * Decommissioned node indices: 0', 2', 2'
+ *
+ * Here decommissioned has done twice to the datanode block index 2.
+ * So in the original list nodes d0, d2, d10 are decommissioned state.
+ *
+ * After sorting the expected block indices list should be,
+ * 0, 1, 2, 3, 6, 7, 8, 0', 2', 2'
+ *
+ * After sorting the expected storage list will be,
+ * d9, d1, d11, d3, d6, d7, d8, d0, d2, d10.
+ *
+ * Note: after sorting block indices will not be in ascending order.
+ */
+ @Test(timeout = 10000)
+ public void testSmallerThanOneStripeWithMultpleDecommnNodes()
+ throws Exception {
+ LOG.info("Starting test testSmallerThanOneStripeWithDecommn");
+ int lbsCount = 2; // two located block groups
+ List<Integer> decommnNodeIndices = new ArrayList<>();
+ decommnNodeIndices.add(0);
+ decommnNodeIndices.add(2);
+ // representing blockIndex 1, later this also decommissioned
+ decommnNodeIndices.add(2);
+
+ List<Integer> targetNodeIndices = new ArrayList<>();
+ targetNodeIndices.addAll(decommnNodeIndices);
+ // map contains decommissioned node details in each located strip block
+ // which will be used for assertions
+ HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
+ lbsCount * decommnNodeIndices.size());
+ int dataBlksNum = NUM_DATA_BLOCKS - 2;
+ List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount, dataBlksNum,
+ NUM_PARITY_BLOCKS, decommnNodeIndices, targetNodeIndices,
+ decommissionedNodes);
+
+ // prepare expected block index and token list.
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList = new ArrayList<>();
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList =
+ new ArrayList<>();
+ prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
+
+ dm.sortLocatedBlocks(null, lbs);
+
+ // After this index all are decommissioned nodes.
+ int blkGrpWidth = dataBlksNum + NUM_PARITY_BLOCKS;
+ assertDecommnNodePosition(blkGrpWidth, decommissionedNodes, lbs);
+ assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
+ }
+
+ /**
+ * Test to verify sorting with decommissioned datanodes exists in storage
+ * list but the corresponding new target datanode doesn't exists.
+ *
+ * We have storage list, marked decommissioned internal blocks with a '
+ * d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11
+ * mapping to indices
+ * 0', 1', 2', 3, 4', 5', 6, 7, 8, 0, 2, 4
+ *
+ * Decommissioned node indices: 0', 1', 2', 4', 5'
+ *
+ * 1 and 5 nodes doesn't exists in the target list. This can happen, the
+ * target node block corrupted or lost after the successful decommissioning.
+ * So in the original list nodes corresponding to the decommissioned block
+ * index 1 and 5 doesn't have any target entries.
+ *
+ * After sorting the expected block indices list should be,
+ * 0, 2, 3, 4, 6, 7, 8, 0', 1', 2', 4', 5'
+ *
+ * After sorting the expected storage list will be,
+ * d9, d10, d3, d11, d6, d7, d8, d0, d1, d2, d4, d5.
+ *
+ * Note: after sorting block indices will not be in ascending order.
+ */
+ @Test(timeout = 10000)
+ public void testTargetDecommnDatanodeDoesntExists() {
+ LOG.info("Starting test testTargetDecommnDatanodeDoesntExists");
+ int lbsCount = 2; // two located block groups
+ List<Integer> decommnNodeIndices = new ArrayList<>();
+ decommnNodeIndices.add(0);
+ decommnNodeIndices.add(1);
+ decommnNodeIndices.add(2);
+ decommnNodeIndices.add(4);
+ decommnNodeIndices.add(5);
+
+ List<Integer> targetNodeIndices = new ArrayList<>();
+ targetNodeIndices.add(0);
+ targetNodeIndices.add(2);
+ targetNodeIndices.add(4);
+ // 1 and 5 nodes doesn't exists in the target list. One such case is, the
+ // target node block corrupted or lost after the successful decommissioning
+
+ // map contains decommissioned node details in each located strip block
+ // which will be used for assertions
+ HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
+ lbsCount * decommnNodeIndices.size());
+ List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
+ NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ targetNodeIndices, decommissionedNodes);
+
+ // prepare expected block index and token list.
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList = new ArrayList<>();
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList =
+ new ArrayList<>();
+ prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
+
+ dm.sortLocatedBlocks(null, lbs);
+
+ // After this index all are decommissioned nodes. Needs to reconstruct two
+ // more block indices.
+ int blkGrpWidth = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS - 2;
+ assertDecommnNodePosition(blkGrpWidth, decommissionedNodes, lbs);
+ assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
+ }
+
+ /**
+ * Test to verify sorting with multiple in-service and decommissioned
+ * datanodes exists in storage lists.
+ *
+ * We have storage list, marked decommissioned internal blocks with a '
+ * d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12, d13
+ * mapping to indices
+ * 0', 1', 2, 3, 4, 5, 6, 7', 8', 0, 1, 7, 8, 1
+ *
+ * Decommissioned node indices: 0', 1', 7', 8'
+ *
+ * Additional In-Service node d13 at the end, block index: 1
+ *
+ * So in the original list nodes d0, d1, d7, d8 are decommissioned state.
+ *
+ * After sorting the expected block indices list will be,
+ * 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 0', 1', 7', 8'
+ *
+ * After sorting the expected storage list will be,
+ * d9, d10, d2, d3, d4, d5, d6, d11, d12, d13, d0, d1, d7, d8.
+ *
+ * Note: after sorting block indices will not be in ascending order.
+ */
+ @Test(timeout = 10000)
+ public void testWithMultipleInServiceAndDecommnDatanodes() {
+ LOG.info("Starting test testWithMultipleInServiceAndDecommnDatanodes");
+ int lbsCount = 2; // two located block groups
+ List<Integer> decommnNodeIndices = new ArrayList<>();
+ decommnNodeIndices.add(0);
+ decommnNodeIndices.add(1);
+ decommnNodeIndices.add(7);
+ decommnNodeIndices.add(8);
+ List<Integer> targetNodeIndices = new ArrayList<>();
+ targetNodeIndices.addAll(decommnNodeIndices);
+
+ // at the end add an additional In-Service node to blockIndex=1
+ targetNodeIndices.add(1);
+
+ // map contains decommissioned node details in each located strip block
+ // which will be used for assertions
+ HashMap<Integer, List<String>> decommissionedNodes = new HashMap<>(
+ lbsCount * decommnNodeIndices.size());
+ List<LocatedBlock> lbs = createLocatedStripedBlocks(lbsCount,
+ NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS, decommnNodeIndices,
+ targetNodeIndices, decommissionedNodes);
+ List <DatanodeInfo> staleDns = new ArrayList<>();
+ for (LocatedBlock lb : lbs) {
+ DatanodeInfo[] locations = lb.getLocations();
+ DatanodeInfo staleDn = locations[locations.length - 1];
+ staleDn
+ .setLastUpdateMonotonic(Time.monotonicNow() - (STALE_INTERVAL * 2));
+ staleDns.add(staleDn);
+ }
+
+ // prepare expected block index and token list.
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList = new ArrayList<>();
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList =
+ new ArrayList<>();
+ prepareBlockIndexAndTokenList(lbs, locToIndexList, locToTokenList);
+
+ dm.sortLocatedBlocks(null, lbs);
+
+ assertDecommnNodePosition(BLK_GROUP_WIDTH + 1, decommissionedNodes, lbs);
+ assertBlockIndexAndTokenPosition(lbs, locToIndexList, locToTokenList);
+
+ for (LocatedBlock lb : lbs) {
+ byte[] blockIndices = ((LocatedStripedBlock) lb).getBlockIndices();
+ // after sorting stale block index will be placed after normal nodes.
+ Assert.assertEquals("Failed to move stale node to bottom!", 1,
+ blockIndices[9]);
+ DatanodeInfo[] locations = lb.getLocations();
+ // After sorting stale node d13 will be placed after normal nodes
+ Assert.assertEquals("Failed to move stale dn after normal one!",
+ staleDns.remove(0), locations[9]);
+ }
+ }
+
+ /**
+ * Verify that decommissioned/stale nodes must be positioned after normal
+ * nodes.
+ */
+ private void assertDecommnNodePosition(int blkGrpWidth,
+ HashMap<Integer, List<String>> decommissionedNodes,
+ List<LocatedBlock> lbs) {
+ for (int i = 0; i < lbs.size(); i++) { // for each block
+ LocatedBlock blk = lbs.get(i);
+ DatanodeInfo[] nodes = blk.getLocations();
+ List<String> decommissionedNodeList = decommissionedNodes.get(i);
+
+ for (int j = 0; j < nodes.length; j++) { // for each replica
+ DatanodeInfo dnInfo = nodes[j];
+ LOG.info("Block Locations size={}, locs={}, j=", nodes.length,
+ dnInfo.toString(), j);
+ if (j < blkGrpWidth) {
+ Assert.assertEquals("Node shouldn't be decommissioned",
+ AdminStates.NORMAL, dnInfo.getAdminState());
+ } else {
+ // check against decommissioned list
+ Assert.assertTrue(
+ "For block " + blk.getBlock() + " decommissioned node " + dnInfo
+ + " is not last node in list: " + j + "th index of "
+ + nodes.length,
+ decommissionedNodeList.contains(dnInfo.getXferAddr()));
+ Assert.assertEquals("Node should be decommissioned",
+ AdminStates.DECOMMISSIONED, dnInfo.getAdminState());
+ }
+ }
+ }
+ }
+
+ private List<LocatedBlock> createLocatedStripedBlocks(int blkGrpCount,
+ int dataNumBlk, int numParityBlk, List<Integer> decommnNodeIndices,
+ List<Integer> targetNodeIndices,
+ HashMap<Integer, List<String>> decommissionedNodes) {
+
+ final List<LocatedBlock> lbs = new ArrayList<>(blkGrpCount);
+ for (int i = 0; i < blkGrpCount; i++) {
+ ArrayList<String> decommNodeInfo = new ArrayList<String>();
+ decommissionedNodes.put(new Integer(i), decommNodeInfo);
+ List<Integer> dummyDecommnNodeIndices = new ArrayList<>();
+ dummyDecommnNodeIndices.addAll(decommnNodeIndices);
+
+ LocatedStripedBlock lsb = createEachLocatedBlock(dataNumBlk, numParityBlk,
+ dummyDecommnNodeIndices, targetNodeIndices, decommNodeInfo);
+ lbs.add(lsb);
+ }
+ return lbs;
+ }
+
+ private LocatedStripedBlock createEachLocatedBlock(int numDataBlk,
+ int numParityBlk, List<Integer> decommnNodeIndices,
+ List<Integer> targetNodeIndices, ArrayList<String> decommNodeInfo) {
+ final long blockGroupID = Long.MIN_VALUE;
+ int totalDns = numDataBlk + numParityBlk + targetNodeIndices.size();
+ DatanodeInfo[] locs = new DatanodeInfo[totalDns];
+ String[] storageIDs = new String[totalDns];
+ StorageType[] storageTypes = new StorageType[totalDns];
+ byte[] blkIndices = new byte[totalDns];
+
+ // Adding data blocks
+ int index = 0;
+ for (; index < numDataBlk; index++) {
+ blkIndices[index] = (byte) index;
+ // Location port always equal to logical index of a block,
+ // for easier verification
+ locs[index] = DFSTestUtil.getLocalDatanodeInfo(blkIndices[index]);
+ locs[index].setLastUpdateMonotonic(Time.monotonicNow());
+ storageIDs[index] = locs[index].getDatanodeUuid();
+ storageTypes[index] = StorageType.DISK;
+ // set decommissioned state
+ if (decommnNodeIndices.contains(index)) {
+ locs[index].setDecommissioned();
+ decommNodeInfo.add(locs[index].toString());
+ // Removing it from the list to ensure that all the given nodes are
+ // successfully marked as decomissioned.
+ decommnNodeIndices.remove(new Integer(index));
+ }
+ }
+ // Adding parity blocks after data blocks
+ index = NUM_DATA_BLOCKS;
+ for (int j = numDataBlk; j < numDataBlk + numParityBlk; j++, index++) {
+ blkIndices[j] = (byte) index;
+ // Location port always equal to logical index of a block,
+ // for easier verification
+ locs[j] = DFSTestUtil.getLocalDatanodeInfo(blkIndices[j]);
+ locs[j].setLastUpdateMonotonic(Time.monotonicNow());
+ storageIDs[j] = locs[j].getDatanodeUuid();
+ storageTypes[j] = StorageType.DISK;
+ // set decommissioned state
+ if (decommnNodeIndices.contains(index)) {
+ locs[j].setDecommissioned();
+ decommNodeInfo.add(locs[j].toString());
+ // Removing it from the list to ensure that all the given nodes are
+ // successfully marked as decomissioned.
+ decommnNodeIndices.remove(new Integer(index));
+ }
+ }
+ // Add extra target nodes to storage list after the parity blocks
+ int basePortValue = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ index = numDataBlk + numParityBlk;
+ for (int i = 0; i < targetNodeIndices.size(); i++, index++) {
+ int blkIndexPos = targetNodeIndices.get(i);
+ blkIndices[index] = (byte) blkIndexPos;
+ // Location port always equal to logical index of a block,
+ // for easier verification
+ locs[index] = DFSTestUtil.getLocalDatanodeInfo(basePortValue++);
+ locs[index].setLastUpdateMonotonic(Time.monotonicNow());
+ storageIDs[index] = locs[index].getDatanodeUuid();
+ storageTypes[index] = StorageType.DISK;
+ // set decommissioned state. This can happen, the target node is again
+ // decommissioned by administrator
+ if (decommnNodeIndices.contains(blkIndexPos)) {
+ locs[index].setDecommissioned();
+ decommNodeInfo.add(locs[index].toString());
+ // Removing it from the list to ensure that all the given nodes are
+ // successfully marked as decomissioned.
+ decommnNodeIndices.remove(new Integer(blkIndexPos));
+ }
+ }
+ return new LocatedStripedBlock(
+ new ExtendedBlock("pool", blockGroupID,
+ StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE, 1001),
+ locs, storageIDs, storageTypes, blkIndices, 0, false, null);
+ }
+
+ private static DatanodeManager mockDatanodeManager() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY,
+ STALE_INTERVAL);
+ FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
+ BlockManager bm = Mockito.mock(BlockManager.class);
+ BlockReportLeaseManager blm = new BlockReportLeaseManager(conf);
+ Mockito.when(bm.getBlockReportLeaseManager()).thenReturn(blm);
+ DatanodeManager dm = new DatanodeManager(bm, fsn, conf);
+ return dm;
+ }
+
+ private void prepareBlockIndexAndTokenList(List<LocatedBlock> lbs,
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList,
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList) {
+ for (LocatedBlock lb : lbs) {
+ HashMap<DatanodeInfo, Byte> locToIndex = new HashMap<DatanodeInfo, Byte>();
+ locToIndexList.add(locToIndex);
+
+ HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken =
+ new HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>();
+ locToTokenList.add(locToToken);
+
+ DatanodeInfo[] di = lb.getLocations();
+ LocatedStripedBlock stripedBlk = (LocatedStripedBlock) lb;
+ for (int i = 0; i < di.length; i++) {
+ locToIndex.put(di[i], stripedBlk.getBlockIndices()[i]);
+ locToToken.put(di[i], stripedBlk.getBlockTokens()[i]);
+ }
+ }
+ }
+
+ /**
+ * Verify block index and token values. Must update block indices and block
+ * tokens after sorting.
+ */
+ private void assertBlockIndexAndTokenPosition(List<LocatedBlock> lbs,
+ List<HashMap<DatanodeInfo, Byte>> locToIndexList,
+ List<HashMap<DatanodeInfo, Token<BlockTokenIdentifier>>> locToTokenList) {
+ for (int i = 0; i < lbs.size(); i++) {
+ LocatedBlock lb = lbs.get(i);
+ LocatedStripedBlock stripedBlk = (LocatedStripedBlock) lb;
+ HashMap<DatanodeInfo, Byte> locToIndex = locToIndexList.get(i);
+ HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken =
+ locToTokenList.get(i);
+ DatanodeInfo[] di = lb.getLocations();
+ for (int j = 0; j < di.length; j++) {
+ Assert.assertEquals("Block index value mismatches after sorting",
+ (byte) locToIndex.get(di[j]), stripedBlk.getBlockIndices()[j]);
+ Assert.assertEquals("Block token value mismatches after sorting",
+ locToToken.get(di[j]), stripedBlk.getBlockTokens()[j]);
+ }
+ }
+ }
+}
\ No newline at end of file
[03/21] hadoop git commit: HDFS-10277. PositionedReadable test
testReadFullyZeroByteFile failing in HDFS. Contributed by Steve Loughran.
Posted by ae...@apache.org.
HDFS-10277. PositionedReadable test testReadFullyZeroByteFile failing in HDFS. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a409508b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a409508b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a409508b
Branch: refs/heads/HDFS-1312
Commit: a409508b3f4c46b419c41b9cdff83429d9d025ce
Parents: 4ee7f0a
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Apr 12 14:53:01 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Apr 12 14:55:01 2016 +0900
----------------------------------------------------------------------
.../markdown/filesystem/fsdatainputstream.md | 35 ++++++++++++++++----
.../org/apache/hadoop/hdfs/DFSInputStream.java | 4 +++
2 files changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a409508b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md
index de6b861..e656d6f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md
@@ -120,7 +120,8 @@ Return the data at the current position.
### <a name="InputStream.read.buffer[]"></a> `InputStream.read(buffer[], offset, length)`
Read `length` bytes of data into the destination buffer, starting at offset
-`offset`
+`offset`. The source of the data is the current position of the stream,
+as implicitly set in `pos`
#### Preconditions
@@ -129,6 +130,7 @@ Read `length` bytes of data into the destination buffer, starting at offset
length >= 0
offset < len(buffer)
length <= len(buffer) - offset
+ pos >= 0 else raise EOFException, IOException
Exceptions that may be raised on precondition failure are
@@ -136,20 +138,39 @@ Exceptions that may be raised on precondition failure are
ArrayIndexOutOfBoundsException
RuntimeException
+Not all filesystems check the `isOpen` state.
+
#### Postconditions
if length == 0 :
result = 0
- elseif pos > len(data):
- result -1
+ else if pos > len(data):
+ result = -1
else
let l = min(length, len(data)-length) :
- buffer' = buffer where forall i in [0..l-1]:
- buffer'[o+i] = data[pos+i]
- FSDIS' = (pos+l, data, true)
- result = l
+ buffer' = buffer where forall i in [0..l-1]:
+ buffer'[o+i] = data[pos+i]
+ FSDIS' = (pos+l, data, true)
+ result = l
+
+The `java.io` API states that if the amount of data to be read (i.e. `length`)
+then the call must block until the amount of data available is greater than
+zero —that is, until there is some data. The call is not required to return
+when the buffer is full, or indeed block until there is no data left in
+the stream.
+
+That is, rather than `l` being simply defined as `min(length, len(data)-length)`,
+it strictly is an integer in the range `1..min(length, len(data)-length)`.
+While the caller may expect for as much as the buffer as possible to be filled
+in, it is within the specification for an implementation to always return
+a smaller number, perhaps only ever 1 byte.
+
+What is critical is that unless the destination buffer size is 0, the call
+must block until at least one byte is returned. Thus, for any data source
+of length greater than zero, repeated invocations of this `read()` operation
+will eventually read all the data.
### <a name="Seekable.seek"></a>`Seekable.seek(s)`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a409508b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index c4975d1..9e67ff2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -972,6 +972,10 @@ public class DFSInputStream extends FSInputStream
@Override
public synchronized int read(@Nonnull final byte buf[], int off, int len)
throws IOException {
+ validatePositionedReadArgs(pos, buf, off, len);
+ if (len == 0) {
+ return 0;
+ }
ReaderStrategy byteArrayReader = new ByteArrayStrategy(buf);
try (TraceScope scope =
dfsClient.newReaderTraceScope("DFSInputStream#byteArrayRead",
[04/21] hadoop git commit: HDFS-10271. Extra bytes are getting
released from reservedSpace for append (Contributed by Brahma Reddy Battula)
Posted by ae...@apache.org.
HDFS-10271. Extra bytes are getting released from reservedSpace for append (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9a607f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9a607f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9a607f8
Branch: refs/heads/HDFS-1312
Commit: a9a607f8fc0d996af3fb37f7efa7591d6655900d
Parents: a409508
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Apr 12 11:22:58 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Apr 12 11:44:34 2016 +0530
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 7 ++-
.../fsdataset/impl/TestSpaceReservation.java | 60 ++++++++++++++++++++
2 files changed, 64 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a607f8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 7e4e8eb..381c799 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1159,7 +1159,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// construct a RBW replica with the new GS
File blkfile = replicaInfo.getBlockFile();
FsVolumeImpl v = (FsVolumeImpl)replicaInfo.getVolume();
- if (v.getAvailable() < estimateBlockLen - replicaInfo.getNumBytes()) {
+ long bytesReserved = estimateBlockLen - replicaInfo.getNumBytes();
+ if (v.getAvailable() < bytesReserved) {
throw new DiskOutOfSpaceException("Insufficient space for appending to "
+ replicaInfo);
}
@@ -1167,7 +1168,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
File oldmeta = replicaInfo.getMetaFile();
ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(
replicaInfo.getBlockId(), replicaInfo.getNumBytes(), newGS,
- v, newBlkFile.getParentFile(), Thread.currentThread(), estimateBlockLen);
+ v, newBlkFile.getParentFile(), Thread.currentThread(), bytesReserved);
File newmeta = newReplicaInfo.getMetaFile();
// rename meta file to rbw directory
@@ -1203,7 +1204,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// Replace finalized replica by a RBW replica in replicas map
volumeMap.add(bpid, newReplicaInfo);
- v.reserveSpaceForReplica(estimateBlockLen - replicaInfo.getNumBytes());
+ v.reserveSpaceForReplica(bytesReserved);
return newReplicaInfo;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a607f8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
index 49e585d..6dbd299 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
@@ -572,4 +572,64 @@ public class TestSpaceReservation {
return numFailures;
}
}
+
+ @Test(timeout = 30000)
+ public void testReservedSpaceForAppend() throws Exception {
+ final short replication = 3;
+ startCluster(BLOCK_SIZE, replication, -1);
+ final String methodName = GenericTestUtils.getMethodName();
+ final Path file = new Path("/" + methodName + ".01.dat");
+
+ // Write 1 byte to the file and kill the writer.
+ FSDataOutputStream os = fs.create(file, replication);
+ os.write(new byte[1024]);
+ os.close();
+
+ final Path file2 = new Path("/" + methodName + ".02.dat");
+
+ // Write 1 byte to the file and keep it open.
+ FSDataOutputStream os2 = fs.create(file2, replication);
+ os2.write(new byte[1]);
+ os2.hflush();
+ int expectedFile2Reserved = BLOCK_SIZE - 1;
+ checkReservedSpace(expectedFile2Reserved);
+
+ // append one byte and verify reservedspace before and after closing
+ os = fs.append(file);
+ os.write(new byte[1]);
+ os.hflush();
+ int expectedFile1Reserved = BLOCK_SIZE - 1025;
+ checkReservedSpace(expectedFile2Reserved + expectedFile1Reserved);
+ os.close();
+ checkReservedSpace(expectedFile2Reserved);
+
+ // append one byte and verify reservedspace before and after abort
+ os = fs.append(file);
+ os.write(new byte[1]);
+ os.hflush();
+ expectedFile1Reserved--;
+ checkReservedSpace(expectedFile2Reserved + expectedFile1Reserved);
+ DFSTestUtil.abortStream(((DFSOutputStream) os.getWrappedStream()));
+ checkReservedSpace(expectedFile2Reserved);
+ }
+
+ private void checkReservedSpace(final long expectedReserved) throws TimeoutException,
+ InterruptedException, IOException {
+ for (final DataNode dn : cluster.getDataNodes()) {
+ try (FsDatasetSpi.FsVolumeReferences volumes = dn.getFSDataset()
+ .getFsVolumeReferences()) {
+ final FsVolumeImpl volume = (FsVolumeImpl) volumes.get(0);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ LOG.info(
+ "dn " + dn.getDisplayName() + " space : " + volume
+ .getReservedForReplicas() + ", Expected ReservedSpace :"
+ + expectedReserved);
+ return (volume.getReservedForReplicas() == expectedReserved);
+ }
+ }, 100, 3000);
+ }
+ }
+ }
}
[09/21] hadoop git commit: HADOOP-12964. Http server vulnerable to
clickjacking (haibochen via rkanter)
Posted by ae...@apache.org.
HADOOP-12964. Http server vulnerable to clickjacking (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042a3ae9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042a3ae9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042a3ae9
Branch: refs/heads/HDFS-1312
Commit: 042a3ae960883c263adc76f16d0ea3438d8b12be
Parents: 6ef4287
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Apr 12 14:40:43 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Apr 12 14:40:43 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/http/HttpServer2.java | 29 ++++++++++++++++----
.../org/apache/hadoop/http/TestHttpServer.java | 10 +++++++
2 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042a3ae9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 45417f6..8ba67dd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -55,10 +55,7 @@ import org.apache.hadoop.conf.ConfServlet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
-import org.apache.hadoop.security.authentication.util.FileSignerSecretProvider;
-import org.apache.hadoop.security.authentication.util.RandomSignerSecretProvider;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
-import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider;
import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
@@ -98,8 +95,6 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.sun.jersey.spi.container.servlet.ServletContainer;
-import static org.apache.hadoop.security.authentication.server
- .AuthenticationFilter.*;
/**
* Create a Jetty embedded server to answer http requests. The primary goal is
* to serve up status information for the server. There are three contexts:
@@ -1124,9 +1119,11 @@ public final class HttpServer2 implements FilterContainer {
/**
* A Servlet input filter that quotes all HTML active characters in the
* parameter names and values. The goal is to quote the characters to make
- * all of the servlets resistant to cross-site scripting attacks.
+ * all of the servlets resistant to cross-site scripting attacks. It also
+ * sets X-FRAME-OPTIONS in the header to mitigate clickjacking attacks.
*/
public static class QuotingInputFilter implements Filter {
+ private static final XFrameOption X_FRAME_OPTION = XFrameOption.SAMEORIGIN;
private FilterConfig config;
public static class RequestQuoter extends HttpServletRequestWrapper {
@@ -1246,6 +1243,7 @@ public final class HttpServer2 implements FilterContainer {
} else if (mime.startsWith("application/xml")) {
httpResponse.setContentType("text/xml; charset=utf-8");
}
+ httpResponse.addHeader("X-FRAME-OPTIONS", X_FRAME_OPTION.toString());
chain.doFilter(quoted, httpResponse);
}
@@ -1262,4 +1260,23 @@ public final class HttpServer2 implements FilterContainer {
}
}
+
+ /**
+ * The X-FRAME-OPTIONS header in HTTP response to mitigate clickjacking
+ * attack.
+ */
+ public enum XFrameOption {
+ DENY("DENY") , SAMEORIGIN ("SAMEORIGIN"), ALLOWFROM ("ALLOW-FROM");
+
+ XFrameOption(String name) {
+ this.name = name;
+ }
+
+ private final String name;
+
+ @Override
+ public String toString() {
+ return this.name;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042a3ae9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index 4d2e1bf..3ed89a8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -235,6 +235,16 @@ public class TestHttpServer extends HttpServerFunctionalTest {
assertEquals("text/html; charset=utf-8", conn.getContentType());
}
+ @Test
+ public void testHttpResonseContainsXFrameOptions() throws IOException {
+ URL url = new URL(baseUrl, "");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.connect();
+
+ String xfoHeader = conn.getHeaderField("X-FRAME-OPTIONS");
+ assertTrue("X-FRAME-OPTIONS is absent in the header", xfoHeader != null);
+ }
+
/**
* Dummy filter that mimics as an authentication filter. Obtains user identity
* from the request parameter user.name. Wraps around the request so that
[02/21] hadoop git commit: YARN-4897. dataTables_wrapper change min
height. Contributed by Bibin A Chundatt.
Posted by ae...@apache.org.
YARN-4897. dataTables_wrapper change min height. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ee7f0a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ee7f0a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ee7f0a3
Branch: refs/heads/HDFS-1312
Commit: 4ee7f0a3e7814d66d6e11aa4b559da5ac93c4b99
Parents: ff722bb
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Apr 12 10:25:56 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Apr 12 10:25:56 2016 +0530
----------------------------------------------------------------------
.../src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ee7f0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css
index 89bd2f0..6f6f414 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css
@@ -109,8 +109,8 @@ table.display thead th div.DataTables_sort_wrapper span {
.dataTables_wrapper {
position: relative;
- min-height: 302px;
- _height: 302px;
+ min-height: 35px;
+ _height: 35px;
clear: both;
}
[11/21] hadoop git commit: HDFS-9772.
TestBlockReplacement#testThrottler doesn't work as expected. Contributed by
Lin Yiqun.
Posted by ae...@apache.org.
HDFS-9772. TestBlockReplacement#testThrottler doesn't work as expected. Contributed by Lin Yiqun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/903428bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/903428bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/903428bf
Branch: refs/heads/HDFS-1312
Commit: 903428bf946827b4d58c7c577ed0c574a7cff029
Parents: 35f0770
Author: Walter Su <wa...@apache.org>
Authored: Wed Apr 13 17:49:45 2016 +0800
Committer: Walter Su <wa...@apache.org>
Committed: Wed Apr 13 17:49:45 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/903428bf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
index 286a180..597dc46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
@@ -81,7 +81,6 @@ public class TestBlockReplacement {
long bytesToSend = TOTAL_BYTES;
long start = Time.monotonicNow();
DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
- long totalBytes = 0L;
long bytesSent = 1024*512L; // 0.5MB
throttler.throttle(bytesSent);
bytesToSend -= bytesSent;
@@ -93,7 +92,7 @@ public class TestBlockReplacement {
} catch (InterruptedException ignored) {}
throttler.throttle(bytesToSend);
long end = Time.monotonicNow();
- assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
+ assertTrue(TOTAL_BYTES * 1000 / (end - start) <= bandwidthPerSec);
}
@Test
[07/21] hadoop git commit: HDFS-10273. Remove duplicate logSync() and
log message in FSN#enterSafemode() (Vinayakumar B via cmccabe)
Posted by ae...@apache.org.
HDFS-10273. Remove duplicate logSync() and log message in FSN#enterSafemode() (Vinayakumar B via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/600d129b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/600d129b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/600d129b
Branch: refs/heads/HDFS-1312
Commit: 600d129bb8d52ae820e7b74b8ce363aabd69d25c
Parents: 437e9d6
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Apr 12 10:52:39 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Apr 12 10:52:39 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/600d129b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 681fc96..00e1241 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4279,10 +4279,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
setManualAndResourceLowSafeMode(!resourcesLow, resourcesLow);
NameNode.stateChangeLog.info("STATE* Safe mode is ON.\n" +
getSafeModeTip());
- if (isEditlogOpenForWrite) {
- getEditLog().logSyncAll();
- }
- NameNode.stateChangeLog.info("STATE* Safe mode is ON" + getSafeModeTip());
} finally {
writeUnlock();
}
[18/21] hadoop git commit: HDFS-10209. Support enable caller context
in HDFS namenode audit log without restart namenode. Contributed by Xiaobing
Zhou.
Posted by ae...@apache.org.
HDFS-10209. Support enable caller context in HDFS namenode audit log without restart namenode. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5566177c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5566177c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5566177c
Branch: refs/heads/HDFS-1312
Commit: 5566177c9af913baf380811dbbb1fa7e70235491
Parents: 4895c73
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Apr 13 16:51:13 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Apr 13 16:51:13 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystem.java | 52 +++++++++++++++++++-
.../hadoop/hdfs/server/namenode/NameNode.java | 24 ++++++++-
.../namenode/TestNameNodeReconfigure.java | 51 ++++++++++++++++++-
3 files changed, 123 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5566177c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 39b3598..471e6b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -902,6 +902,36 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return null;
}
+ /**
+ * Locate DefaultAuditLogger, if any, to enable/disable CallerContext.
+ *
+ * @param value
+ * true, enable CallerContext, otherwise false to disable it.
+ */
+ void setCallerContextEnabled(final boolean value) {
+ for (AuditLogger logger : auditLoggers) {
+ if (logger instanceof DefaultAuditLogger) {
+ ((DefaultAuditLogger) logger).setCallerContextEnabled(value);
+ break;
+ }
+ }
+ }
+
+ /**
+ * Get the value indicating if CallerContext is enabled.
+ *
+ * @return true, if CallerContext is enabled, otherwise false, if it's
+ * disabled.
+ */
+ boolean getCallerContextEnabled() {
+ for (AuditLogger logger : auditLoggers) {
+ if (logger instanceof DefaultAuditLogger) {
+ return ((DefaultAuditLogger) logger).getCallerContextEnabled();
+ }
+ }
+ return false;
+ }
+
private List<AuditLogger> initAuditLoggers(Configuration conf) {
// Initialize the custom access loggers if configured.
Collection<String> alClasses =
@@ -6848,13 +6878,33 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
};
- private boolean isCallerContextEnabled;
+ private volatile boolean isCallerContextEnabled;
private int callerContextMaxLen;
private int callerSignatureMaxLen;
private boolean logTokenTrackingId;
private Set<String> debugCmdSet = new HashSet<String>();
+ /**
+ * Enable or disable CallerContext.
+ *
+ * @param value
+ * true, enable CallerContext, otherwise false to disable it.
+ */
+ void setCallerContextEnabled(final boolean value) {
+ isCallerContextEnabled = value;
+ }
+
+ /**
+ * Get the value indicating if CallerContext is enabled.
+ *
+ * @return true, if CallerContext is enabled, otherwise false, if it's
+ * disabled.
+ */
+ boolean getCallerContextEnabled() {
+ return isCallerContextEnabled;
+ }
+
@Override
public void initialize(Configuration conf) {
isCallerContextEnabled = conf.getBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5566177c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 1301940..b39c842 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -107,6 +107,8 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY;
@@ -277,7 +279,8 @@ public class NameNode extends ReconfigurableBase implements
.unmodifiableList(Arrays
.asList(DFS_HEARTBEAT_INTERVAL_KEY,
DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
- FS_PROTECTED_DIRECTORIES));
+ FS_PROTECTED_DIRECTORIES,
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY));
private static final String USAGE = "Usage: hdfs namenode ["
+ StartupOption.BACKUP.getName() + "] | \n\t["
@@ -2008,7 +2011,9 @@ public class NameNode extends ReconfigurableBase implements
+ datanodeManager.getHeartbeatRecheckInterval());
}
case FS_PROTECTED_DIRECTORIES:
- return getNamesystem().getFSDirectory().setProtectedDirectories(newVal);
+ return reconfProtectedDirectories(newVal);
+ case HADOOP_CALLER_CONTEXT_ENABLED_KEY:
+ return reconfCallerContextEnabled(newVal);
default:
break;
}
@@ -2016,6 +2021,21 @@ public class NameNode extends ReconfigurableBase implements
.get(property));
}
+ private String reconfProtectedDirectories(String newVal) {
+ return getNamesystem().getFSDirectory().setProtectedDirectories(newVal);
+ }
+
+ private String reconfCallerContextEnabled(String newVal) {
+ Boolean callerContextEnabled;
+ if (newVal == null) {
+ callerContextEnabled = HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
+ } else {
+ callerContextEnabled = Boolean.parseBoolean(newVal);
+ }
+ namesystem.setCallerContextEnabled(callerContextEnabled);
+ return Boolean.toString(callerContextEnabled);
+ }
+
@Override // ReconfigurableBase
protected Configuration getNewConf() {
return new HdfsConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5566177c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
index abdb1ea..4fd7af6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
@@ -50,13 +52,60 @@ public class TestNameNodeReconfigure {
public void setUp() throws IOException {
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ }
+
+ @Test
+ public void testReconfigureCallerContextEnabled()
+ throws ReconfigurationException {
+ final NameNode nameNode = cluster.getNameNode();
+ final FSNamesystem nameSystem = nameNode.getNamesystem();
+
+ // try invalid values
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "text");
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
+ false,
+ nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
+ HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
+
+ // enable CallerContext
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "true");
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", true,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
+ true,
+ nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
+ HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
+
+ // disable CallerContext
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, "false");
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(
+ HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value",
+ false,
+ nameNode.getConf().getBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY,
+ HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT));
+
+ // revert to default
+ nameNode.reconfigureProperty(HADOOP_CALLER_CONTEXT_ENABLED_KEY, null);
+
+ // verify default
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", false,
+ nameSystem.getCallerContextEnabled());
+ assertEquals(HADOOP_CALLER_CONTEXT_ENABLED_KEY + " has wrong value", null,
+ nameNode.getConf().get(HADOOP_CALLER_CONTEXT_ENABLED_KEY));
}
/**
* Test that we can modify configuration properties.
*/
@Test
- public void testReconfigure() throws ReconfigurationException {
+ public void testReconfigureHearbeatCheck1() throws ReconfigurationException {
final NameNode nameNode = cluster.getNameNode();
final DatanodeManager datanodeManager = nameNode.namesystem
.getBlockManager().getDatanodeManager();