You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yq...@apache.org on 2018/01/06 06:32:12 UTC
hadoop git commit: HDFS-11848. Enhance dfsadmin listOpenFiles command
to list files under a given path. Contributed by Yiqun Lin.
Repository: hadoop
Updated Branches:
refs/heads/trunk 836e3c45e -> bf5c94899
HDFS-11848. Enhance dfsadmin listOpenFiles command to list files under a given path. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf5c9489
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf5c9489
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf5c9489
Branch: refs/heads/trunk
Commit: bf5c94899537011465350d5d999fad9ffaeb605d
Parents: 836e3c4
Author: Yiqun Lin <yq...@apache.org>
Authored: Sat Jan 6 14:31:08 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Sat Jan 6 14:31:08 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 36 +++++++++-
.../hadoop/hdfs/DistributedFileSystem.java | 4 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 4 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 5 +-
.../hadoop/hdfs/protocol/OpenFilesIterator.java | 10 ++-
.../ClientNamenodeProtocolTranslatorPB.java | 8 ++-
.../src/main/proto/ClientNamenodeProtocol.proto | 1 +
...tNamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../federation/router/RouterRpcServer.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 27 +++++---
.../hdfs/server/namenode/LeaseManager.java | 26 ++++++--
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 ++-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 23 +++++--
.../src/site/markdown/HDFSCommands.md | 4 +-
.../apache/hadoop/hdfs/TestDecommission.java | 38 ++++++++++-
.../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 4 +-
.../hdfs/server/namenode/TestLeaseManager.java | 8 ++-
.../hdfs/server/namenode/TestListOpenFiles.java | 20 ++++--
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 69 +++++++++++++++++++-
19 files changed, 249 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index c774132..f0769c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3088,11 +3088,26 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
@Deprecated
public RemoteIterator<OpenFileEntry> listOpenFiles() throws IOException {
checkOpen();
- return listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES));
+ return listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
}
/**
- * Get a remote iterator to the open files list by type, managed by NameNode.
+ * Get a remote iterator to the open files list by path,
+ * managed by NameNode.
+ *
+ * @param path
+ * @throws IOException
+ */
+ public RemoteIterator<OpenFileEntry> listOpenFiles(String path)
+ throws IOException {
+ checkOpen();
+ return listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES), path);
+ }
+
+ /**
+ * Get a remote iterator to the open files list by type,
+ * managed by NameNode.
*
* @param openFilesTypes
* @throws IOException
@@ -3100,6 +3115,21 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
public RemoteIterator<OpenFileEntry> listOpenFiles(
EnumSet<OpenFilesType> openFilesTypes) throws IOException {
checkOpen();
- return new OpenFilesIterator(namenode, tracer, openFilesTypes);
+ return listOpenFiles(openFilesTypes,
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
+ }
+
+ /**
+ * Get a remote iterator to the open files list by type and path,
+ * managed by NameNode.
+ *
+ * @param openFilesTypes
+ * @param path
+ * @throws IOException
+ */
+ public RemoteIterator<OpenFileEntry> listOpenFiles(
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
+ checkOpen();
+ return new OpenFilesIterator(namenode, tracer, openFilesTypes, path);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 85e5964..3883f2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -3086,8 +3086,8 @@ public class DistributedFileSystem extends FileSystem
}
public RemoteIterator<OpenFileEntry> listOpenFiles(
- EnumSet<OpenFilesType> openFilesTypes) throws IOException {
- return dfs.listOpenFiles(openFilesTypes);
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
+ return dfs.listOpenFiles(openFilesTypes, path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index e620039..2c0659a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -659,8 +659,8 @@ public class HdfsAdmin {
}
public RemoteIterator<OpenFileEntry> listOpenFiles(
- EnumSet<OpenFilesType> openFilesTypes) throws IOException {
- return dfs.listOpenFiles(openFilesTypes);
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
+ return dfs.listOpenFiles(openFilesTypes, path);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 38c242a..fbef037 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1724,10 +1724,11 @@ public interface ClientProtocol {
* the the list across batches are not atomic.
*
* @param prevId the cursor INode id.
- * @param openFilesTypes types to filter the open files
+ * @param openFilesTypes types to filter the open files.
+ * @param path path to filter the open files.
* @throws IOException
*/
@Idempotent
BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
- EnumSet<OpenFilesType> openFilesTypes) throws IOException;
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java
index d113d65..c2b3781 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java
@@ -37,6 +37,9 @@ import org.apache.htrace.core.Tracer;
public class OpenFilesIterator extends
BatchedRemoteIterator<Long, OpenFileEntry> {
+ /** No path to be filtered by default. */
+ public static final String FILTER_PATH_DEFAULT = "/";
+
/**
* Open file types to filter the results.
*/
@@ -67,20 +70,23 @@ public class OpenFilesIterator extends
private final ClientProtocol namenode;
private final Tracer tracer;
private final EnumSet<OpenFilesType> types;
+ /** List files filtered by given path. */
+ private String path;
public OpenFilesIterator(ClientProtocol namenode, Tracer tracer,
- EnumSet<OpenFilesType> types) {
+ EnumSet<OpenFilesType> types, String path) {
super(HdfsConstants.GRANDFATHER_INODE_ID);
this.namenode = namenode;
this.tracer = tracer;
this.types = types;
+ this.path = path;
}
@Override
public BatchedEntries<OpenFileEntry> makeRequest(Long prevId)
throws IOException {
try (TraceScope ignored = tracer.newScope("listOpenFiles")) {
- return namenode.listOpenFiles(prevId, types);
+ return namenode.listOpenFiles(prevId, types, path);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index ea5c951..4a22da9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@@ -1898,17 +1899,20 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId)
throws IOException {
- return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES));
+ return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
}
@Override
public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
- EnumSet<OpenFilesType> openFilesTypes) throws IOException {
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
ListOpenFilesRequestProto.Builder req =
ListOpenFilesRequestProto.newBuilder().setId(prevId);
if (openFilesTypes != null) {
req.addAllTypes(PBHelperClient.convertOpenFileTypes(openFilesTypes));
}
+ req.setPath(path);
+
try {
ListOpenFilesResponseProto response =
rpcProxy.listOpenFiles(null, req.build());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index f247da8..db31e22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -804,6 +804,7 @@ enum OpenFilesTypeProto {
message ListOpenFilesRequestProto {
required int64 id = 1;
repeated OpenFilesTypeProto types = 2;
+ optional string path = 3;
}
message OpenFilesBatchResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index a9d2d1e..d68669f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -1856,7 +1856,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
EnumSet<OpenFilesType> openFilesTypes =
PBHelperClient.convertOpenFileTypes(req.getTypesList());
BatchedEntries<OpenFileEntry> entries = server.listOpenFiles(req.getId(),
- openFilesTypes);
+ openFilesTypes, req.getPath());
ListOpenFilesResponseProto.Builder builder =
ListOpenFilesResponseProto.newBuilder();
builder.setHasMore(entries.hasMore());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 537eaf4..11f7fa6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@@ -1940,12 +1941,13 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
@Override
public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId)
throws IOException {
- return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES));
+ return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
}
@Override
public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
- EnumSet<OpenFilesType> openFilesTypes) throws IOException {
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
checkOperation(OperationCategory.READ, false);
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 54decc8..ece649d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1767,11 +1767,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* TODO: HDFS-12969 - to report open files by type.
*
* @param prevId the cursor INode id.
- * @param openFilesTypes
+ * @param openFilesTypes types to filter the open files.
+ * @param path path to filter the open files.
* @throws IOException
*/
BatchedListEntries<OpenFileEntry> listOpenFiles(long prevId,
- EnumSet<OpenFilesType> openFilesTypes) throws IOException {
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
final String operationName = "listOpenFiles";
checkSuperuserPrivilege();
checkOperation(OperationCategory.READ);
@@ -1780,10 +1781,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
try {
checkOperation(OperationCategory.READ);
if(openFilesTypes.contains(OpenFilesType.ALL_OPEN_FILES)) {
- batchedListEntries = leaseManager.getUnderConstructionFiles(prevId);
+ batchedListEntries = leaseManager.getUnderConstructionFiles(prevId,
+ path);
} else {
if(openFilesTypes.contains(OpenFilesType.BLOCKING_DECOMMISSION)) {
- batchedListEntries = getFilesBlockingDecom(prevId);
+ batchedListEntries = getFilesBlockingDecom(prevId, path);
} else {
throw new IllegalArgumentException("Unknown OpenFileType: "
+ openFilesTypes);
@@ -1799,7 +1801,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return batchedListEntries;
}
- public BatchedListEntries<OpenFileEntry> getFilesBlockingDecom(long prevId) {
+ public BatchedListEntries<OpenFileEntry> getFilesBlockingDecom(long prevId,
+ String path) {
assert hasReadLock();
final List<OpenFileEntry> openFileEntries = Lists.newArrayList();
LightWeightHashSet<Long> openFileIds = new LightWeightHashSet<>();
@@ -1817,10 +1820,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
Preconditions.checkState(ucFile instanceof INodeFile);
openFileIds.add(ucFileId);
INodeFile inodeFile = ucFile.asFile();
- openFileEntries.add(new OpenFileEntry(
- inodeFile.getId(), inodeFile.getFullPathName(),
- inodeFile.getFileUnderConstructionFeature().getClientName(),
- inodeFile.getFileUnderConstructionFeature().getClientMachine()));
+
+ String fullPathName = inodeFile.getFullPathName();
+ if (org.apache.commons.lang.StringUtils.isEmpty(path)
+ || fullPathName.startsWith(path)) {
+ openFileEntries.add(new OpenFileEntry(inodeFile.getId(),
+ inodeFile.getFullPathName(),
+ inodeFile.getFileUnderConstructionFeature().getClientName(),
+ inodeFile.getFileUnderConstructionFeature().getClientMachine()));
+ }
+
if (openFileIds.size() >= this.maxListOpenFilesResponses) {
return new BatchedListEntries<>(openFileEntries, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 3746e13..1e7a174 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -37,10 +37,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.google.common.collect.Lists;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.util.Daemon;
@@ -258,6 +261,12 @@ public class LeaseManager {
return iipSet;
}
+ public BatchedListEntries<OpenFileEntry> getUnderConstructionFiles(
+ final long prevId) throws IOException {
+ return getUnderConstructionFiles(prevId,
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
+ }
+
/**
* Get a batch of under construction files from the currently active leases.
* File INodeID is the cursor used to fetch new batch of results and the
@@ -270,7 +279,7 @@ public class LeaseManager {
* @throws IOException
*/
public BatchedListEntries<OpenFileEntry> getUnderConstructionFiles(
- final long prevId) throws IOException {
+ final long prevId, final String path) throws IOException {
assert fsnamesystem.hasReadLock();
SortedMap<Long, Lease> remainingLeases;
synchronized (this) {
@@ -283,6 +292,7 @@ public class LeaseManager {
Lists.newArrayListWithExpectedSize(numResponses);
int count = 0;
+ String fullPathName = null;
for (Long inodeId: inodeIds) {
final INodeFile inodeFile =
fsnamesystem.getFSDirectory().getInode(inodeId).asFile();
@@ -291,11 +301,15 @@ public class LeaseManager {
inodeFile.getFullPathName());
continue;
}
- openFileEntries.add(new OpenFileEntry(
- inodeFile.getId(), inodeFile.getFullPathName(),
- inodeFile.getFileUnderConstructionFeature().getClientName(),
- inodeFile.getFileUnderConstructionFeature().getClientMachine()));
- count++;
+
+ fullPathName = inodeFile.getFullPathName();
+ if (StringUtils.isEmpty(path) || fullPathName.startsWith(path)) {
+ openFileEntries.add(new OpenFileEntry(inodeFile.getId(), fullPathName,
+ inodeFile.getFileUnderConstructionFeature().getClientName(),
+ inodeFile.getFileUnderConstructionFeature().getClientMachine()));
+ count++;
+ }
+
if (count >= numResponses) {
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 80f1ba3..121d17c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -115,6 +115,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -1339,14 +1340,15 @@ public class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId)
throws IOException {
- return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES));
+ return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
}
@Override // ClientProtocol
public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
- EnumSet<OpenFilesType> openFilesTypes) throws IOException {
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
checkNNStartup();
- return namesystem.listOpenFiles(prevId, openFilesTypes);
+ return namesystem.listOpenFiles(prevId, openFilesTypes, path);
}
@Override // ClientProtocol
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 7367309..1bedd82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -464,7 +464,7 @@ public class DFSAdmin extends FsShell {
"\t[-getDatanodeInfo <datanode_host:ipc_port>]\n" +
"\t[-metasave filename]\n" +
"\t[-triggerBlockReport [-incremental] <datanode_host:ipc_port>]\n" +
- "\t[-listOpenFiles [-blockingDecommission]]\n" +
+ "\t[-listOpenFiles [-blockingDecommission] [-path <path>]]\n" +
"\t[-help [cmd]]\n";
/**
@@ -918,16 +918,29 @@ public class DFSAdmin extends FsShell {
* @param argv
*/
public int listOpenFiles(String[] argv) throws IOException {
+ String path = null;
List<OpenFilesType> types = new ArrayList<>();
if (argv != null) {
List<String> args = new ArrayList<>(Arrays.asList(argv));
if (StringUtils.popOption("-blockingDecommission", args)) {
types.add(OpenFilesType.BLOCKING_DECOMMISSION);
}
+
+ path = StringUtils.popOptionWithArgument("-path", args);
}
if (types.isEmpty()) {
types.add(OpenFilesType.ALL_OPEN_FILES);
}
+
+ if (path != null) {
+ path = path.trim();
+ if (path.length() == 0) {
+ path = OpenFilesIterator.FILTER_PATH_DEFAULT;
+ }
+ } else {
+ path = OpenFilesIterator.FILTER_PATH_DEFAULT;
+ }
+
EnumSet<OpenFilesType> openFilesTypes = EnumSet.copyOf(types);
DistributedFileSystem dfs = getDFS();
@@ -941,9 +954,9 @@ public class DFSAdmin extends FsShell {
dfsConf, HAUtil.getAddressOfActive(getDFS()), ClientProtocol.class,
UserGroupInformation.getCurrentUser(), false);
openFilesRemoteIterator = new OpenFilesIterator(proxy.getProxy(),
- FsTracer.get(dfsConf), openFilesTypes);
+ FsTracer.get(dfsConf), openFilesTypes, path);
} else {
- openFilesRemoteIterator = dfs.listOpenFiles(openFilesTypes);
+ openFilesRemoteIterator = dfs.listOpenFiles(openFilesTypes, path);
}
printOpenFiles(openFilesRemoteIterator);
return 0;
@@ -1982,7 +1995,7 @@ public class DFSAdmin extends FsShell {
+ " [-triggerBlockReport [-incremental] <datanode_host:ipc_port>]");
} else if ("-listOpenFiles".equals(cmd)) {
System.err.println("Usage: hdfs dfsadmin"
- + " [-listOpenFiles [-blockingDecommission]]");
+ + " [-listOpenFiles [-blockingDecommission] [-path <path>]]");
} else {
System.err.println("Usage: hdfs dfsadmin");
System.err.println("Note: Administrative commands can only be run as the HDFS superuser.");
@@ -2137,7 +2150,7 @@ public class DFSAdmin extends FsShell {
return exitCode;
}
} else if ("-listOpenFiles".equals(cmd)) {
- if ((argv.length != 1) && (argv.length != 2)) {
+ if ((argv.length > 4)) {
printUsage(cmd);
return exitCode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index a13116f..58d9547 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -372,7 +372,7 @@ Usage:
hdfs dfsadmin [-getDatanodeInfo <datanode_host:ipc_port>]
hdfs dfsadmin [-metasave filename]
hdfs dfsadmin [-triggerBlockReport [-incremental] <datanode_host:ipc_port>]
- hdfs dfsadmin [-listOpenFiles]
+ hdfs dfsadmin [-listOpenFiles [-blockingDecommission] [-path <path>]]
hdfs dfsadmin [-help [cmd]]
| COMMAND\_OPTION | Description |
@@ -409,7 +409,7 @@ Usage:
| `-getDatanodeInfo` \<datanode\_host:ipc\_port\> | Get the information about the given datanode. See [Rolling Upgrade document](./HdfsRollingUpgrade.html#dfsadmin_-getDatanodeInfo) for the detail. |
| `-metasave` filename | Save Namenode's primary data structures to *filename* in the directory specified by hadoop.log.dir property. *filename* is overwritten if it exists. *filename* will contain one line for each of the following<br/>1. Datanodes heart beating with Namenode<br/>2. Blocks waiting to be replicated<br/>3. Blocks currently being replicated<br/>4. Blocks waiting to be deleted |
| `-triggerBlockReport` `[-incremental]` \<datanode\_host:ipc\_port\> | Trigger a block report for the given datanode. If 'incremental' is specified, it will be otherwise, it will be a full block report. |
-| `-listOpenFiles` `[-blockingDecommission]` | List all open files currently managed by the NameNode along with client name and client machine accessing them. |
+| `-listOpenFiles` `[-blockingDecommission]` `[-path <path>]` | List all open files currently managed by the NameNode along with client name and client machine accessing them. Open files list will be filtered by given type and path. |
| `-help` [cmd] | Displays help for the given command or all commands if none is specified. |
Runs a HDFS dfsadmin client.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index d82025c..c0a595b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -710,13 +710,49 @@ public class TestDecommission extends AdminStatesBaseTest {
@Override
public Boolean get() {
try {
+ boolean result1 = false;
+ boolean result2 = false;
toolOut.reset();
assertEquals(0, ToolRunner.run(dfsAdmin,
new String[]{"-listOpenFiles", "-blockingDecommission"}));
toolOut.flush();
- return verifyOpenFilesListing(
+ result1 = verifyOpenFilesListing(
"dfsadmin -listOpenFiles -blockingDecommission",
closedFileSet, openFilesMap, toolOut, maxOpenFiles);
+
+ // test -blockingDecommission with option -path
+ if (openFilesMap.size() > 0) {
+ String firstOpenFile = null;
+ // Construct a new open-file and close-file map.
+ // Pick the first open file into new open-file map, remaining
+ // open files move into close-files map.
+ HashMap<Path, FSDataOutputStream> newOpenFilesMap =
+ new HashMap<>();
+ HashSet<Path> newClosedFileSet = new HashSet<>();
+ for (Map.Entry<Path, FSDataOutputStream> entry : openFilesMap
+ .entrySet()) {
+ if (firstOpenFile == null) {
+ newOpenFilesMap.put(entry.getKey(), entry.getValue());
+ firstOpenFile = entry.getKey().toString();
+ } else {
+ newClosedFileSet.add(entry.getKey());
+ }
+ }
+
+ toolOut.reset();
+ assertEquals(0,
+ ToolRunner.run(dfsAdmin, new String[] {"-listOpenFiles",
+ "-blockingDecommission", "-path", firstOpenFile}));
+ toolOut.flush();
+ result2 = verifyOpenFilesListing(
+ "dfsadmin -listOpenFiles -blockingDecommission -path"
+ + firstOpenFile,
+ newClosedFileSet, newOpenFilesMap, toolOut, 1);
+ } else {
+ result2 = true;
+ }
+
+ return result1 && result2;
} catch (Exception e) {
LOG.warn("Unexpected exception: " + e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
index 3cb10bf..cc32a3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.junit.After;
@@ -256,7 +257,8 @@ public class TestHdfsAdmin {
HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
HashSet<Path> openFiles = new HashSet<>(openFileMap.keySet());
RemoteIterator<OpenFileEntry> openFilesRemoteItr =
- hdfsAdmin.listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES));
+ hdfsAdmin.listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
while (openFilesRemoteItr.hasNext()) {
String filePath = openFilesRemoteItr.next().getFilePath();
assertFalse(filePath + " should not be listed under open files!",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index 0a8da4b..ccd908b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
@@ -405,8 +406,11 @@ public class TestLeaseManager {
leaseManager.getINodeWithLeases(ancestorDirectory).size());
assertEquals(iNodeIdWithLeaseCount,
leaseManager.getUnderConstructionFiles(0).size());
- assertEquals(0, (fsNamesystem.getFilesBlockingDecom(0) == null ?
- 0 : fsNamesystem.getFilesBlockingDecom(0).size()));
+ assertEquals(0,
+ (fsNamesystem.getFilesBlockingDecom(0,
+ OpenFilesIterator.FILTER_PATH_DEFAULT) == null ? 0
+ : fsNamesystem.getFilesBlockingDecom(0,
+ OpenFilesIterator.FILTER_PATH_DEFAULT).size()));
}
private Map<String, INode> createINodeTree(INodeDirectory parentDir,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java
index cfee7ba..70550d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
@@ -97,11 +98,13 @@ public class TestListOpenFiles {
verifyOpenFiles(openFiles);
BatchedEntries<OpenFileEntry> openFileEntryBatchedEntries =
- nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.ALL_OPEN_FILES));
+ nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
assertTrue("Open files list should be empty!",
openFileEntryBatchedEntries.size() == 0);
BatchedEntries<OpenFileEntry> openFilesBlockingDecomEntries =
- nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION));
+ nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
assertTrue("Open files list blocking decommission should be empty!",
openFilesBlockingDecomEntries.size() == 0);
@@ -128,15 +131,16 @@ public class TestListOpenFiles {
}
private void verifyOpenFiles(Map<Path, FSDataOutputStream> openFiles,
- EnumSet<OpenFilesType> openFilesTypes) throws IOException {
+ EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
HashSet<Path> remainingFiles = new HashSet<>(openFiles.keySet());
OpenFileEntry lastEntry = null;
BatchedEntries<OpenFileEntry> batchedEntries;
do {
if (lastEntry == null) {
- batchedEntries = nnRpc.listOpenFiles(0, openFilesTypes);
+ batchedEntries = nnRpc.listOpenFiles(0, openFilesTypes, path);
} else {
- batchedEntries = nnRpc.listOpenFiles(lastEntry.getId(), openFilesTypes);
+ batchedEntries = nnRpc.listOpenFiles(lastEntry.getId(),
+ openFilesTypes, path);
}
assertTrue("Incorrect open files list size!",
batchedEntries.size() <= BATCH_SIZE);
@@ -154,9 +158,11 @@ public class TestListOpenFiles {
private void verifyOpenFiles(Map<Path, FSDataOutputStream> openFiles)
throws IOException {
- verifyOpenFiles(openFiles, EnumSet.of(OpenFilesType.ALL_OPEN_FILES));
+ verifyOpenFiles(openFiles, EnumSet.of(OpenFilesType.ALL_OPEN_FILES),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
verifyOpenFiles(new HashMap<>(),
- EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION));
+ EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION),
+ OpenFilesIterator.FILTER_PATH_DEFAULT);
}
private Set<Path> createFiles(FileSystem fileSystem, String fileNamePrefix,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf5c9489/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
index 6a01de2..7237c88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
@@ -725,6 +725,67 @@ public class TestDFSAdmin {
new String[]{"-listOpenFiles"}));
verifyOpenFilesListing(closedFileSet, openFilesMap);
}
+
+ // test -listOpenFiles command with option <path>
+ openFilesMap.clear();
+ Path file;
+ HashMap<Path, FSDataOutputStream> openFiles1 = new HashMap<>();
+ HashMap<Path, FSDataOutputStream> openFiles2 = new HashMap<>();
+ for (int i = 0; i < numOpenFiles; i++) {
+ if (i % 2 == 0) {
+ file = new Path(new Path("/tmp/files/a"), "open-file-" + i);
+ } else {
+ file = new Path(new Path("/tmp/files/b"), "open-file-" + i);
+ }
+
+ DFSTestUtil.createFile(fs, file, fileLength, replFactor, 12345L);
+ FSDataOutputStream outputStream = fs.append(file);
+
+ if (i % 2 == 0) {
+ openFiles1.put(file, outputStream);
+ } else {
+ openFiles2.put(file, outputStream);
+ }
+ openFilesMap.put(file, outputStream);
+ }
+
+ resetStream();
+ // list all open files
+ assertEquals(0,
+ ToolRunner.run(dfsAdmin, new String[] {"-listOpenFiles"}));
+ verifyOpenFilesListing(null, openFilesMap);
+
+ resetStream();
+ // list open files under directory path /tmp/files/a
+ assertEquals(0, ToolRunner.run(dfsAdmin,
+ new String[] {"-listOpenFiles", "-path", "/tmp/files/a"}));
+ verifyOpenFilesListing(null, openFiles1);
+
+ resetStream();
+ // list open files without input path
+ assertEquals(-1, ToolRunner.run(dfsAdmin,
+ new String[] {"-listOpenFiles", "-path"}));
+ // verify the error
+ String outStr = scanIntoString(err);
+ assertTrue(outStr.contains("listOpenFiles: option"
+ + " -path requires 1 argument"));
+
+ resetStream();
+ // list open files with empty path
+ assertEquals(0, ToolRunner.run(dfsAdmin,
+ new String[] {"-listOpenFiles", "-path", ""}));
+ // all the open files will be listed
+ verifyOpenFilesListing(null, openFilesMap);
+
+ resetStream();
+ // list invalid path file
+ assertEquals(0, ToolRunner.run(dfsAdmin,
+ new String[] {"-listOpenFiles", "-path", "/invalid_path"}));
+ outStr = scanIntoString(out);
+ for (Path openFilePath : openFilesMap.keySet()) {
+ assertThat(outStr, not(containsString(openFilePath.toString())));
+ }
+ DFSTestUtil.closeOpenFiles(openFilesMap, openFilesMap.size());
}
}
@@ -732,9 +793,13 @@ public class TestDFSAdmin {
HashMap<Path, FSDataOutputStream> openFilesMap) {
final String outStr = scanIntoString(out);
LOG.info("dfsadmin -listOpenFiles output: \n" + out);
- for (Path closedFilePath : closedFileSet) {
- assertThat(outStr, not(containsString(closedFilePath.toString() + "\n")));
+ if (closedFileSet != null) {
+ for (Path closedFilePath : closedFileSet) {
+ assertThat(outStr,
+ not(containsString(closedFilePath.toString() + "\n")));
+ }
}
+
for (Path openFilePath : openFilesMap.keySet()) {
assertThat(outStr, is(containsString(openFilePath.toString() + "\n")));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org