You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/06/08 12:05:00 UTC
[ozone] branch HDDS-2939 updated (0323530 -> 94fb6f3)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a change to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 0323530 HDDS-5311. [FSO] Fix checkstyle warnings after branch rebase (#2307)
add 39954ad HDDS-5111. DataNode should not always report full information in heartbeat (#2182)
add 3d3a87f HDDS-5235. Add precheck to avoid ContainerHealthTask setting Deleted containers to Missing state (#2250)
add 6e278ba HDDS-5243. Return latest key location for clients (#2261)
new 94fb6f3 Merge remote-tracking branch 'origin/master' into HDDS-2939
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../common/src/main/resources/ozone-default.xml | 10 +-
.../container/common/report/ReportPublisher.java | 8 +-
.../common/statemachine/StateContext.java | 131 ++++++++++++-------
.../states/endpoint/HeartbeatEndpointTask.java | 5 +-
.../transport/server/ratis/XceiverServerRatis.java | 3 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 2 +-
.../common/report/TestReportPublisher.java | 2 +-
.../common/statemachine/TestStateContext.java | 140 ++++++++++-----------
.../states/endpoint/TestHeartbeatEndpointTask.java | 14 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 10 ++
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 18 ++-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 17 +--
...OzoneManagerProtocolClientSideTranslatorPB.java | 4 +
.../commandhandler/TestBlockDeletion.java | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 101 +++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 3 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 59 +++++++--
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../ozone/recon/fsck/ContainerHealthStatus.java | 6 +
.../ozone/recon/fsck/ContainerHealthTask.java | 51 +++++++-
.../scm/ReconStorageContainerManagerFacade.java | 5 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 88 +++++++++++++
23 files changed, 530 insertions(+), 157 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'origin/master' into
HDDS-2939
Posted by ra...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 94fb6f33a6eb696db7016ecc9fb027df0c79aa3b
Merge: 0323530 6e278ba
Author: Rakesh Radhakrishnan <ra...@apache.org>
AuthorDate: Tue Jun 8 17:31:28 2021 +0530
Merge remote-tracking branch 'origin/master' into HDDS-2939
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../common/src/main/resources/ozone-default.xml | 10 +-
.../container/common/report/ReportPublisher.java | 8 +-
.../common/statemachine/StateContext.java | 131 ++++++++++++-------
.../states/endpoint/HeartbeatEndpointTask.java | 5 +-
.../transport/server/ratis/XceiverServerRatis.java | 3 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 2 +-
.../common/report/TestReportPublisher.java | 2 +-
.../common/statemachine/TestStateContext.java | 140 ++++++++++-----------
.../states/endpoint/TestHeartbeatEndpointTask.java | 14 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 10 ++
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 18 ++-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 17 +--
...OzoneManagerProtocolClientSideTranslatorPB.java | 4 +
.../commandhandler/TestBlockDeletion.java | 3 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 101 +++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 3 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 59 +++++++--
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
.../ozone/recon/fsck/ContainerHealthStatus.java | 6 +
.../ozone/recon/fsck/ContainerHealthTask.java | 51 +++++++-
.../scm/ReconStorageContainerManagerFacade.java | 5 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 88 +++++++++++++
23 files changed, 530 insertions(+), 157 deletions(-)
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index 2dea94e,3a9133f..81d174c
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -2864,35 -2863,10 +2863,42 @@@
</property>
<property>
+ <name>ozone.client.key.latest.version.location</name>
+ <tag>OZONE, CLIENT</tag>
+ <value>true</value>
+ <description>Ozone client gets the latest version location.
+ </description>
+ </property>
++
++ <property>
+ <name>ozone.om.metadata.layout</name>
+ <tag>OZONE, OM</tag>
+ <value>SIMPLE</value>
+ <description>
+ This property is used to define the metadata layout of file system
+ paths. If it is configured as PREFIX in combination with
+ ozone.om.enable.filesystem.paths to true then this allows to perform
+ atomic rename and delete of any directory at any level in the namespace.
+ Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX.
+ </description>
+ </property>
+ <property>
+ <name>ozone.directory.deleting.service.interval</name>
+ <value>1m</value>
+ <tag>OZONE, PERFORMANCE, OM</tag>
+ <description>Time interval of the directory deleting service. It runs on OM
+ periodically and cleanup orphan directory and its sub-tree. For every
+ orphan directory it deletes the sub-path tree structure(dirs/files). It
+ sends sub-files to KeyDeletingService to deletes its blocks. Unit could
+ be defined with postfix (ns,ms,s,m,h,d)
+ </description>
+ </property>
+ <property>
+ <name>ozone.path.deleting.limit.per.task</name>
+ <value>10000</value>
+ <tag>OZONE, PERFORMANCE, OM</tag>
+ <description>A maximum number of paths(dirs/files) to be deleted by
+ directory deleting service per time interval.
+ </description>
+ </property>
-
</configuration>
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index 9a0bf6b,868da8b..7b298d4
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@@ -46,7 -46,7 +46,8 @@@ public final class OmKeyArgs implement
private boolean refreshPipeline;
private boolean sortDatanodesInPipeline;
private List<OzoneAcl> acls;
+ private boolean latestVersionLocation;
+ private boolean recursive;
@SuppressWarnings("parameternumber")
private OmKeyArgs(String volumeName, String bucketName, String keyName,
@@@ -54,7 -54,8 +55,8 @@@
List<OmKeyLocationInfo> locationInfoList, boolean isMultipart,
String uploadID, int partNumber,
Map<String, String> metadataMap, boolean refreshPipeline,
- List<OzoneAcl> acls, boolean sortDatanode, boolean recursive) {
+ List<OzoneAcl> acls, boolean sortDatanode,
- boolean latestVersionLocation) {
++ boolean latestVersionLocation, boolean recursive) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.keyName = keyName;
@@@ -68,7 -69,7 +70,8 @@@
this.refreshPipeline = refreshPipeline;
this.acls = acls;
this.sortDatanodesInPipeline = sortDatanode;
+ this.latestVersionLocation = latestVersionLocation;
+ this.recursive = recursive;
}
public boolean getIsMultipartKey() {
@@@ -135,10 -136,10 +138,14 @@@
return sortDatanodesInPipeline;
}
+ public boolean getLatestVersionLocation() {
+ return latestVersionLocation;
+ }
+
+ public boolean isRecursive() {
+ return recursive;
+ }
+
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
@@@ -193,8 -195,8 +201,9 @@@
private Map<String, String> metadata = new HashMap<>();
private boolean refreshPipeline;
private boolean sortDatanodesInPipeline;
+ private boolean latestVersionLocation;
private List<OzoneAcl> acls;
+ private boolean recursive;
public Builder setVolumeName(String volume) {
this.volumeName = volume;
@@@ -266,17 -268,17 +275,22 @@@
return this;
}
+ public Builder setLatestVersionLocation(boolean latest) {
+ this.latestVersionLocation = latest;
+ return this;
+ }
+
+ public Builder setRecursive(boolean isRecursive) {
+ this.recursive = isRecursive;
+ return this;
+ }
+
public OmKeyArgs build() {
return new OmKeyArgs(volumeName, bucketName, keyName, dataSize,
replicationConfig, locationInfoList, isMultipartKey,
multipartUploadID,
multipartUploadPartNumber, metadata, refreshPipeline, acls,
- sortDatanodesInPipeline, recursive);
- sortDatanodesInPipeline, latestVersionLocation);
++ sortDatanodesInPipeline, latestVersionLocation, recursive);
}
}
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 295d2ad,846ed9a..650e9bb
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -741,9 -735,7 +741,10 @@@ message KeyArgs
// This will be set by leader OM in HA and update the original request.
optional FileEncryptionInfoProto fileEncryptionInfo = 15;
+ optional bool latestVersionLocation = 16;
+
+ // This will be set when user performs delete directory recursively.
- optional bool recursive = 16;
++ optional bool recursive = 17;
}
message KeyLocation {
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index ac65aab,6c28ab0..bcb0ccf
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@@ -673,9 -665,13 +673,13 @@@ public class KeyManagerImpl implements
LOG.debug("volume:{} bucket:{} Key:{} not found", volumeName,
bucketName, keyName);
}
- throw new OMException("Key not found", KEY_NOT_FOUND);
+ throw new OMException("Key:" + keyName + " not found", KEY_NOT_FOUND);
}
+ if (args.getLatestVersionLocation()) {
+ slimLocationVersion(value);
+ }
+
// add block token for read.
addBlockToken4Read(value);
@@@ -1901,12 -1765,9 +1910,14 @@@
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
+ if (OzoneManagerRatisUtils.isBucketFSOptimized()) {
+ return getOzoneFileStatusFSO(volumeName, bucketName, keyName,
- args.getSortDatanodes(), clientAddress, false);
++ args.getSortDatanodes(), clientAddress,
++ args.getLatestVersionLocation(), false);
+ }
return getOzoneFileStatus(volumeName, bucketName, keyName,
- args.getRefreshPipeline(), args.getSortDatanodes(), clientAddress);
+ args.getRefreshPipeline(), args.getSortDatanodes(),
+ args.getLatestVersionLocation(), clientAddress);
}
private OzoneFileStatus getOzoneFileStatus(String volumeName,
@@@ -1969,65 -1834,6 +1984,67 @@@
FILE_NOT_FOUND);
}
+
+ private OzoneFileStatus getOzoneFileStatusFSO(String volumeName,
+ String bucketName, String keyName, boolean sortDatanodes,
- String clientAddress, boolean skipFileNotFoundError) throws IOException {
++ String clientAddress, boolean latestLocationVersion,
++ boolean skipFileNotFoundError) throws IOException {
+ OzoneFileStatus fileStatus = null;
+ metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ try {
+ // Check if this is the root of the filesystem.
+ if (keyName.length() == 0) {
+ OMFileRequest.validateBucket(metadataManager, volumeName, bucketName);
+ return new OzoneFileStatus();
+ }
+
+ fileStatus = OMFileRequest.getOMKeyInfoIfExists(metadataManager,
+ volumeName, bucketName, keyName, scmBlockSize);
+
+ } finally {
+ metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ }
+
+ if (fileStatus != null) {
+ // if the key is a file then do refresh pipeline info in OM by asking SCM
+ if (fileStatus.isFile()) {
-
+ OmKeyInfo fileKeyInfo = fileStatus.getKeyInfo();
-
++ if (latestLocationVersion) {
++ slimLocationVersion(fileKeyInfo);
++ }
+ // refreshPipeline flag check has been removed as part of
+ // https://issues.apache.org/jira/browse/HDDS-3658.
+ // Please refer this jira for more details.
+ refresh(fileKeyInfo);
+
+ if (sortDatanodes) {
+ sortDatanodes(clientAddress, fileKeyInfo);
+ }
+ return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false);
+ } else {
+ return fileStatus;
+ }
+ }
+
+ // Key not found.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to get file status for the key: volume: {}, bucket:" +
+ " {}, key: {}, with error: No such file exists.",
+ volumeName, bucketName, keyName);
+ }
+
+ // don't throw exception if this flag is true.
+ if (skipFileNotFoundError) {
+ return fileStatus;
+ }
+
+ throw new OMException("Unable to get file status: volume: " +
+ volumeName + " bucket: " + bucketName + " key: " + keyName,
+ FILE_NOT_FOUND);
+ }
+
/**
* Ozone FS api to create a directory. Parent directories if do not exist
* are created for the input directory.
@@@ -2169,16 -1975,10 +2186,17 @@@
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
- OzoneFileStatus fileStatus = getOzoneFileStatus(volumeName, bucketName,
- keyName, args.getRefreshPipeline(), args.getSortDatanodes(),
- args.getLatestVersionLocation(), clientAddress);
- //if key is not of type file or if key is not found we throw an exception
+ OzoneFileStatus fileStatus;
+ if (OzoneManagerRatisUtils.isBucketFSOptimized()) {
+ fileStatus = getOzoneFileStatusFSO(volumeName, bucketName, keyName,
- args.getSortDatanodes(), clientAddress, false);
++ args.getSortDatanodes(), clientAddress,
++ args.getLatestVersionLocation(),false);
+ } else {
+ fileStatus = getOzoneFileStatus(volumeName, bucketName,
+ keyName, args.getRefreshPipeline(), args.getSortDatanodes(),
- clientAddress);
++ args.getLatestVersionLocation(), clientAddress);
+ }
+ //if key is not of type file or if key is not found we throw an exception
if (fileStatus.isFile()) {
// add block token for read.
addBlockToken4Read(fileStatus.getKeyInfo());
@@@ -2418,389 -2216,6 +2439,392 @@@
return fileStatusList;
}
+ @SuppressWarnings("methodlength")
+ public List<OzoneFileStatus> listStatusFSO(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries, String clientAddress)
+ throws IOException {
+ Preconditions.checkNotNull(args, "Key args can not be null");
+
+ // unsorted OMKeyInfo list contains combine results from TableCache and DB.
+ List<OzoneFileStatus> fileStatusFinalList = new ArrayList<>();
+
+ if (numEntries <= 0) {
+ return fileStatusFinalList;
+ }
+
+ /**
+ * A map sorted by OmKey to combine results from TableCache and DB for
+ * each entity - Dir & File.
+ *
+ * Two separate maps are required because the order of seek -> (1)Seek
+ * files in fileTable (2)Seek dirs in dirTable.
+ *
+ * StartKey should be added to the final listStatuses, so if we combine
+ * files and dirs into a single map then directory with lower precedence
+ * will appear at the top of the list even if the startKey is given as
+ * fileName.
+ *
+ * For example, startKey="a/file1". As per the seek order, first fetches
+ * all the files and then it will start seeking all the directories.
+ * Assume a directory name exists "a/b". With one map, the sorted list will
+ * be ["a/b", "a/file1"]. But the expected list is: ["a/file1", "a/b"],
+ * startKey element should always be at the top of the listStatuses.
+ */
+ TreeMap<String, OzoneFileStatus> cacheFileMap = new TreeMap<>();
+ TreeMap<String, OzoneFileStatus> cacheDirMap = new TreeMap<>();
+
+ String volumeName = args.getVolumeName();
+ String bucketName = args.getBucketName();
+ String keyName = args.getKeyName();
+ String seekFileInDB;
+ String seekDirInDB;
+ long prefixKeyInDB;
+ String prefixPath = keyName;
+ int countEntries = 0;
+
+ // TODO: recursive flag=true will be handled in HDDS-4360 jira.
+ metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ try {
+ if (Strings.isNullOrEmpty(startKey)) {
+ OzoneFileStatus fileStatus = getFileStatus(args, clientAddress);
+ if (fileStatus.isFile()) {
+ return Collections.singletonList(fileStatus);
+ }
+
+ // Not required to search in DeletedTable because all the deleted
+ // keys will be marked directly in dirTable or in keyTable by
+ // breaking the pointer to its sub-dirs and sub-files. So, there is no
+ // issue of inconsistency.
+
+ /*
+ * keyName is a directory.
+ * Say, "/a" is the dir name and its objectID is 1024, then seek
+ * will be doing with "1024/" to get all immediate descendants.
+ */
+ if (fileStatus.getKeyInfo() != null) {
+ prefixKeyInDB = fileStatus.getKeyInfo().getObjectID();
+ } else {
+ // list root directory.
+ String bucketKey = metadataManager.getBucketKey(volumeName,
+ bucketName);
+ OmBucketInfo omBucketInfo =
+ metadataManager.getBucketTable().get(bucketKey);
+ prefixKeyInDB = omBucketInfo.getObjectID();
+ }
+ seekFileInDB = metadataManager.getOzonePathKey(prefixKeyInDB, "");
+ seekDirInDB = metadataManager.getOzonePathKey(prefixKeyInDB, "");
+
+ // Order of seek -> (1)Seek files in fileTable (2)Seek dirs in dirTable
+ // 1. Seek the given key in key table.
+ countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB,
+ prefixPath, prefixKeyInDB, startKey, countEntries, numEntries);
+ // 2. Seek the given key in dir table.
+ getDirectories(cacheDirMap, seekDirInDB, prefixPath, prefixKeyInDB,
+ startKey, countEntries, numEntries, volumeName, bucketName,
+ recursive);
+ } else {
+ /*
+ * startKey will be used in iterator seek and sets the beginning point
+ * for key traversal.
+ * keyName will be used as parentID where the user has requested to
+ * list the keys from.
+ *
+ * When recursive flag=false, parentID won't change between two pages.
+ * For example: OM has a namespace like,
+ * /a/1...1M files and /a/b/1...1M files.
+ * /a/1...1M directories and /a/b/1...1M directories.
+ * Listing "/a", will always have the parentID as "a" irrespective of
+ * the startKey value.
+ */
+
+ // Check startKey is an immediate child of keyName. For example,
+ // keyName=/a/ and expected startKey=/a/b. startKey can't be /xyz/b.
+ if (StringUtils.isNotBlank(keyName) &&
+ !OzoneFSUtils.isImmediateChild(keyName, startKey)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("StartKey {} is not an immediate child of keyName {}. " +
+ "Returns empty list", startKey, keyName);
+ }
+ return Collections.emptyList();
+ }
+
+ // assign startKeyPath if prefixPath is empty string.
+ if (StringUtils.isBlank(prefixPath)) {
+ prefixPath = OzoneFSUtils.getParentDir(startKey);
+ }
+
+ OzoneFileStatus fileStatusInfo = getOzoneFileStatusFSO(volumeName,
- bucketName, startKey, false, null, true);
++ bucketName, startKey, false, null,
++ args.getLatestVersionLocation(),true);
+
+ if (fileStatusInfo != null) {
+ prefixKeyInDB = fileStatusInfo.getKeyInfo().getParentObjectID();
+ if(fileStatusInfo.isDirectory()){
+ seekDirInDB = metadataManager.getOzonePathKey(prefixKeyInDB,
+ fileStatusInfo.getKeyInfo().getFileName());
+
+ // Order of seek -> (1) Seek dirs only in dirTable. In OM, always
+ // the order of search is, first seek into fileTable and then
+ // dirTable. So, its not required to search again in the fileTable.
+
+ // Seek the given key in dirTable.
+ getDirectories(cacheDirMap, seekDirInDB, prefixPath,
+ prefixKeyInDB, startKey, countEntries, numEntries,
+ volumeName, bucketName, recursive);
+ } else {
+ seekFileInDB = metadataManager.getOzonePathKey(prefixKeyInDB,
+ fileStatusInfo.getKeyInfo().getFileName());
+ // begins from the first sub-dir under the parent dir
+ seekDirInDB = metadataManager.getOzonePathKey(prefixKeyInDB, "");
+
+ // 1. Seek the given key in key table.
+ countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB,
+ prefixPath, prefixKeyInDB, startKey, countEntries,
+ numEntries);
+ // 2. Seek the given key in dir table.
+ getDirectories(cacheDirMap, seekDirInDB, prefixPath,
+ prefixKeyInDB, startKey, countEntries, numEntries,
+ volumeName, bucketName, recursive);
+ }
+ } else {
+ // TODO: HDDS-4364: startKey can be a non-existed key
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("StartKey {} is a non-existed key and returning empty " +
+ "list", startKey);
+ }
+ return Collections.emptyList();
+ }
+ }
+ } finally {
+ metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ }
+
+ List<OmKeyInfo> keyInfoList = new ArrayList<>();
+ for (OzoneFileStatus fileStatus : cacheFileMap.values()) {
+ fileStatusFinalList.add(fileStatus);
+ keyInfoList.add(fileStatus.getKeyInfo());
+ }
+ for (OzoneFileStatus fileStatus : cacheDirMap.values()) {
+ fileStatusFinalList.add(fileStatus);
+ }
-
++ if (args.getLatestVersionLocation()) {
++ slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0]));
++ }
+ // refreshPipeline flag check has been removed as part of
+ // https://issues.apache.org/jira/browse/HDDS-3658.
+ // Please refer this jira for more details.
+ refreshPipeline(keyInfoList);
+ if (args.getSortDatanodes()) {
+ sortDatanodes(clientAddress, keyInfoList.toArray(new OmKeyInfo[0]));
+ }
+ return fileStatusFinalList;
+ }
+
+ @SuppressWarnings("parameternumber")
+ protected int getDirectories(
+ TreeMap<String, OzoneFileStatus> cacheKeyMap,
+ String seekDirInDB, String prefixPath, long prefixKeyInDB,
+ String startKey, int countEntries, long numEntries, String volumeName,
+ String bucketName, boolean recursive) throws IOException {
+
+ // A set to keep track of keys deleted in cache but not flushed to DB.
+ Set<String> deletedKeySet = new TreeSet<>();
+
+ Table dirTable = metadataManager.getDirectoryTable();
+ countEntries = listStatusFindDirsInTableCache(cacheKeyMap, dirTable,
+ prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName,
+ bucketName, countEntries, numEntries, deletedKeySet);
+ TableIterator<String, ? extends Table.KeyValue<String, OmDirectoryInfo>>
+ iterator = dirTable.iterator();
+
+ iterator.seek(seekDirInDB);
+
+ while (iterator.hasNext() && numEntries - countEntries > 0) {
+ OmDirectoryInfo dirInfo = iterator.value().getValue();
+ if (deletedKeySet.contains(dirInfo.getPath())) {
+ iterator.next(); // move to next entry in the table
+ // entry is actually deleted in cache and can exists in DB
+ continue;
+ }
+ if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(),
+ prefixKeyInDB)) {
+ break;
+ }
+
+ // TODO: recursive list will be handled in HDDS-4360 jira.
+ if (!recursive) {
+ String dirName = OMFileRequest.getAbsolutePath(prefixPath,
+ dirInfo.getName());
+ OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(volumeName,
+ bucketName, dirInfo, dirName);
+ cacheKeyMap.put(dirName, new OzoneFileStatus(omKeyInfo, scmBlockSize,
+ true));
+ countEntries++;
+ }
+ // move to next entry in the DirTable
+ iterator.next();
+ }
+
+ return countEntries;
+ }
+
+ private int getFilesFromDirectory(
+ TreeMap<String, OzoneFileStatus> cacheKeyMap,
+ String seekKeyInDB, String prefixKeyPath, long prefixKeyInDB,
+ String startKey, int countEntries, long numEntries) throws IOException {
+
+ // A set to keep track of keys deleted in cache but not flushed to DB.
+ Set<String> deletedKeySet = new TreeSet<>();
+
+ Table<String, OmKeyInfo> keyTable = metadataManager.getKeyTable();
+ countEntries = listStatusFindFilesInTableCache(cacheKeyMap, keyTable,
+ prefixKeyInDB, seekKeyInDB, prefixKeyPath, startKey,
+ countEntries, numEntries, deletedKeySet);
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ iterator = keyTable.iterator();
+ iterator.seek(seekKeyInDB);
+ while (iterator.hasNext() && numEntries - countEntries > 0) {
+ OmKeyInfo keyInfo = iterator.value().getValue();
+ if (deletedKeySet.contains(keyInfo.getPath())) {
+ iterator.next(); // move to next entry in the table
+ // entry is actually deleted in cache and can exists in DB
+ continue;
+ }
+ if (!OMFileRequest.isImmediateChild(keyInfo.getParentObjectID(),
+ prefixKeyInDB)) {
+ break;
+ }
+
+ keyInfo.setFileName(keyInfo.getKeyName());
+ String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath,
+ keyInfo.getKeyName());
+ keyInfo.setKeyName(fullKeyPath);
+ cacheKeyMap.put(fullKeyPath,
+ new OzoneFileStatus(keyInfo, scmBlockSize, false));
+ countEntries++;
+ iterator.next(); // move to next entry in the table
+ }
+ return countEntries;
+ }
+
+ /**
+ * Helper function for listStatus to find key in FileTableCache.
+ */
+ @SuppressWarnings("parameternumber")
+ private int listStatusFindFilesInTableCache(
+ TreeMap<String, OzoneFileStatus> cacheKeyMap, Table<String,
+ OmKeyInfo> keyTable, long prefixKeyInDB, String seekKeyInDB,
+ String prefixKeyPath, String startKey, int countEntries,
+ long numEntries, Set<String> deletedKeySet) {
+
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>>
+ cacheIter = keyTable.cacheIterator();
+
+ // TODO: recursive list will be handled in HDDS-4360 jira.
+ while (cacheIter.hasNext() && numEntries - countEntries > 0) {
+ Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry =
+ cacheIter.next();
+ String cacheKey = entry.getKey().getCacheKey();
+ OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue();
+ // cacheOmKeyInfo is null if an entry is deleted in cache
+ if(cacheOmKeyInfo == null){
+ deletedKeySet.add(cacheKey);
+ continue;
+ }
+
+ // make OmKeyInfo local copy to reset keyname to "fullKeyPath".
+ // In DB keyName stores only the leaf node but the list
+ // returning to the user should have full path.
+ OmKeyInfo omKeyInfo = cacheOmKeyInfo.copyObject();
+
+ omKeyInfo.setFileName(omKeyInfo.getKeyName());
+ String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath,
+ omKeyInfo.getKeyName());
+ omKeyInfo.setKeyName(fullKeyPath);
+
+ countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB,
+ seekKeyInDB, startKey, countEntries, cacheKey, omKeyInfo,
+ false);
+ }
+ return countEntries;
+ }
+
+ /**
+ * Helper function for listStatus to find key in DirTableCache.
+ */
+ @SuppressWarnings("parameternumber")
+ private int listStatusFindDirsInTableCache(
+ TreeMap<String, OzoneFileStatus> cacheKeyMap, Table<String,
+ OmDirectoryInfo> dirTable, long prefixKeyInDB, String seekKeyInDB,
+ String prefixKeyPath, String startKey, String volumeName,
+ String bucketName, int countEntries, long numEntries,
+ Set<String> deletedKeySet) {
+
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>>>
+ cacheIter = dirTable.cacheIterator();
+ // seekKeyInDB will have two type of values.
+ // 1. "1024/" -> startKey is null or empty
+ // 2. "1024/b" -> startKey exists
+ // TODO: recursive list will be handled in HDDS-4360 jira.
+ while (cacheIter.hasNext() && numEntries - countEntries > 0) {
+ Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>> entry =
+ cacheIter.next();
+ String cacheKey = entry.getKey().getCacheKey();
+ OmDirectoryInfo cacheOmDirInfo = entry.getValue().getCacheValue();
+ // cacheOmKeyInfo is null if an entry is deleted in cache
+ if(cacheOmDirInfo == null){
+ deletedKeySet.add(cacheKey);
+ continue;
+ }
+ String fullDirPath = OMFileRequest.getAbsolutePath(prefixKeyPath,
+ cacheOmDirInfo.getName());
+ OmKeyInfo cacheDirKeyInfo = OMFileRequest.getOmKeyInfo(volumeName,
+ bucketName, cacheOmDirInfo, fullDirPath);
+
+ countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB,
+ seekKeyInDB, startKey, countEntries, cacheKey, cacheDirKeyInfo,
+ true);
+ }
+ return countEntries;
+ }
+
+ @SuppressWarnings("parameternumber")
+ private int addKeyInfoToFileStatusList(
+ TreeMap<String, OzoneFileStatus> cacheKeyMap,
+ long prefixKeyInDB, String seekKeyInDB, String startKey,
+ int countEntries, String cacheKey, OmKeyInfo cacheOmKeyInfo,
+ boolean isDirectory) {
+ // seekKeyInDB will have two type of values.
+ // 1. "1024/" -> startKey is null or empty
+ // 2. "1024/b" -> startKey exists
+ if (StringUtils.isBlank(startKey)) {
+ // startKey is null or empty, then the seekKeyInDB="1024/"
+ if (cacheKey.startsWith(seekKeyInDB)) {
+ OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo,
+ scmBlockSize, isDirectory);
+ cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus);
+ countEntries++;
+ }
+ } else {
+ // startKey not empty, then the seekKeyInDB="1024/b" and
+ // seekKeyInDBWithOnlyParentID = "1024/". This is to avoid case of
+ // parentID with "102444" cache entries.
+ // Here, it has to list all the keys after "1024/b" and requires >=0
+ // string comparison.
+ String seekKeyInDBWithOnlyParentID = prefixKeyInDB + OM_KEY_PREFIX;
+ if (cacheKey.startsWith(seekKeyInDBWithOnlyParentID) &&
+ cacheKey.compareTo(seekKeyInDB) >= 0) {
+ OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo,
+ scmBlockSize, isDirectory);
+ cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus);
+ countEntries++;
+ }
+ }
+ return countEntries;
+ }
+
private String getNextGreaterString(String volumeName, String bucketName,
String keyPrefix) throws IOException {
// Increment the last character of the string and return the new ozone key.
@@@ -2953,88 -2368,22 +2977,105 @@@
}
return nodeSet;
}
-
+ private void slimLocationVersion(OmKeyInfo... keyInfos) {
+ if (keyInfos != null) {
+ for (OmKeyInfo keyInfo : keyInfos) {
+ OmKeyLocationInfoGroup key = keyInfo.getLatestVersionLocations();
+ if (key == null) {
+ LOG.warn("No location version for key {}", keyInfo);
+ continue;
+ }
+ int keyLocationVersionLength = keyInfo.getKeyLocationVersions().size();
+ if (keyLocationVersionLength <= 1) {
+ continue;
+ }
+ keyInfo.setKeyLocationVersions(keyInfo.getKeyLocationVersions()
- .subList(keyLocationVersionLength-1, keyLocationVersionLength));
++ .subList(keyLocationVersionLength - 1, keyLocationVersionLength));
+ }
+ }
+ }
+
+ @Override
+ public OmKeyInfo getPendingDeletionDir() throws IOException {
+ OmKeyInfo omKeyInfo = null;
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirItr = metadataManager.getDeletedDirTable().iterator()) {
+ if (deletedDirItr.hasNext()) {
+ Table.KeyValue<String, OmKeyInfo> keyValue = deletedDirItr.next();
+ if (keyValue != null) {
+ omKeyInfo = keyValue.getValue();
+ }
+ }
+ }
+ return omKeyInfo;
+ }
+
+ @Override
+ public List<OmKeyInfo> getPendingDeletionSubDirs(OmKeyInfo parentInfo,
+ long numEntries) throws IOException {
+ List<OmKeyInfo> directories = new ArrayList<>();
+ String seekDirInDB = metadataManager.getOzonePathKey(
+ parentInfo.getObjectID(), "");
+ long countEntries = 0;
+
+ Table dirTable = metadataManager.getDirectoryTable();
+ TableIterator<String, ? extends Table.KeyValue<String, OmDirectoryInfo>>
+ iterator = dirTable.iterator();
+
+ iterator.seek(seekDirInDB);
+
+ while (iterator.hasNext() && numEntries - countEntries > 0) {
+ OmDirectoryInfo dirInfo = iterator.value().getValue();
+ if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(),
+ parentInfo.getObjectID())) {
+ break;
+ }
+ String dirName = OMFileRequest.getAbsolutePath(parentInfo.getKeyName(),
+ dirInfo.getName());
+ OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(
+ parentInfo.getVolumeName(), parentInfo.getBucketName(), dirInfo,
+ dirName);
+ directories.add(omKeyInfo);
+ countEntries++;
+
+ // move to next entry in the DirTable
+ iterator.next();
+ }
+
+ return directories;
+ }
+
+ @Override
+ public List<OmKeyInfo> getPendingDeletionSubFiles(OmKeyInfo parentInfo,
+ long numEntries) throws IOException {
+ List<OmKeyInfo> files = new ArrayList<>();
+ String seekFileInDB = metadataManager.getOzonePathKey(
+ parentInfo.getObjectID(), "");
+ long countEntries = 0;
+
+ Table fileTable = metadataManager.getKeyTable();
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ iterator = fileTable.iterator();
+
+ iterator.seek(seekFileInDB);
+
+ while (iterator.hasNext() && numEntries - countEntries > 0) {
+ OmKeyInfo fileInfo = iterator.value().getValue();
+ if (!OMFileRequest.isImmediateChild(fileInfo.getParentObjectID(),
+ parentInfo.getObjectID())) {
+ break;
+ }
+ fileInfo.setFileName(fileInfo.getKeyName());
+ String fullKeyPath = OMFileRequest.getAbsolutePath(
+ parentInfo.getKeyName(), fileInfo.getKeyName());
+ fileInfo.setKeyName(fullKeyPath);
+
+ files.add(fileInfo);
+ countEntries++;
+ // move to next entry in the KeyTable
+ iterator.next();
+ }
+
+ return files;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org