You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ms...@apache.org on 2022/06/03 03:11:34 UTC
[ozone] branch master updated: HDDS-6810. Add a optional flag to trigger listStatus as part of listKeys for FSO buckets. (#3461)
This is an automated email from the ASF dual-hosted git repository.
msingh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e1fb4a50bb HDDS-6810. Add a optional flag to trigger listStatus as part of listKeys for FSO buckets. (#3461)
e1fb4a50bb is described below
commit e1fb4a50bb64bf3dc86ada366e3ec41a5e284c6f
Author: Mukul Kumar Singh <mk...@gmail.com>
AuthorDate: Fri Jun 3 08:41:29 2022 +0530
HDDS-6810. Add a optional flag to trigger listStatus as part of listKeys for FSO buckets. (#3461)
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 24 ++++++++-
.../ozone/client/protocol/ClientProtocol.java | 19 +++++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 26 ++++++++--
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 23 +++++++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 18 +++++++
...OzoneManagerProtocolClientSideTranslatorPB.java | 21 ++++++--
.../org/apache/hadoop/ozone/om/TestListStatus.java | 48 +++++++++++-------
.../src/main/proto/OmClientProtocol.proto | 1 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 14 +++--
.../hadoop/ozone/om/OzoneListStatusHelper.java | 59 +++++++++++++++++-----
.../org/apache/hadoop/ozone/om/OzoneManager.java | 11 +++-
.../apache/hadoop/ozone/om/fs/OzoneManagerFS.java | 21 ++++++++
.../protocolPB/OzoneManagerRequestHandler.java | 5 +-
13 files changed, 242 insertions(+), 48 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 8ee3b8a195..90f92fd770 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -937,6 +937,28 @@ public class OzoneBucket extends WithMetadata {
.listStatus(volumeName, name, keyName, recursive, startKey, numEntries);
}
+ /**
+ * List the status for a file or a directory and its contents.
+ *
+ * @param keyName Absolute path of the entry to be listed
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey exists
+ * its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param allowPartialPrefix allow partial prefixes during listStatus,
+ * this is used in context of listKeys calling
+ * listStatus
+ * @return list of file status
+ */
+ public List<OzoneFileStatus> listStatus(String keyName, boolean recursive,
+ String startKey, long numEntries, boolean allowPartialPrefix)
+ throws IOException {
+ return proxy
+ .listStatus(volumeName, name, keyName, recursive, startKey,
+ numEntries, allowPartialPrefix);
+ }
+
/**
* Return with the list of the in-flight multipart uploads.
*
@@ -1223,7 +1245,7 @@ public class OzoneBucket extends WithMetadata {
// 2. Get immediate children of keyPrefix, starting with startKey
List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
- keyPrefix, false, startKey, listCacheSize);
+ keyPrefix, false, startKey, listCacheSize, true);
// 3. Special case: ListKey expects keyPrefix element should present in
// the resultList, only if startKey is blank. If startKey is not blank
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 9fc30814ad..70a04406a0 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -815,6 +815,25 @@ public interface ClientProtocol {
throws IOException;
+ /**
+ * List the status for a file or a directory and its contents.
+ *
+ * @param volumeName Volume name
+ * @param bucketName Bucket name
+ * @param keyName Absolute path of the entry to be listed
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey exists
+ * its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param allowPartialPrefixes if partial prefixes should be allowed,
+ * this is needed in context of ListKeys
+ * @return list of file status
+ */
+ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
+ String keyName, boolean recursive, String startKey,
+ long numEntries, boolean allowPartialPrefixes) throws IOException;
+
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 187469c36a..1c59ab1e4b 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1719,11 +1719,9 @@ public class RpcClient implements ClientProtocol {
return createOutputStream(keySession, UUID.randomUUID().toString());
}
- @Override
- public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
- String keyName, boolean recursive, String startKey, long numEntries)
- throws IOException {
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ private OmKeyArgs prepareOmKeyArgs(String volumeName, String bucketName,
+ String keyName) {
+ return new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
@@ -1731,10 +1729,28 @@ public class RpcClient implements ClientProtocol {
.setSortDatanodesInPipeline(topologyAwareReadEnabled)
.setLatestVersionLocation(getLatestVersionLocation)
.build();
+ }
+
+
+ @Override
+ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
+ String keyName, boolean recursive, String startKey, long numEntries)
+ throws IOException {
+ OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
return ozoneManagerClient
.listStatus(keyArgs, recursive, startKey, numEntries);
}
+ @Override
+ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
+ String keyName, boolean recursive, String startKey,
+ long numEntries, boolean allowPartialPrefixes) throws IOException {
+ OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
+ return ozoneManagerClient
+ .listStatus(keyArgs, recursive, startKey, numEntries,
+ allowPartialPrefixes);
+ }
+
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
index 1de593441e..728bd57b4f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
@@ -150,6 +150,28 @@ public final class OzoneFSUtils {
return keyName;
}
+ /**
+ * Verifies whether the childKey is a sibling of a given
+ * parentKey.
+ *
+ * @param parentKey parent key name
+ * @param childKey child key name
+ * @return true if childKey is a sibling of parentKey
+ */
+ public static boolean isSibling(String parentKey, String childKey) {
+ // Empty childKey has no parent, so just returning false.
+ if (org.apache.commons.lang3.StringUtils.isBlank(childKey)) {
+ return false;
+ }
+ java.nio.file.Path parentPath = Paths.get(parentKey);
+ java.nio.file.Path childPath = Paths.get(childKey);
+
+ java.nio.file.Path childParent = childPath.getParent();
+ java.nio.file.Path parentParent = parentPath.getParent();
+
+ return childParent == parentParent;
+ }
+
/**
* Verifies whether the childKey is an immediate path under the given
* parentKey.
@@ -168,6 +190,7 @@ public final class OzoneFSUtils {
java.nio.file.Path childPath = Paths.get(childKey);
java.nio.file.Path childParent = childPath.getParent();
+
// Following are the valid parentKey formats:
// parentKey="" or parentKey="/" or parentKey="/a" or parentKey="a"
// Following are the valid childKey formats:
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index aa2b24f3ec..6017121706 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -748,6 +748,24 @@ public interface OzoneManagerProtocol
List<OzoneFileStatus> listStatus(OmKeyArgs keyArgs, boolean recursive,
String startKey, long numEntries) throws IOException;
+ /**
+ * List the status for a file or a directory and its contents.
+ *
+ * @param keyArgs Key args
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey exists
+ * its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param allowPartialPrefixes if partial prefixes should be allowed,
+ * this is needed in context of ListKeys
+ * @return list of file status
+ */
+ List<OzoneFileStatus> listStatus(OmKeyArgs keyArgs, boolean recursive,
+ String startKey, long numEntries,
+ boolean allowPartialPrefixes)
+ throws IOException;
+
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 4b5414c3cb..f533078c64 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -1835,7 +1835,8 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
@Override
public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
- String startKey, long numEntries) throws IOException {
+ String startKey, long numEntries, boolean allowPartialPrefixes)
+ throws IOException {
KeyArgs keyArgs = KeyArgs.newBuilder()
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
@@ -1843,15 +1844,19 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.setSortDatanodes(args.getSortDatanodes())
.setLatestVersionLocation(args.getLatestVersionLocation())
.build();
- ListStatusRequest listStatusRequest =
+ ListStatusRequest.Builder listStatusRequestBuilder =
ListStatusRequest.newBuilder()
.setKeyArgs(keyArgs)
.setRecursive(recursive)
.setStartKey(startKey)
- .setNumEntries(numEntries)
- .build();
+ .setNumEntries(numEntries);
+
+ if (allowPartialPrefixes) {
+ listStatusRequestBuilder.setAllowPartialPrefix(allowPartialPrefixes);
+ }
+
OMRequest omRequest = createOMRequest(Type.ListStatus)
- .setListStatusRequest(listStatusRequest)
+ .setListStatusRequest(listStatusRequestBuilder.build())
.build();
ListStatusResponse listStatusResponse =
handleError(submitRequest(omRequest)).getListStatusResponse();
@@ -1864,6 +1869,12 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
return statusList;
}
+ @Override
+ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries) throws IOException {
+ return listStatus(args, recursive, startKey, numEntries, false);
+ }
+
@Override
public List<RepeatedOmKeyInfo> listTrash(String volumeName,
String bucketName, String startKeyName, String keyPrefix, int maxKeys)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java
index 2fe2d68704..b798d1aefc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java
@@ -92,29 +92,41 @@ public class TestListStatus {
@Test
public void testSortedListStatus() throws Exception {
// a) test if output is sorted
- checkKeyList("", "", 1000, 10);
+ checkKeyList("", "", 1000, 10, false);
// b) number of keys returns is expected
- checkKeyList("", "", 2, 2);
+ checkKeyList("", "", 2, 2, false);
// c) check if full prefix works
- checkKeyList("a1", "", 100, 3);
+ checkKeyList("a1", "", 100, 3, false);
// d) check if full prefix with numEntries work
- checkKeyList("a1", "", 2, 2);
+ checkKeyList("a1", "", 2, 2, false);
// e) check if existing start key >>>
- checkKeyList("a1", "a1/a12", 100, 2);
+ checkKeyList("a1", "a1/a12", 100, 2, false);
- // f) check with non existing start key>>>
- checkKeyList("", "a7", 100, 6);
+ // f) check with non-existing start key
+ checkKeyList("", "a7", 100, 6, false);
- // TODO: Enable the following test after listKeys changes
-// // g) check if half prefix works <<<<
-// checkKeyList("b", "", 100, 4);
-//
-// // h) check half prefix with non-existing start key
-// checkKeyList("b", "b5", 100, 2);
+ // g) check if half prefix works
+ checkKeyList("b", "", 100, 4, true);
+
+ // h) check half prefix with non-existing start key
+ checkKeyList("b", "b5", 100, 2, true);
+
+ // i) check half prefix with non-existing parent in start key
+ checkKeyList("b", "c", 100, 0, true);
+
+ // i) check half prefix with non-existing parent in start key
+ checkKeyList("b", "b/g5", 100, 4, true);
+
+ // i) check half prefix with non-existing parent in start key
+ checkKeyList("b", "c/g5", 100, 0, true);
+
+ // j) check prefix with non-existing prefix key
+ // and non-existing parent in start key
+ checkKeyList("a1/a111", "a1/a111/a100", 100, 0, true);
}
private static void createFile(OzoneBucket bucket, String keyName)
@@ -143,8 +155,8 @@ public class TestListStatus {
"b1" File
"b2" File
- "b3" File
- "b4" File
+ "b7" File
+ "b8" File
*/
ozoneBucket.createDirectory("/a1");
createFile(ozoneBucket, "/a2");
@@ -167,11 +179,13 @@ public class TestListStatus {
}
private void checkKeyList(String keyPrefix, String startKey,
- long numEntries, int expectedNumKeys)
+ long numEntries, int expectedNumKeys,
+ boolean isPartialPrefix)
throws Exception {
List<OzoneFileStatus> statuses =
- fsoOzoneBucket.listStatus(keyPrefix, false, startKey, numEntries);
+ fsoOzoneBucket.listStatus(keyPrefix, false, startKey,
+ numEntries, isPartialPrefix);
Assert.assertEquals(expectedNumKeys, statuses.size());
System.out.println("BEGIN:::keyPrefix---> " + keyPrefix + ":::---> " +
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 60a03545a1..19f699756f 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -973,6 +973,7 @@ message ListStatusRequest {
required bool recursive = 2;
required string startKey = 3;
required uint64 numEntries = 4;
+ optional bool allowPartialPrefix = 5;
}
message ListStatusResponse {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 35ebbd1589..93c59a524a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1476,6 +1476,14 @@ public class KeyManagerImpl implements KeyManager {
return listStatus(args, recursive, startKey, numEntries, null);
}
+ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries,
+ String clientAddress)
+ throws IOException {
+ return listStatus(args, recursive, startKey, numEntries,
+ clientAddress, false);
+ }
+
/**
* List the status for a file or a directory and its contents.
*
@@ -1492,8 +1500,8 @@ public class KeyManagerImpl implements KeyManager {
@Override
@SuppressWarnings("methodlength")
public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
- String startKey, long numEntries, String clientAddress)
- throws IOException {
+ String startKey, long numEntries, String clientAddress,
+ boolean allowPartialPrefixes) throws IOException {
Preconditions.checkNotNull(args, "Key args can not be null");
String volName = args.getVolumeName();
String buckName = args.getBucketName();
@@ -1511,7 +1519,7 @@ public class KeyManagerImpl implements KeyManager {
this::getOzoneFileStatusFSO);
Collection<OzoneFileStatus> statuses =
statusHelper.listStatusFSO(args, startKey, numEntries,
- clientAddress);
+ clientAddress, allowPartialPrefixes);
return buildFinalStatusList(statuses, args, clientAddress);
} else {
return listStatusFSO(args, recursive, startKey, numEntries,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java
index 8d35027b21..7558ab85bd 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -48,6 +49,8 @@ import java.util.Collection;
import java.util.Collections;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.
+ ResultCodes.FILE_NOT_FOUND;
import static org.apache.hadoop.ozone.om.lock.
OzoneManagerLock.Resource.BUCKET_LOCK;
@@ -86,11 +89,10 @@ public class OzoneListStatusHelper {
}
public Collection<OzoneFileStatus> listStatusFSO(OmKeyArgs args,
- String startKey, long numEntries, String clientAddress)
- throws IOException {
+ String startKey, long numEntries, String clientAddress,
+ boolean allowPartialPrefixes) throws IOException {
Preconditions.checkNotNull(args, "Key args can not be null");
- boolean listKeysMode = false;
final String volumeName = args.getVolumeName();
final String bucketName = args.getBucketName();
String keyName = args.getKeyName();
@@ -118,15 +120,16 @@ public class OzoneListStatusHelper {
// if the keyName is null
if (StringUtils.isNotBlank(startKey)) {
if (StringUtils.isNotBlank(keyName)) {
- if (!listKeysMode &&
+ if (!OzoneFSUtils.isSibling(keyName, startKey) &&
!OzoneFSUtils.isImmediateChild(keyName, startKey)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("StartKey {} is not an immediate child of keyName {}. " +
- "Returns empty list", startKey, keyName);
+ LOG.debug("StartKey {} is not an immediate child or not a sibling"
+ + " of keyName {}. Returns empty list", startKey, keyName);
}
return new ArrayList<>();
}
} else {
+ // if the prefix is blank
keyName = OzoneFSUtils.getParentDir(startKey);
prefixKey = keyName;
args = args.toBuilder()
@@ -137,15 +140,27 @@ public class OzoneListStatusHelper {
}
OzoneFileStatus fileStatus =
- getStatusHelper.apply(args, clientAddress, listKeysMode);
+ getStatusHelper.apply(args, clientAddress, allowPartialPrefixes);
String dbPrefixKey;
if (fileStatus == null) {
// if the file status is null, prefix is a not a valid filesystem path
// this should only work in list keys mode.
// fetch the db key based on the prefix path.
- dbPrefixKey = getDbKey(keyName, args, volumeInfo, omBucketInfo);
- prefixKey = OzoneFSUtils.getParentDir(keyName);
+ try {
+ dbPrefixKey = getDbKey(keyName, args, volumeInfo, omBucketInfo);
+ prefixKey = OzoneFSUtils.getParentDir(keyName);
+ } catch (OMException ome) {
+ if (ome.getResult() == FILE_NOT_FOUND) {
+ // the parent dir cannot be found return null list
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Parent directory of keyName:{} does not exist." +
+ "Returns empty list", keyName);
+ }
+ return new ArrayList<>();
+ }
+ throw ome;
+ }
} else {
// If the keyname is a file just return one entry
if (fileStatus.isFile()) {
@@ -161,9 +176,16 @@ public class OzoneListStatusHelper {
}
// Determine startKeyPrefix for DB iteration
- String startKeyPrefix =
- Strings.isNullOrEmpty(startKey) ? "" :
- getDbKey(startKey, args, volumeInfo, omBucketInfo);
+ String startKeyPrefix = "";
+ try {
+ if (!Strings.isNullOrEmpty(startKey)) {
+ startKeyPrefix = getDbKey(startKey, args, volumeInfo, omBucketInfo);
+ }
+ } catch (OMException ome) {
+ if (ome.getResult() != FILE_NOT_FOUND) {
+ throw ome;
+ }
+ }
TreeMap<String, OzoneFileStatus> map = new TreeMap<>();
@@ -198,7 +220,7 @@ public class OzoneListStatusHelper {
.setSortDatanodesInPipeline(false)
.build();
OzoneFileStatus fileStatusInfo = getStatusHelper.apply(startKeyArgs,
- null, true);
+ null, false);
Preconditions.checkNotNull(fileStatusInfo);
startKeyParentId = getId(fileStatusInfo, omBucketInfo);
final long volumeId = volumeInfo.getObjectID();
@@ -325,7 +347,16 @@ public class OzoneListStatusHelper {
tableIterator.seek(prefixKey);
}
- if (!StringUtils.isBlank(startKey)) {
+ // only seek for the start key if the start key is lexicographically
+ // after the prefix key. For example
+ // Prefix key = 1024/c, Start key = 1024/a
+ // then do not seek for the start key
+ //
+ // on the other hand,
+ // Prefix key = 1024/a, Start key = 1024/c
+ // then seek for the start key
+ if (!StringUtils.isBlank(startKey) &&
+ startKey.compareTo(prefixKey) > 0) {
tableIterator.seek(startKey);
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 0e27ac0565..0ce27ab92f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -3456,7 +3456,14 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
@Override
public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
- String startKey, long numEntries) throws IOException {
+ String startKey, long numEntries)
+ throws IOException {
+ return listStatus(args, recursive, startKey, numEntries, false);
+ }
+
+ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries, boolean allowPartialPrefixes)
+ throws IOException {
ResolvedBucket bucket = resolveBucketLink(args);
@@ -3473,7 +3480,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
try {
metrics.incNumListStatus();
return keyManager.listStatus(args, recursive, startKey, numEntries,
- getClientAddress());
+ getClientAddress(), allowPartialPrefixes);
} catch (Exception ex) {
metrics.incNumListStatusFails();
auditSuccess = false;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java
index 4db738aa2f..ac004b9043 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java
@@ -96,4 +96,25 @@ public interface OzoneManagerFS extends IOzoneAcl {
List<OzoneFileStatus> listStatus(OmKeyArgs keyArgs, boolean recursive,
String startKey, long numEntries, String clientAddress)
throws IOException;
+
+ /**
+ * List the status for a file or a directory and its contents.
+ *
+ * @param keyArgs the args of the key provided by client.
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey
+ * exists its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param clientAddress a hint to key manager, order the datanode in returned
+ * pipeline by distance between client and datanode.
+ * @param allowPartialPrefixes if partial prefixes should be allowed,
+ * this is needed in context of ListKeys
+ * @return list of file status
+ * @throws IOException if file or bucket or volume does not exist
+ */
+ List<OzoneFileStatus> listStatus(OmKeyArgs keyArgs, boolean recursive,
+ String startKey, long numEntries, String clientAddress,
+ boolean allowPartialPrefixes)
+ throws IOException;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 1ecbf1cf21..b560e9860d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -1022,9 +1022,12 @@ public class OzoneManagerRequestHandler implements RequestHandler {
.setLatestVersionLocation(keyArgs.getLatestVersionLocation())
.setHeadOp(keyArgs.getHeadOp())
.build();
+ boolean allowPartialPrefixes =
+ request.hasAllowPartialPrefix() && request.getAllowPartialPrefix();
List<OzoneFileStatus> statuses =
impl.listStatus(omKeyArgs, request.getRecursive(),
- request.getStartKey(), request.getNumEntries());
+ request.getStartKey(), request.getNumEntries(),
+ allowPartialPrefixes);
ListStatusResponse.Builder
listStatusResponseBuilder =
ListStatusResponse.newBuilder();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org