You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by bh...@apache.org on 2021/06/08 10:08:35 UTC
[ozone] branch master updated: HDDS-5243. Return latest key
location for clients (#2261)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6e278ba HDDS-5243. Return latest key location for clients (#2261)
6e278ba is described below
commit 6e278baf8f33fe6d5e9765414644c0c22870622b
Author: Symious <yi...@foxmail.com>
AuthorDate: Tue Jun 8 18:08:15 2021 +0800
HDDS-5243. Return latest key location for clients (#2261)
---
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../common/src/main/resources/ozone-default.xml | 8 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 10 ++
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 18 +++-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 17 +---
...OzoneManagerProtocolClientSideTranslatorPB.java | 4 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 101 +++++++++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 1 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 41 ++++++++-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +
10 files changed, 189 insertions(+), 18 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 370292f..7492a3a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -449,6 +449,11 @@ public final class OzoneConfigKeys {
public static final long OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT =
TimeUnit.DAYS.toMillis(10); // 10 days
+ public static final String OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION =
+ "ozone.client.key.latest.version.location";
+ public static final boolean OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION_DEFAULT =
+ true;
+
/**
* There is no need to instantiate this class.
*/
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 027681b..3a9133f 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2849,7 +2849,6 @@
</description>
</property>
-
<property>
<name>ozone.scm.ca.list.retry.interval</name>
<tag>OZONE, SCM, OM, DATANODE</tag>
@@ -2863,4 +2862,11 @@
</description>
</property>
+ <property>
+ <name>ozone.client.key.latest.version.location</name>
+ <tag>OZONE, CLIENT</tag>
+ <value>true</value>
+ <description>Ozone client gets the latest version location.
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 904c552..a7707f4 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -160,6 +160,7 @@ public class RpcClient implements ClientProtocol {
private final boolean checkKeyNameEnabled;
private final OzoneClientConfig clientConfig;
private final Cache<URI, KeyProvider> keyProviderCache;
+ private final boolean getLatestVersionLocation;
/**
* Creates RpcClient instance with the given configuration.
@@ -230,6 +231,9 @@ public class RpcClient implements ClientProtocol {
checkKeyNameEnabled = conf.getBoolean(
OMConfigKeys.OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_KEY,
OMConfigKeys.OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_DEFAULT);
+ getLatestVersionLocation = conf.getBoolean(
+ OzoneConfigKeys.OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION,
+ OzoneConfigKeys.OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION_DEFAULT);
long keyProviderCacheExpiryMs = conf.getTimeDuration(
OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY,
@@ -817,6 +821,7 @@ public class RpcClient implements ClientProtocol {
.setKeyName(keyName)
.setRefreshPipeline(true)
.setSortDatanodesInPipeline(topologyAwareReadEnabled)
+ .setLatestVersionLocation(getLatestVersionLocation)
.build();
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
return getInputStreamWithRetryFunction(keyInfo);
@@ -928,6 +933,7 @@ public class RpcClient implements ClientProtocol {
.setKeyName(keyName)
.setRefreshPipeline(true)
.setSortDatanodesInPipeline(topologyAwareReadEnabled)
+ .setLatestVersionLocation(getLatestVersionLocation)
.build();
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
@@ -1147,6 +1153,7 @@ public class RpcClient implements ClientProtocol {
.setKeyName(keyName)
.setRefreshPipeline(true)
.setSortDatanodesInPipeline(topologyAwareReadEnabled)
+ .setLatestVersionLocation(getLatestVersionLocation)
.build();
return ozoneManagerClient.getFileStatus(keyArgs);
}
@@ -1170,6 +1177,7 @@ public class RpcClient implements ClientProtocol {
.setBucketName(bucketName)
.setKeyName(keyName)
.setSortDatanodesInPipeline(topologyAwareReadEnabled)
+ .setLatestVersionLocation(getLatestVersionLocation)
.build();
OmKeyInfo keyInfo = ozoneManagerClient.lookupFile(keyArgs);
return getInputStreamWithRetryFunction(keyInfo);
@@ -1202,6 +1210,7 @@ public class RpcClient implements ClientProtocol {
.setKeyName(omKeyInfo.getKeyName())
.setRefreshPipeline(true)
.setSortDatanodesInPipeline(topologyAwareReadEnabled)
+ .setLatestVersionLocation(getLatestVersionLocation)
.build();
return ozoneManagerClient.lookupKey(omKeyArgs);
} catch (IOException e) {
@@ -1239,6 +1248,7 @@ public class RpcClient implements ClientProtocol {
.setKeyName(keyName)
.setRefreshPipeline(true)
.setSortDatanodesInPipeline(topologyAwareReadEnabled)
+ .setLatestVersionLocation(getLatestVersionLocation)
.build();
return ozoneManagerClient
.listStatus(keyArgs, recursive, startKey, numEntries);
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index dff74e3..868da8b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@ -46,6 +46,7 @@ public final class OmKeyArgs implements Auditable {
private boolean refreshPipeline;
private boolean sortDatanodesInPipeline;
private List<OzoneAcl> acls;
+ private boolean latestVersionLocation;
@SuppressWarnings("parameternumber")
private OmKeyArgs(String volumeName, String bucketName, String keyName,
@@ -53,7 +54,8 @@ public final class OmKeyArgs implements Auditable {
List<OmKeyLocationInfo> locationInfoList, boolean isMultipart,
String uploadID, int partNumber,
Map<String, String> metadataMap, boolean refreshPipeline,
- List<OzoneAcl> acls, boolean sortDatanode) {
+ List<OzoneAcl> acls, boolean sortDatanode,
+ boolean latestVersionLocation) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.keyName = keyName;
@@ -67,6 +69,7 @@ public final class OmKeyArgs implements Auditable {
this.refreshPipeline = refreshPipeline;
this.acls = acls;
this.sortDatanodesInPipeline = sortDatanode;
+ this.latestVersionLocation = latestVersionLocation;
}
public boolean getIsMultipartKey() {
@@ -133,6 +136,10 @@ public final class OmKeyArgs implements Auditable {
return sortDatanodesInPipeline;
}
+ public boolean getLatestVersionLocation() {
+ return latestVersionLocation;
+ }
+
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
@@ -168,6 +175,7 @@ public final class OmKeyArgs implements Auditable {
.addAllMetadata(metadata)
.setRefreshPipeline(refreshPipeline)
.setSortDatanodesInPipeline(sortDatanodesInPipeline)
+ .setLatestVersionLocation(latestVersionLocation)
.setAcls(acls);
}
@@ -187,6 +195,7 @@ public final class OmKeyArgs implements Auditable {
private Map<String, String> metadata = new HashMap<>();
private boolean refreshPipeline;
private boolean sortDatanodesInPipeline;
+ private boolean latestVersionLocation;
private List<OzoneAcl> acls;
public Builder setVolumeName(String volume) {
@@ -259,12 +268,17 @@ public final class OmKeyArgs implements Auditable {
return this;
}
+ public Builder setLatestVersionLocation(boolean latest) {
+ this.latestVersionLocation = latest;
+ return this;
+ }
+
public OmKeyArgs build() {
return new OmKeyArgs(volumeName, bucketName, keyName, dataSize,
replicationConfig, locationInfoList, isMultipartKey,
multipartUploadID,
multipartUploadPartNumber, metadata, refreshPipeline, acls,
- sortDatanodesInPipeline);
+ sortDatanodesInPipeline, latestVersionLocation);
}
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index c47b05b..5fba43c 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLoca
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.util.Time;
-import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,17 +73,6 @@ public final class OmKeyInfo extends WithObjectID {
this.bucketName = bucketName;
this.keyName = keyName;
this.dataSize = dataSize;
- // it is important that the versions are ordered from old to new.
- // Do this sanity check when versions got loaded on creating OmKeyInfo.
- // TODO : this is not necessary, here only because versioning is still a
- // work in-progress, remove this following check when versioning is
- // complete and prove correctly functioning
- long currentVersion = -1;
- for (OmKeyLocationInfoGroup version : versions) {
- Preconditions.checkArgument(
- currentVersion + 1 == version.getVersion());
- currentVersion = version.getVersion();
- }
this.keyLocationVersions = versions;
this.creationTime = creationTime;
this.modificationTime = modificationTime;
@@ -133,6 +121,11 @@ public final class OmKeyInfo extends WithObjectID {
return keyLocationVersions;
}
+ public void setKeyLocationVersions(
+ List<OmKeyLocationInfoGroup> keyLocationVersions) {
+ this.keyLocationVersions = keyLocationVersions;
+ }
+
public void updateModifcationTime() {
this.modificationTime = Time.monotonicNow();
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index e3de0b7..94fb3f9 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -681,6 +681,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.setKeyName(args.getKeyName())
.setDataSize(args.getDataSize())
.setSortDatanodes(args.getSortDatanodes())
+ .setLatestVersionLocation(args.getLatestVersionLocation())
.build();
req.setKeyArgs(keyArgs);
@@ -1214,6 +1215,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
.setSortDatanodes(args.getSortDatanodes())
+ .setLatestVersionLocation(args.getLatestVersionLocation())
.build();
GetFileStatusRequest req =
GetFileStatusRequest.newBuilder()
@@ -1267,6 +1269,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
.setSortDatanodes(args.getSortDatanodes())
+ .setLatestVersionLocation(args.getLatestVersionLocation())
.build();
LookupFileRequest lookupFileRequest = LookupFileRequest.newBuilder()
.setKeyArgs(keyArgs)
@@ -1431,6 +1434,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
.setSortDatanodes(args.getSortDatanodes())
+ .setLatestVersionLocation(args.getLatestVersionLocation())
.build();
ListStatusRequest listStatusRequest =
ListStatusRequest.newBuilder()
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
index e27afa7..a06a48b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -819,6 +819,107 @@ public class TestKeyManagerImpl {
}
@Test
+ public void testLatestLocationVersion() throws IOException {
+ String keyName = RandomStringUtils.randomAlphabetic(5);
+ OmKeyArgs keyArgs = createBuilder()
+ .setKeyName(keyName)
+ .setLatestVersionLocation(true)
+ .build();
+
+ // lookup for a non-existent key
+ try {
+ keyManager.lookupKey(keyArgs, null);
+ Assert.fail("Lookup key should fail for non existent key");
+ } catch (OMException ex) {
+ if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) {
+ throw ex;
+ }
+ }
+
+ // create a key
+ OpenKeySession keySession = keyManager.createFile(keyArgs, false, false);
+ // randomly select 3 datanodes
+ List<DatanodeDetails> nodeList = new ArrayList<>();
+ nodeList.add((DatanodeDetails)scm.getClusterMap().getNode(
+ 0, null, null, null, null, 0));
+ nodeList.add((DatanodeDetails)scm.getClusterMap().getNode(
+ 1, null, null, null, null, 0));
+ nodeList.add((DatanodeDetails)scm.getClusterMap().getNode(
+ 2, null, null, null, null, 0));
+ Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(1)));
+ Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(2)));
+ // create a pipeline using 3 datanodes
+ Pipeline pipeline = scm.getPipelineManager().createPipeline(
+ new RatisReplicationConfig(ReplicationFactor.THREE), nodeList);
+ List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
+ List<OmKeyLocationInfo> locationList =
+ keySession.getKeyInfo().getLatestVersionLocations().getLocationList();
+ Assert.assertEquals(1, locationList.size());
+ locationInfoList.add(
+ new OmKeyLocationInfo.Builder().setPipeline(pipeline)
+ .setBlockID(new BlockID(locationList.get(0).getContainerID(),
+ locationList.get(0).getLocalID())).build());
+ keyArgs.setLocationInfoList(locationInfoList);
+
+ keyManager.commitKey(keyArgs, keySession.getId());
+ OmKeyInfo key = keyManager.lookupKey(keyArgs, null);
+ Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
+
+ keySession = keyManager.createFile(keyArgs, true, true);
+ keyManager.commitKey(keyArgs, keySession.getId());
+
+ // Test lookupKey (latestLocationVersion == true)
+ key = keyManager.lookupKey(keyArgs, null);
+ Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
+
+ // Test ListStatus (latestLocationVersion == true)
+ List<OzoneFileStatus> fileStatuses =
+ keyManager.listStatus(keyArgs, false, "", 1);
+ Assert.assertEquals(fileStatuses.size(), 1);
+ Assert.assertEquals(fileStatuses.get(0).getKeyInfo()
+ .getKeyLocationVersions().size(), 1);
+
+ // Test GetFileStatus (latestLocationVersion == true)
+ OzoneFileStatus ozoneFileStatus = keyManager.getFileStatus(keyArgs, null);
+ Assert.assertEquals(ozoneFileStatus.getKeyInfo()
+ .getKeyLocationVersions().size(), 1);
+
+ // Test LookupFile (latestLocationVersion == true)
+ key = keyManager.lookupFile(keyArgs, null);
+ Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
+
+ keyArgs = createBuilder()
+ .setKeyName(keyName)
+ .setLatestVersionLocation(false)
+ .build();
+
+ // Test lookupKey (latestLocationVersion == false)
+ key = keyManager.lookupKey(keyArgs, null);
+ Assert.assertEquals(key.getKeyLocationVersions().size(), 2);
+
+ // Test ListStatus (latestLocationVersion == false)
+ fileStatuses = keyManager.listStatus(keyArgs, false, "", 100);
+ Assert.assertEquals(fileStatuses.size(), 1);
+ Assert.assertEquals(fileStatuses.get(0).getKeyInfo()
+ .getKeyLocationVersions().size(), 2);
+
+ // Test GetFileStatus (latestLocationVersion == false)
+ ozoneFileStatus = keyManager.getFileStatus(keyArgs, null);
+ Assert.assertEquals(ozoneFileStatus.getKeyInfo()
+ .getKeyLocationVersions().size(), 2);
+
+ // Test LookupFile (latestLocationVersion == false)
+ key = keyManager.lookupFile(keyArgs, null);
+ Assert.assertEquals(key.getKeyLocationVersions().size(), 2);
+
+ // Test ListKeys (latestLocationVersion is always true for ListKeys)
+ List<OmKeyInfo> keyInfos = keyManager.listKeys(keyArgs.getVolumeName(),
+ keyArgs.getBucketName(), "", keyArgs.getKeyName(), 100);
+ Assert.assertEquals(keyInfos.size(), 1);
+ Assert.assertEquals(keyInfos.get(0).getKeyLocationVersions().size(), 1);
+ }
+
+ @Test
public void testListStatusWithTableCache() throws Exception {
// Inspired by TestOmMetadataManager#testListKeys
String prefixKeyInDB = "key-d";
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index a10bb67..846ed9a 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -735,6 +735,7 @@ message KeyArgs {
// This will be set by leader OM in HA and update the original request.
optional FileEncryptionInfoProto fileEncryptionInfo = 15;
+ optional bool latestVersionLocation = 16;
}
message KeyLocation {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 44002ed..6c28ab0 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -668,6 +668,10 @@ public class KeyManagerImpl implements KeyManager {
throw new OMException("Key not found", KEY_NOT_FOUND);
}
+ if (args.getLatestVersionLocation()) {
+ slimLocationVersion(value);
+ }
+
// add block token for read.
addBlockToken4Read(value);
@@ -911,6 +915,11 @@ public class KeyManagerImpl implements KeyManager {
List<OmKeyInfo> keyList = metadataManager.listKeys(volumeName, bucketName,
startKey, keyPrefix, maxKeys);
+ // For listKeys, we return the latest Key Location by default
+ for (OmKeyInfo omKeyInfo : keyList) {
+ slimLocationVersion(omKeyInfo);
+ }
+
return keyList;
}
@@ -1757,7 +1766,8 @@ public class KeyManagerImpl implements KeyManager {
String keyName = args.getKeyName();
return getOzoneFileStatus(volumeName, bucketName, keyName,
- args.getRefreshPipeline(), args.getSortDatanodes(), clientAddress);
+ args.getRefreshPipeline(), args.getSortDatanodes(),
+ args.getLatestVersionLocation(), clientAddress);
}
private OzoneFileStatus getOzoneFileStatus(String volumeName,
@@ -1765,6 +1775,7 @@ public class KeyManagerImpl implements KeyManager {
String keyName,
boolean refreshPipeline,
boolean sortDatanodes,
+ boolean latestLocationVersion,
String clientAddress)
throws IOException {
OmKeyInfo fileKeyInfo = null;
@@ -1798,6 +1809,9 @@ public class KeyManagerImpl implements KeyManager {
// if the key is a file then do refresh pipeline info in OM by asking SCM
if (fileKeyInfo != null) {
+ if (latestLocationVersion) {
+ slimLocationVersion(fileKeyInfo);
+ }
// refreshPipeline flag check has been removed as part of
// https://issues.apache.org/jira/browse/HDDS-3658.
// Please refer this jira for more details.
@@ -1962,8 +1976,8 @@ public class KeyManagerImpl implements KeyManager {
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
OzoneFileStatus fileStatus = getOzoneFileStatus(volumeName, bucketName,
- keyName, args.getRefreshPipeline(), args.getSortDatanodes(),
- clientAddress);
+ keyName, args.getRefreshPipeline(), args.getSortDatanodes(),
+ args.getLatestVersionLocation(), clientAddress);
//if key is not of type file or if key is not found we throw an exception
if (fileStatus.isFile()) {
// add block token for read.
@@ -2190,6 +2204,9 @@ public class KeyManagerImpl implements KeyManager {
for (OzoneFileStatus fileStatus : fileStatusList) {
keyInfoList.add(fileStatus.getKeyInfo());
}
+ if (args.getLatestVersionLocation()) {
+ slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0]));
+ }
refreshPipeline(keyInfoList);
if (args.getSortDatanodes()) {
@@ -2351,4 +2368,22 @@ public class KeyManagerImpl implements KeyManager {
}
return nodeSet;
}
+
+ private void slimLocationVersion(OmKeyInfo... keyInfos) {
+ if (keyInfos != null) {
+ for (OmKeyInfo keyInfo : keyInfos) {
+ OmKeyLocationInfoGroup key = keyInfo.getLatestVersionLocations();
+ if (key == null) {
+ LOG.warn("No location version for key {}", keyInfo);
+ continue;
+ }
+ int keyLocationVersionLength = keyInfo.getKeyLocationVersions().size();
+ if (keyLocationVersionLength <= 1) {
+ continue;
+ }
+ keyInfo.setKeyLocationVersions(keyInfo.getKeyLocationVersions()
+ .subList(keyLocationVersionLength-1, keyLocationVersionLength));
+ }
+ }
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index c190768..4cbbd7f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -362,6 +362,7 @@ public class OzoneManagerRequestHandler implements RequestHandler {
.setKeyName(keyArgs.getKeyName())
.setRefreshPipeline(true)
.setSortDatanodesInPipeline(keyArgs.getSortDatanodes())
+ .setLatestVersionLocation(keyArgs.getLatestVersionLocation())
.build();
OmKeyInfo keyInfo = impl.lookupKey(omKeyArgs);
resp.setKeyInfo(keyInfo.getProtobuf(false, clientVersion));
@@ -566,6 +567,7 @@ public class OzoneManagerRequestHandler implements RequestHandler {
.setKeyName(keyArgs.getKeyName())
.setRefreshPipeline(true)
.setSortDatanodesInPipeline(keyArgs.getSortDatanodes())
+ .setLatestVersionLocation(keyArgs.getLatestVersionLocation())
.build();
return LookupFileResponse.newBuilder()
.setKeyInfo(impl.lookupFile(omKeyArgs).getProtobuf(clientVersion))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org