You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2020/11/24 00:09:04 UTC
[ozone] branch HDDS-3698-upgrade updated (4384789 -> bf7624b)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a change to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 4384789 Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
add 8ae1408 HDDS-4478. Large deletedKeyset slows down OM via listStatus. (#1598)
add 517358b HDDS-4488. Open RocksDB read only when loading containers at Datanode startup (#1605)
add 49cd3ec HDDS-4432. Update Ratis version to latest snapshot. (#1586)
add fd879be HDDS-4476. Improve the ZH translation of the HA.md in doc. (#1597)
add f71fc12 HDDS-4417. Simplify Ozone client code with configuration object -- addendum (#1581)
add 6cc4a43 HDDS-4468. Fix Goofys listBucket large than 1000 objects will stuck forever (#1595)
new bf7624b Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 5 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 12 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 12 +-
.../CreatePipelineCommandHandler.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 4 +-
.../container/common/utils/ContainerCache.java | 2 +-
.../container/keyvalue/helpers/BlockUtils.java | 12 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 7 +-
.../container/metadata/AbstractDatanodeStore.java | 6 +-
.../metadata/DatanodeStoreSchemaOneImpl.java | 7 +-
.../metadata/DatanodeStoreSchemaTwoImpl.java | 7 +-
.../ozone/container/common/TestContainerCache.java | 2 +-
.../TestCreatePipelineCommandHandler.java | 13 +-
hadoop-hdds/docs/content/feature/HA.zh.md | 6 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 9 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 4 -
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 3 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 12 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 11 +-
.../rpc/TestOzoneClientRetriesOnException.java | 13 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 2 -
.../client/rpc/TestOzoneRpcClientWithRatis.java | 2 -
.../ozone/om/ratis/OzoneManagerRatisServer.java | 15 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 2 +-
.../om/response/file/OMFileCreateResponse.java | 5 +
.../ozone/om/response/key/OMKeyCreateResponse.java | 3 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 10 +-
.../ozone/om/response/TestCleanupTableInfo.java | 343 +++++++++++++++++++--
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 8 +-
.../ozone/s3/endpoint/ListObjectResponse.java | 11 +
.../freon/FollowerAppendLogEntryGenerator.java | 11 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 20 +-
pom.xml | 2 +-
37 files changed, 472 insertions(+), 133 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'upstream/master' into
HDDS-3698-upgrade
Posted by av...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit bf7624bc2a4e0a4ad9901126b1baf924551cbedd
Merge: 4384789 6cc4a43
Author: Aravindan Vijayan <av...@cloudera.com>
AuthorDate: Mon Nov 23 16:08:37 2020 -0800
Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 5 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 12 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 12 +-
.../CreatePipelineCommandHandler.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 4 +-
.../container/common/utils/ContainerCache.java | 2 +-
.../container/keyvalue/helpers/BlockUtils.java | 12 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 7 +-
.../container/metadata/AbstractDatanodeStore.java | 6 +-
.../metadata/DatanodeStoreSchemaOneImpl.java | 7 +-
.../metadata/DatanodeStoreSchemaTwoImpl.java | 7 +-
.../ozone/container/common/TestContainerCache.java | 2 +-
.../TestCreatePipelineCommandHandler.java | 13 +-
hadoop-hdds/docs/content/feature/HA.zh.md | 6 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 9 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 4 -
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 3 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 12 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 11 +-
.../rpc/TestOzoneClientRetriesOnException.java | 13 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 2 -
.../client/rpc/TestOzoneRpcClientWithRatis.java | 2 -
.../ozone/om/ratis/OzoneManagerRatisServer.java | 15 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 2 +-
.../om/response/file/OMFileCreateResponse.java | 5 +
.../ozone/om/response/key/OMKeyCreateResponse.java | 3 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 10 +-
.../ozone/om/response/TestCleanupTableInfo.java | 343 +++++++++++++++++++--
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 8 +-
.../ozone/s3/endpoint/ListObjectResponse.java | 11 +
.../freon/FollowerAppendLogEntryGenerator.java | 11 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 20 +-
pom.xml | 2 +-
37 files changed, 472 insertions(+), 133 deletions(-)
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
index f813000,a59a4e1..90c08cb
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
@@@ -55,41 -138,234 +138,239 @@@ public class TestCleanupTableInfo
}
@Test
- public void checkAllWriteResponseHasCleanupTableAnnotation() {
- getResponseClasses().forEach(aClass -> {
- Assert.assertTrue(
- aClass + "does not have annotation of CleanupTableInfo",
- aClass.isAnnotationPresent(CleanupTableInfo.class));
- });
- }
+ public void checkAnnotationAndTableName() {
+ OMMetadataManager omMetadataManager = om.getMetadataManager();
- @Test
- public void checkWriteResponseIsAnnotatedWithKnownTableNames()
- throws Exception {
- OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(conf);
Set<String> tables = omMetadataManager.listTableNames();
-
- getResponseClasses().forEach(aClass -> {
-
+ Set<Class<? extends OMClientResponse>> subTypes = responseClasses();
+ subTypes.forEach(aClass -> {
+ Assert.assertTrue(aClass + "does not have annotation of" +
+ " CleanupTableInfo",
+ aClass.isAnnotationPresent(CleanupTableInfo.class));
- String[] cleanupTables =
- aClass.getAnnotation(CleanupTableInfo.class).cleanupTables();
- Assert.assertTrue(cleanupTables.length >=1);
- for (String tableName : cleanupTables) {
- Assert.assertTrue(tables.contains(tableName));
+ CleanupTableInfo annotation =
+ aClass.getAnnotation(CleanupTableInfo.class);
+ String[] cleanupTables = annotation.cleanupTables();
+ boolean cleanupAll = annotation.cleanupAll();
-
+ if (cleanupTables.length >= 1) {
- assertTrue(
++ Assert.assertTrue(
+ Arrays.stream(cleanupTables).allMatch(tables::contains)
+ );
+ } else {
- assertTrue(cleanupAll);
++ Assert.assertTrue(cleanupAll);
}
-
});
}
- private Set<Class<? extends OMClientResponse>> getResponseClasses() {
- Reflections reflections =
- new Reflections("org.apache.hadoop.ozone.om.response");
++
+ private Set<Class<? extends OMClientResponse>> responseClasses() {
+ Reflections reflections = new Reflections(OM_RESPONSE_PACKAGE);
return reflections.getSubTypesOf(OMClientResponse.class);
}
- }
+
+ @Test
+ public void testFileCreateRequestSetsAllTouchedTableCachesForEviction() {
+ OMFileCreateRequest request = anOMFileCreateRequest();
+ Map<String, Integer> cacheItemCount = recordCacheItemCounts();
+
+ request.validateAndUpdateCache(om, 1, dbh);
+
+ assertCacheItemCounts(cacheItemCount, OMFileCreateResponse.class);
+ verify(omMetrics, times(1)).incNumCreateFile();
+ }
+
+ @Test
+ public void testKeyCreateRequestSetsAllTouchedTableCachesForEviction() {
+ OMKeyCreateRequest request = anOMKeyCreateRequest();
+ when(om.getEnableFileSystemPaths()).thenReturn(true);
+
+ Map<String, Integer> cacheItemCount = recordCacheItemCounts();
+
+ request.validateAndUpdateCache(om, 1, dbh);
+
+ assertCacheItemCounts(cacheItemCount, OMKeyCreateResponse.class);
+ verify(omMetrics, times(1)).incNumKeyAllocates();
+ }
+
-
-
+ private Map<String, Integer> recordCacheItemCounts() {
+ Map<String, Integer> cacheItemCount = new HashMap<>();
+ for (String tableName : om.getMetadataManager().listTableNames()) {
+ cacheItemCount.put(
+ tableName,
+ Iterators.size(
+ om.getMetadataManager().getTable(tableName).cacheIterator()
+ )
+ );
+ }
+ return cacheItemCount;
+ }
+
- private void assertCacheItemCounts(Map<String, Integer> cacheItemCount,
++ private void assertCacheItemCounts(
++ Map<String, Integer> cacheItemCount,
+ Class<? extends OMClientResponse> responseClass
+ ) {
+ CleanupTableInfo ann = responseClass.getAnnotation(CleanupTableInfo.class);
+ List<String> cleanup = Arrays.asList(ann.cleanupTables());
+ for (String tableName : om.getMetadataManager().listTableNames()) {
+ if (!cleanup.contains(tableName)) {
+ assertEquals(
+ "Cache item count of table " +tableName,
+ cacheItemCount.get(tableName).intValue(),
+ Iterators.size(
+ om.getMetadataManager().getTable(tableName).cacheIterator()
+ )
+ );
+ }
+ }
+ }
+
+ /**
+ * Adds the volume info to the volumeTable in the MetadataManager, and also
+ * add the value to the table's cache.
+ *
+ * @param volumeArgs the OMVolumeArgs object specifying the volume propertes
+ * @throws IOException if an IO issue occurs while wrtiing to RocksDB
+ */
+ private void addVolumeToMetaTable(OmVolumeArgs volumeArgs)
+ throws IOException {
+ String volumeKey = om.getMetadataManager().getVolumeKey(TEST_VOLUME_NAME);
+ om.getMetadataManager().getVolumeTable().put(volumeKey, volumeArgs);
+ om.getMetadataManager().getVolumeTable().addCacheEntry(
+ new CacheKey<>(volumeKey),
+ new CacheValue<>(Optional.of(volumeArgs), 2)
+ );
+ }
+
+ /**
+ * Adds the bucket info to the bucketTable in the MetadataManager, and also
+ * adds the value to the table's cache.
+ *
+ * @param bucketInfo the OMBucketInfo object specifying the bucket properties
+ * @throws IOException if an IO issue occurs while writing to RocksDB
+ */
+ private void addBucketToMetaTable(OmBucketInfo bucketInfo)
+ throws IOException {
+ String bucketKey = om.getMetadataManager()
+ .getBucketKey(bucketInfo.getVolumeName(), bucketInfo.getBucketName());
+ om.getMetadataManager().getBucketTable().put(bucketKey, bucketInfo);
+ om.getMetadataManager().getBucketTable().addCacheEntry(
+ new CacheKey<>(bucketKey),
+ new CacheValue<>(Optional.of(bucketInfo), 1)
+ );
+ }
+
+ /**
+ * Creates a spy object over an instantiated OMMetadataManager, giving the
+ * possibility to redefine behaviour. In the current implementation
+ * there isn't any behaviour which is redefined.
+ *
+ * @return the OMMetadataManager spy instance created.
+ * @throws IOException if I/O error occurs in setting up data store for the
+ * metadata manager.
+ */
+ private OMMetadataManager createOMMetadataManagerSpy() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ File newFolder = folder.newFolder();
+ if (!newFolder.exists()) {
+ Assert.assertTrue(newFolder.mkdirs());
+ }
+ ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString());
+ return spy(new OmMetadataManagerImpl(conf));
+ }
+
+ private OMFileCreateRequest anOMFileCreateRequest() {
+ OMRequest protoRequest = mock(OMRequest.class);
+ when(protoRequest.getCreateFileRequest()).thenReturn(aCreateFileRequest());
+ when(protoRequest.getCmdType()).thenReturn(Type.CreateFile);
+ when(protoRequest.getTraceID()).thenReturn("");
+ return new OMFileCreateRequest(protoRequest);
+ }
+
+ private OMKeyCreateRequest anOMKeyCreateRequest(){
+ OMRequest protoRequest = mock(OMRequest.class);
+ when(protoRequest.getCreateKeyRequest()).thenReturn(aKeyCreateRequest());
+ when(protoRequest.getCmdType()).thenReturn(Type.CreateKey);
+ when(protoRequest.getTraceID()).thenReturn("");
+ return new OMKeyCreateRequest(protoRequest);
+ }
+
+ private OmBucketInfo aBucketInfo() {
+ return OmBucketInfo.newBuilder()
+ .setVolumeName(TEST_VOLUME_NAME)
+ .setBucketName(TEST_BUCKET_NAME)
+ .setAcls(Collections.emptyList())
+ .setIsVersionEnabled(false)
+ .setStorageType(StorageType.DEFAULT)
+ .build();
+ }
+
+ private OmVolumeArgs aVolumeArgs() {
+ return OmVolumeArgs.newBuilder()
+ .setAdminName("admin")
+ .setOwnerName("owner")
+ .setVolume(TEST_VOLUME_NAME)
+ .build();
+ }
+
+ private CreateFileRequest aCreateFileRequest() {
+ return CreateFileRequest.newBuilder()
+ .setKeyArgs(aKeyArgs())
+ .setIsRecursive(true)
+ .setIsOverwrite(false)
+ .setClientID(1L)
+ .build();
+ }
+
+ private CreateKeyRequest aKeyCreateRequest() {
+ return CreateKeyRequest.newBuilder()
+ .setKeyArgs(aKeyArgs())
+ .setClientID(1L)
+ .build();
+ }
+
+ private KeyArgs aKeyArgs() {
+ return KeyArgs.newBuilder()
+ .setVolumeName(TEST_VOLUME_NAME)
+ .setBucketName(TEST_BUCKET_NAME)
+ .setKeyName(TEST_KEY)
+ .setDataSize(512L)
+ .addKeyLocations(aKeyLocation(TEST_BLOCK_ID))
+ .addKeyLocations(aKeyLocation(TEST_BLOCK_ID))
+ .addKeyLocations(aKeyLocation(TEST_BLOCK_ID))
+ .build();
+ }
+
+ private KeyLocation aKeyLocation(
+ HddsProtos.BlockID blockID) {
+ return KeyLocation.newBuilder()
+ .setBlockID(blockID)
+ .setOffset(0)
+ .setLength(512)
+ .setCreateVersion(0)
+ .setPipeline(aPipeline())
+ .build();
+ }
+
+ private Pipeline aPipeline() {
+ return Pipeline.newBuilder()
+ .setId(aPipelineID())
+ .addMembers(aDatanodeDetailsProto("192.168.1.1", "host1"))
+ .addMembers(aDatanodeDetailsProto("192.168.1.2", "host2"))
+ .addMembers(aDatanodeDetailsProto("192.168.1.3", "host3"))
+ .build();
+ }
+
+ private DatanodeDetailsProto aDatanodeDetailsProto(String s,
- String host1) {
++ String host1) {
+ return DatanodeDetailsProto.newBuilder()
+ .setUuid(UUID.randomUUID().toString())
+ .setIpAddress(s)
+ .setHostName(host1)
+ .build();
+ }
+
+ private HddsProtos.PipelineID aPipelineID() {
+ return HddsProtos.PipelineID.newBuilder()
+ .setId(UUID.randomUUID().toString())
+ .build();
+ }
-}
++}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org