You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ra...@apache.org on 2021/04/28 07:16:16 UTC
[ozone] branch HDDS-2939 updated: HDDS-5097. [FSO] Cleanup
integration tests and reduce the execution time (#2185)
This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch HDDS-2939
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-2939 by this push:
new b6908b0 HDDS-5097. [FSO] Cleanup integration tests and reduce the execution time (#2185)
b6908b0 is described below
commit b6908b0b1f6553f1f2688baf048b45ec0d319eda
Author: Rakesh Radhakrishnan <ra...@apache.org>
AuthorDate: Wed Apr 28 12:45:57 2021 +0530
HDDS-5097. [FSO] Cleanup integration tests and reduce the execution time (#2185)
---
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 88 ++++------
.../apache/hadoop/fs/ozone/TestOzoneDirectory.java | 183 ---------------------
.../fs/ozone/TestOzoneFileInterfacesWithFSO.java | 30 ++++
.../apache/hadoop/fs/ozone/TestOzoneFileOps.java | 174 --------------------
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 16 +-
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 90 +++++-----
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 154 ++++++++++++++++-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 6 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 46 +++++-
.../fs/ozone/contract/ITestOzoneContractSeek.java | 26 +--
.../ozone/contract/ITestOzoneContractUnbuffer.java | 26 +--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 11 +-
12 files changed, 318 insertions(+), 532 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
index 50d056d..8ccaaa3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
@@ -51,9 +51,11 @@ import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
import static org.junit.Assert.fail;
@@ -83,8 +85,10 @@ public class TestDirectoryDeletingServiceWithFSO {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 3);
+ conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 1);
conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 5);
+ conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
+ TimeUnit.MILLISECONDS);
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
conf.setBoolean(OZONE_ACL_ENABLED, true);
if (isBucketFSOptimized) {
@@ -319,7 +323,7 @@ public class TestDirectoryDeletingServiceWithFSO {
}
@Test
- public void testDeleteSubFiles() throws Exception {
+ public void testDeleteFilesAndSubFiles() throws Exception {
Table<String, OmKeyInfo> deletedDirTable =
cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
@@ -335,7 +339,7 @@ public class TestDirectoryDeletingServiceWithFSO {
fs.mkdirs(root);
// Added 10 sub files inside root dir
- for (int i = 0; i<10; i++) {
+ for (int i = 0; i < 5; i++) {
Path path = new Path(root, "testKey" + i);
try (FSDataOutputStream stream = fs.create(path)) {
stream.write(1);
@@ -348,87 +352,51 @@ public class TestDirectoryDeletingServiceWithFSO {
// Before delete
assertTableRowCount(deletedDirTable, 0);
- assertTableRowCount(keyTable, 10);
+ assertTableRowCount(keyTable, 5);
assertTableRowCount(dirTable, 1);
long prevDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
- fs.delete(root, true);
+ // Case-1) Delete 3 Files directly.
+ for (int i = 0; i < 3; i++) {
+ Path path = new Path(root, "testKey" + i);
+ fs.delete(path, true);
+ }
DirectoryDeletingService dirDeletingService =
(DirectoryDeletingService) cluster.getOzoneManager().getKeyManager()
.getDirDeletingService();
- // After delete
- assertTableRowCount(keyTable, 0);
- assertTableRowCount(dirTable, 0);
+
+ // After delete. 2 more files left out under the root dir
+ assertTableRowCount(keyTable, 2);
+ assertTableRowCount(dirTable, 1);
// Eventually keys would get cleaned up from deletedTables too
assertTableRowCount(deletedDirTable, 0);
assertTableRowCount(deletedKeyTable, 0);
- assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 10);
- assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 1);
+ assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0);
+ assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0);
// verify whether KeyDeletingService has purged the keys
long currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
- Assert.assertEquals(currentDeletedKeyCount, prevDeletedKeyCount + 10);
- }
+ Assert.assertEquals(prevDeletedKeyCount + 3, currentDeletedKeyCount);
- @Test
- public void testDeleteFiles() throws Exception {
-
- Table<String, OmKeyInfo> deletedDirTable =
- cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
- Table<String, OmKeyInfo> keyTable =
- cluster.getOzoneManager().getMetadataManager().getKeyTable();
- Table<String, OmDirectoryInfo> dirTable =
- cluster.getOzoneManager().getMetadataManager().getDirectoryTable();
- Table<String, RepeatedOmKeyInfo> deletedKeyTable =
- cluster.getOzoneManager().getMetadataManager().getDeletedTable();
-
- Path root = new Path("/rootDir2");
- // Create parent dir from root.
- fs.mkdirs(root);
-
- // Added 10 sub files inside root dir
- for (int i = 0; i<10; i++) {
- Path path = new Path(root, "testKey" + i);
- try (FSDataOutputStream stream = fs.create(path)) {
- stream.write(1);
- }
- }
-
- KeyDeletingService keyDeletingService =
- (KeyDeletingService) cluster.getOzoneManager().getKeyManager()
- .getDeletingService();
-
- // Before delete
- assertTableRowCount(deletedDirTable, 0);
- assertTableRowCount(keyTable, 10);
- assertTableRowCount(dirTable, 1);
- long prevDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
-
- for (int i = 0; i<10; i++) {
- Path path = new Path(root, "testKey" + i);
- fs.delete(path, true);
- }
-
- DirectoryDeletingService dirDeletingService =
- (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager()
- .getDirDeletingService();
+ // Case-2) Delete dir, this will cleanup sub-files under the deleted dir.
+ fs.delete(root, true);
- // After delete
+ // After delete. 2 sub files to be deleted.
assertTableRowCount(keyTable, 0);
- assertTableRowCount(dirTable, 1);
+ assertTableRowCount(dirTable, 0);
// Eventually keys would get cleaned up from deletedTables too
assertTableRowCount(deletedDirTable, 0);
assertTableRowCount(deletedKeyTable, 0);
- assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0);
- assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0);
+ assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 2);
+ assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 1);
// verify whether KeyDeletingService has purged the keys
- long currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
- Assert.assertEquals(currentDeletedKeyCount, prevDeletedKeyCount + 10);
+ currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
+ Assert.assertEquals(prevDeletedKeyCount + 5, currentDeletedKeyCount);
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDirectory.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDirectory.java
deleted file mode 100644
index 3763295..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDirectory.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.TestDataUtil;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
-import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
-
-/**
- * Test verifies the entries and operations in directory table.
- */
-public class TestOzoneDirectory {
-
- @Rule
- public Timeout timeout = new Timeout(300000);
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestOzoneDirectory.class);
-
- private MiniOzoneCluster cluster;
- private FileSystem fs;
- private OzoneFileSystem o3fs;
- private String volumeName;
- private String bucketName;
-
- @Test(timeout = 300_000)
- public void testMultiLevelDirs() throws Exception {
- setupOzoneFileSystem();
- // Op 1. create dir -> /d1/d2/d3/d4/
- // Op 2. create dir -> /d1/d2/d3/d4/d5
- // Op 3. create dir -> /d1/d2/d3/d4/d6
- Path parent = new Path("/d1/d2/d3/d4/");
- fs.mkdirs(parent);
-
- OMMetadataManager omMgr = cluster.getOzoneManager().getMetadataManager();
- OmBucketInfo omBucketInfo = omMgr.getBucketTable().get(
- omMgr.getBucketKey(volumeName, bucketName));
- Assert.assertNotNull("Failed to find bucketInfo", omBucketInfo);
-
- ArrayList<String> dirKeys = new ArrayList<>();
- long d1ObjectID = verifyDirKey(omBucketInfo.getObjectID(), "d1", "/d1",
- dirKeys, omMgr);
- long d2ObjectID = verifyDirKey(d1ObjectID, "d2", "/d1/d2", dirKeys,
- omMgr);
- long d3ObjectID = verifyDirKey(d2ObjectID, "d3", "/d1/d2/d3",
- dirKeys, omMgr);
- long d4ObjectID = verifyDirKey(d3ObjectID, "d4", "/d1/d2/d3/d4",
- dirKeys, omMgr);
-
- Assert.assertEquals("Wrong OM numKeys metrics",
- 4, cluster.getOzoneManager().getMetrics().getNumKeys());
-
- // create sub-dirs under same parent
- Path subDir5 = new Path("/d1/d2/d3/d4/d5");
- fs.mkdirs(subDir5);
- Path subDir6 = new Path("/d1/d2/d3/d4/d6");
- fs.mkdirs(subDir6);
- long d5ObjectID = verifyDirKey(d4ObjectID, "d5",
- "/d1/d2/d3/d4/d5", dirKeys, omMgr);
- long d6ObjectID = verifyDirKey(d4ObjectID, "d6",
- "/d1/d2/d3/d4/d6", dirKeys, omMgr);
- Assert.assertTrue("Wrong objectIds for sub-dirs[" + d5ObjectID +
- "/d5, " + d6ObjectID + "/d6] of same parent!",
- d5ObjectID != d6ObjectID);
-
- Assert.assertEquals("Wrong OM numKeys metrics",
- 6, cluster.getOzoneManager().getMetrics().getNumKeys());
- }
-
- /**
- * Verify key name format and the DB key existence in the expected dirKeys
- * list.
- *
- * @param key table keyName
- * @param dirKeys expected keyName
- */
- private void verifyKeyFormat(String key, ArrayList<String> dirKeys) {
- String[] keyParts = StringUtils.split(key,
- OzoneConsts.OM_KEY_PREFIX.charAt(0));
- Assert.assertEquals("Invalid KeyName", 2, keyParts.length);
- boolean removed = dirKeys.remove(key);
- Assert.assertTrue("Key:" + key + " doesn't exists in directory table!",
- removed);
- }
-
- long verifyDirKey(long parentId, String dirKey, String absolutePath,
- ArrayList<String> dirKeys, OMMetadataManager omMgr)
- throws Exception {
- String dbKey = parentId + "/" + dirKey;
- dirKeys.add(dbKey);
- OmDirectoryInfo dirInfo = omMgr.getDirectoryTable().get(dbKey);
- Assert.assertNotNull("Failed to find " + absolutePath +
- " using dbKey: " + dbKey, dirInfo);
- Assert.assertEquals("Parent Id mismatches", parentId,
- dirInfo.getParentObjectID());
- Assert.assertEquals("Mismatches directory name", dirKey,
- dirInfo.getName());
- Assert.assertTrue("Mismatches directory creation time param",
- dirInfo.getCreationTime() > 0);
- Assert.assertEquals("Mismatches directory modification time param",
- dirInfo.getCreationTime(), dirInfo.getModificationTime());
- Assert.assertEquals("Wrong representation!",
- dbKey + ":" + dirInfo.getObjectID(), dirInfo.toString());
- return dirInfo.getObjectID();
- }
-
- private void setupOzoneFileSystem()
- throws IOException, TimeoutException, InterruptedException {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setInt(FS_TRASH_INTERVAL_KEY, 1);
- TestOMRequestUtils.configureFSOptimizedPaths(conf,
- true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
- .build();
- cluster.waitForClusterToBeReady();
- // create a volume and a bucket to be used by OzoneFileSystem
- OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
- volumeName = bucket.getVolumeName();
- bucketName = bucket.getName();
-
- String rootPath = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucket.getName(),
- bucket.getVolumeName());
-
- // Set the fs.defaultFS and start the filesystem
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
- // Set the number of keys to be processed during batch operate.
- conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5);
- fs = FileSystem.get(conf);
- }
-
- @After
- public void tearDown() {
- IOUtils.closeQuietly(fs);
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
-}
\ No newline at end of file
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesWithFSO.java
index ea49462..069154d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesWithFSO.java
@@ -27,6 +27,10 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
/**
* Test OzoneFileSystem Interfaces - prefix layout.
*
@@ -36,6 +40,11 @@ import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class TestOzoneFileInterfacesWithFSO extends TestOzoneFileInterfaces {
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {{false, true, true}});
+ }
+
public TestOzoneFileInterfacesWithFSO(boolean setDefaultFs,
boolean useAbsolutePath, boolean enabledFileSystemPaths) {
super(setDefaultFs, useAbsolutePath, enabledFileSystemPaths);
@@ -52,6 +61,27 @@ public class TestOzoneFileInterfacesWithFSO extends TestOzoneFileInterfaces {
@Override
@Test
+ @Ignore("HDDS-2939")
+ public void testReplication() throws IOException {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testPathToKey() throws Exception {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testFileSystemInit() throws IOException {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
@Ignore("TODO:HDDS-2939")
public void testDirectory() {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java
deleted file mode 100644
index c0f228d..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.TestDataUtil;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
-
-/**
- * Test verifies the entries and operations in file table, open file table etc.
- */
-public class TestOzoneFileOps {
-
- @Rule
- public Timeout timeout = new Timeout(300000);
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestOzoneFileOps.class);
-
- private MiniOzoneCluster cluster;
- private FileSystem fs;
- private String volumeName;
- private String bucketName;
-
- @Before
- public void setupOzoneFileSystem()
- throws IOException, TimeoutException, InterruptedException {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setInt(FS_TRASH_INTERVAL_KEY, 1);
- TestOMRequestUtils.configureFSOptimizedPaths(conf,
- true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
- .build();
- cluster.waitForClusterToBeReady();
- // create a volume and a bucket to be used by OzoneFileSystem
- OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
- volumeName = bucket.getVolumeName();
- bucketName = bucket.getName();
-
- String rootPath = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucket.getName(),
- bucket.getVolumeName());
-
- // Set the fs.defaultFS and start the filesystem
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
- // Set the number of keys to be processed during batch operate.
- conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5);
- fs = FileSystem.get(conf);
- }
-
- @After
- public void tearDown() {
- IOUtils.closeQuietly(fs);
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @Test(timeout = 300_000)
- public void testCreateFile() throws Exception {
- // Op 1. create dir -> /d1/d2/d3/d4/
- Path parent = new Path("/d1/d2/");
- Path file = new Path(parent, "file1");
- FSDataOutputStream outputStream = fs.create(file);
- String openFileKey = "";
-
- OMMetadataManager omMgr = cluster.getOzoneManager().getMetadataManager();
- OmBucketInfo omBucketInfo = omMgr.getBucketTable().get(
- omMgr.getBucketKey(volumeName, bucketName));
- Assert.assertNotNull("Failed to find bucketInfo", omBucketInfo);
-
- ArrayList<String> dirKeys = new ArrayList<>();
- long d1ObjectID = verifyDirKey(omBucketInfo.getObjectID(), "d1", "/d1",
- dirKeys, omMgr);
- long d2ObjectID = verifyDirKey(d1ObjectID, "d2", "/d1/d2", dirKeys,
- omMgr);
- openFileKey = d2ObjectID + OzoneConsts.OM_KEY_PREFIX + file.getName();
-
- // trigger CommitKeyRequest
- outputStream.close();
-
- OmKeyInfo omKeyInfo = omMgr.getKeyTable().get(openFileKey);
- Assert.assertNotNull("Invalid Key!", omKeyInfo);
- verifyOMFileInfoFormat(omKeyInfo, file.getName(), d2ObjectID);
-
- // wait for DB updates
- GenericTestUtils.waitFor(() -> {
- try {
- return omMgr.getOpenKeyTable().isEmpty();
- } catch (IOException e) {
- LOG.error("DB failure!", e);
- Assert.fail("DB failure!");
- return false;
- }
- }, 1000, 120000);
- }
-
- private void verifyOMFileInfoFormat(OmKeyInfo omKeyInfo, String fileName,
- long parentID) {
- Assert.assertEquals("Wrong keyName", fileName,
- omKeyInfo.getKeyName());
- Assert.assertEquals("Wrong parentID", parentID,
- omKeyInfo.getParentObjectID());
- String dbKey = parentID + OzoneConsts.OM_KEY_PREFIX + fileName;
- Assert.assertEquals("Wrong path format", dbKey,
- omKeyInfo.getPath());
- }
-
- long verifyDirKey(long parentId, String dirKey, String absolutePath,
- ArrayList<String> dirKeys, OMMetadataManager omMgr)
- throws Exception {
- String dbKey = parentId + OzoneConsts.OM_KEY_PREFIX + dirKey;
- dirKeys.add(dbKey);
- OmDirectoryInfo dirInfo = omMgr.getDirectoryTable().get(dbKey);
- Assert.assertNotNull("Failed to find " + absolutePath +
- " using dbKey: " + dbKey, dirInfo);
- Assert.assertEquals("Parent Id mismatches", parentId,
- dirInfo.getParentObjectID());
- Assert.assertEquals("Mismatches directory name", dirKey,
- dirInfo.getName());
- Assert.assertTrue("Mismatches directory creation time param",
- dirInfo.getCreationTime() > 0);
- Assert.assertEquals("Mismatches directory modification time param",
- dirInfo.getCreationTime(), dirInfo.getModificationTime());
- return dirInfo.getObjectID();
- }
-
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index 47c462b..acc335d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.commons.io.IOUtils;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
@@ -144,8 +145,11 @@ public class TestOzoneFileSystem {
private void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setInt(FS_TRASH_INTERVAL_KEY, 2);
- conf.setInt(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, 1);
+ conf.setFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, (float) 0.15);
+ // Trash with 9 second deletes and 6 seconds checkpoints
+ conf.setFloat(FS_TRASH_INTERVAL_KEY, (float) 0.15); // 9 seconds
+ conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, (float) 0.1); // 6 seconds
+
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
conf.setBoolean(OZONE_ACL_ENABLED, true);
if (isBucketFSOptimized) {
@@ -199,6 +203,10 @@ public class TestOzoneFileSystem {
}
}
+ public static MiniOzoneCluster getCluster() {
+ return cluster;
+ }
+
public static FileSystem getFs() {
return fs;
}
@@ -1233,8 +1241,8 @@ public class TestOzoneFileSystem {
Assert.assertTrue(trash.getConf().getClass(
"fs.trash.classname", TrashPolicy.class).
isAssignableFrom(TrashPolicyOzone.class));
- Assert.assertEquals(1, trash.getConf().
- getInt(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, 0));
+ Assert.assertEquals((float) 0.15, trash.getConf().
+ getFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, 0), 0);
// Call moveToTrash. We can't call protected fs.rename() directly
trash.moveToTrash(path);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java
index 4cd8750..fef2472 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.ozone.debug.PrefixParser;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
-import org.junit.After;
-import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.Assert;
@@ -44,18 +44,21 @@ import java.net.URI;
*/
public class TestOzoneFileSystemPrefixParser {
- private MiniOzoneCluster cluster = null;
+ private static MiniOzoneCluster cluster = null;
- private FileSystem fs;
+ private static FileSystem fs;
- private String volumeName;
+ private static String volumeName;
- private String bucketName;
+ private static String bucketName;
- private OzoneConfiguration configuration;
+ private static OzoneConfiguration configuration;
- @Before
- public void init() throws Exception {
+ private static Path dir;
+ private static Path file;
+
+ @BeforeClass
+ public static void init() throws Exception {
volumeName = RandomStringUtils.randomAlphabetic(10).toLowerCase();
bucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase();
@@ -76,50 +79,35 @@ public class TestOzoneFileSystemPrefixParser {
.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
volumeName);
fs = FileSystem.get(new URI(rootPath + "/test.txt"), configuration);
+
+ dir = new Path("/a/b/c/d/e");
+ fs.mkdirs(dir);
+ file = new Path("/a/b/c/file1");
+ FSDataOutputStream os = fs.create(file);
+ os.close();
}
- @After
- public void teardown() throws IOException {
+ @AfterClass
+ public static void teardown() throws IOException {
if (cluster != null) {
cluster.shutdown();
}
IOUtils.closeQuietly(fs);
}
- @Test
- public void testPrefixParseDir() throws Exception {
- Path dir = new Path("/a/b/c/d/e");
- fs.mkdirs(dir);
- Path file = new Path("/a/b/c/file1");
- FSDataOutputStream os = fs.create(file);
- os.close();
+ @Test(timeout = 120000)
+ public void testPrefixParsePath() throws Exception {
cluster.stop();
- PrefixParser parser = new PrefixParser();
- parser.parse(volumeName, bucketName,
- OMStorage.getOmDbDir(configuration).getPath(),
- dir.getParent().getParent().toString());
-
- assertPrefixStats(parser, 1, 1, 3, 0, 1, 1);
- }
+ // Directory Path
+ verifyPrefixParsePath(dir.getParent(), 4, 0, 0, 1);
- @Test
- public void testPrefixParseFile() throws Exception {
- Path dir = new Path("/a/b/c/d/e");
- fs.mkdirs(dir);
- Path file = new Path("/a/b/file1");
- FSDataOutputStream os = fs.create(file);
- os.close();
+ // File Path
+ verifyPrefixParsePath(file, 3, 1, 1, 1);
- cluster.stop();
- PrefixParser parser = new PrefixParser();
-
- parser.parse(volumeName, bucketName,
- OMStorage.getOmDbDir(configuration).getPath(),
- file.toString());
-
- assertPrefixStats(parser, 1, 1, 2, 1, 1, 1);
+ // Verify invalid path
+ testPrefixParseWithInvalidPaths();
}
private void assertPrefixStats(PrefixParser parser, int volumeCount,
@@ -139,16 +127,7 @@ public class TestOzoneFileSystemPrefixParser {
parser.getParserStats(PrefixParser.Types.DIRECTORY));
}
- @Test
- public void testPrefixParseWithInvalidPaths() throws Exception {
- Path dir = new Path("/a/b/c/d/e");
- fs.mkdirs(dir);
- Path file = new Path("/a/b/file1");
- FSDataOutputStream os = fs.create(file);
- os.close();
-
- cluster.stop();
-
+ private void testPrefixParseWithInvalidPaths() throws Exception {
PrefixParser invalidVolumeParser = new PrefixParser();
String invalidVolumeName =
RandomStringUtils.randomAlphabetic(10).toLowerCase();
@@ -172,9 +151,18 @@ public class TestOzoneFileSystemPrefixParser {
OMStorage.getOmDbDir(configuration).getPath(),
invalidIntermediateDir.toString());
- assertPrefixStats(invalidIntermediateDirParser, 1, 1, 2, 1, 1, 1);
+ assertPrefixStats(invalidIntermediateDirParser, 1, 1, 3, 1, 1, 1);
}
+ private void verifyPrefixParsePath(Path parent, int intermediateDirCount,
+ int nonExistentDirCount, int fileCount, int dirCount) throws Exception {
+ PrefixParser parser = new PrefixParser();
+
+ parser.parse(volumeName, bucketName,
+ OMStorage.getOmDbDir(configuration).getPath(), parent.toString());
+ assertPrefixStats(parser, 1, 1, intermediateDirCount, nonExistentDirCount,
+ fileCount, dirCount);
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
index 23bb916..36f0cb6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
@@ -24,6 +24,12 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.After;
import org.junit.BeforeClass;
@@ -56,9 +62,7 @@ public class TestOzoneFileSystemWithFSO extends TestOzoneFileSystem {
public static Collection<Object[]> data() {
return Arrays.asList(
new Object[]{true, true},
- new Object[]{true, false},
- new Object[]{false, true},
- new Object[]{false, false});
+ new Object[]{true, false});
}
@BeforeClass
@@ -86,6 +90,27 @@ public class TestOzoneFileSystemWithFSO extends TestOzoneFileSystem {
private static final Logger LOG =
LoggerFactory.getLogger(TestOzoneFileSystemWithFSO.class);
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testGetDirectoryModificationTime() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testOzoneFsServiceLoader() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testCreateWithInvalidPaths() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
@Test
public void testListStatusWithoutRecursiveSearch() throws Exception {
/*
@@ -282,11 +307,6 @@ public class TestOzoneFileSystemWithFSO extends TestOzoneFileSystem {
*/
@Test
public void testRenameDirToItsOwnSubDir() throws Exception {
- // Skip as this will run only in new layout
- if (!isEnabledFileSystemPaths()) {
- return;
- }
-
final String root = "/root";
final String dir1 = root + "/dir1";
final Path dir1Path = new Path(getFs().getUri().toString() + dir1);
@@ -343,4 +363,122 @@ public class TestOzoneFileSystemWithFSO extends TestOzoneFileSystem {
@Ignore("TODO:HDDS-5012")
public void testListStatusOnLargeDirectory() throws Exception {
}
+
+ @Test
+ public void testMultiLevelDirs() throws Exception {
+ // reset metrics
+ long numKeys = getCluster().getOzoneManager().getMetrics().getNumKeys();
+ getCluster().getOzoneManager().getMetrics().decNumKeys(numKeys);
+ Assert.assertEquals(0,
+ getCluster().getOzoneManager().getMetrics().getNumKeys());
+
+ // Op 1. create dir -> /d1/d2/d3/d4/
+ // Op 2. create dir -> /d1/d2/d3/d4/d5
+ // Op 3. create dir -> /d1/d2/d3/d4/d6
+ Path parent = new Path("/d1/d2/d3/d4/");
+ getFs().mkdirs(parent);
+
+ OMMetadataManager omMgr =
+ getCluster().getOzoneManager().getMetadataManager();
+ OmBucketInfo omBucketInfo = omMgr.getBucketTable()
+ .get(omMgr.getBucketKey(getVolumeName(), getBucketName()));
+ Assert.assertNotNull("Failed to find bucketInfo", omBucketInfo);
+
+ ArrayList<String> dirKeys = new ArrayList<>();
+ long d1ObjectID =
+ verifyDirKey(omBucketInfo.getObjectID(), "d1", "/d1", dirKeys, omMgr);
+ long d2ObjectID = verifyDirKey(d1ObjectID, "d2", "/d1/d2", dirKeys, omMgr);
+ long d3ObjectID =
+ verifyDirKey(d2ObjectID, "d3", "/d1/d2/d3", dirKeys, omMgr);
+ long d4ObjectID =
+ verifyDirKey(d3ObjectID, "d4", "/d1/d2/d3/d4", dirKeys, omMgr);
+
+ Assert.assertEquals("Wrong OM numKeys metrics", 4,
+ getCluster().getOzoneManager().getMetrics().getNumKeys());
+
+ // create sub-dirs under same parent
+ Path subDir5 = new Path("/d1/d2/d3/d4/d5");
+ getFs().mkdirs(subDir5);
+ Path subDir6 = new Path("/d1/d2/d3/d4/d6");
+ getFs().mkdirs(subDir6);
+ long d5ObjectID =
+ verifyDirKey(d4ObjectID, "d5", "/d1/d2/d3/d4/d5", dirKeys, omMgr);
+ long d6ObjectID =
+ verifyDirKey(d4ObjectID, "d6", "/d1/d2/d3/d4/d6", dirKeys, omMgr);
+ Assert.assertTrue(
+ "Wrong objectIds for sub-dirs[" + d5ObjectID + "/d5, " + d6ObjectID
+ + "/d6] of same parent!", d5ObjectID != d6ObjectID);
+
+ Assert.assertEquals("Wrong OM numKeys metrics", 6,
+ getCluster().getOzoneManager().getMetrics().getNumKeys());
+ }
+
+ @Test
+ public void testCreateFile() throws Exception {
+ // Op 1. create dir -> /d1/d2/d3/d4/
+ Path parent = new Path("/d1/d2/");
+ Path file = new Path(parent, "file1");
+ FSDataOutputStream outputStream = getFs().create(file);
+ String openFileKey = "";
+
+ OMMetadataManager omMgr =
+ getCluster().getOzoneManager().getMetadataManager();
+ OmBucketInfo omBucketInfo = omMgr.getBucketTable()
+ .get(omMgr.getBucketKey(getVolumeName(), getBucketName()));
+ Assert.assertNotNull("Failed to find bucketInfo", omBucketInfo);
+
+ ArrayList<String> dirKeys = new ArrayList<>();
+ long d1ObjectID =
+ verifyDirKey(omBucketInfo.getObjectID(), "d1", "/d1", dirKeys, omMgr);
+ long d2ObjectID = verifyDirKey(d1ObjectID, "d2", "/d1/d2", dirKeys, omMgr);
+ openFileKey = d2ObjectID + OzoneConsts.OM_KEY_PREFIX + file.getName();
+
+ // trigger CommitKeyRequest
+ outputStream.close();
+
+ OmKeyInfo omKeyInfo = omMgr.getKeyTable().get(openFileKey);
+ Assert.assertNotNull("Invalid Key!", omKeyInfo);
+ verifyOMFileInfoFormat(omKeyInfo, file.getName(), d2ObjectID);
+
+ // wait for DB updates
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return omMgr.getOpenKeyTable().isEmpty();
+ } catch (IOException e) {
+ LOG.error("DB failure!", e);
+ Assert.fail("DB failure!");
+ return false;
+ }
+ }, 1000, 120000);
+ }
+
+ private void verifyOMFileInfoFormat(OmKeyInfo omKeyInfo, String fileName,
+ long parentID) {
+ Assert.assertEquals("Wrong keyName", fileName, omKeyInfo.getKeyName());
+ Assert.assertEquals("Wrong parentID", parentID,
+ omKeyInfo.getParentObjectID());
+ String dbKey = parentID + OzoneConsts.OM_KEY_PREFIX + fileName;
+ Assert.assertEquals("Wrong path format", dbKey, omKeyInfo.getPath());
+ }
+
+ long verifyDirKey(long parentId, String dirKey, String absolutePath,
+ ArrayList<String> dirKeys, OMMetadataManager omMgr)
+ throws Exception {
+ String dbKey = parentId + "/" + dirKey;
+ dirKeys.add(dbKey);
+ OmDirectoryInfo dirInfo = omMgr.getDirectoryTable().get(dbKey);
+ Assert.assertNotNull("Failed to find " + absolutePath +
+ " using dbKey: " + dbKey, dirInfo);
+ Assert.assertEquals("Parent Id mismatches", parentId,
+ dirInfo.getParentObjectID());
+ Assert.assertEquals("Mismatches directory name", dirKey,
+ dirInfo.getName());
+ Assert.assertTrue("Mismatches directory creation time param",
+ dirInfo.getCreationTime() > 0);
+ Assert.assertEquals("Mismatches directory modification time param",
+ dirInfo.getCreationTime(), dirInfo.getModificationTime());
+ Assert.assertEquals("Wrong representation!",
+ dbKey + ":" + dirInfo.getObjectID(), dirInfo.toString());
+ return dirInfo.getObjectID();
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 813965e..6c7c36c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -80,6 +80,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
@@ -152,7 +153,10 @@ public class TestRootedOzoneFileSystem {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- conf.setInt(FS_TRASH_INTERVAL_KEY, 1);
+ conf.setFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, (float) 0.15);
+ // Trash with 9 second deletes and 6 seconds checkpoints
+ conf.setFloat(FS_TRASH_INTERVAL_KEY, (float) 0.15); // 9 seconds
+ conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, (float) 0.1); // 6 seconds
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
if (isBucketFSOptimized) {
TestOMRequestUtils.configureFSOptimizedPaths(conf,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java
index 0c43508..ea99e73 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java
@@ -44,9 +44,7 @@ public class TestRootedOzoneFileSystemWithFSO
public static Collection<Object[]> data() {
return Arrays.asList(
new Object[]{true, true},
- new Object[]{true, false},
- new Object[]{false, true},
- new Object[]{false, false});
+ new Object[]{true, false});
}
public TestRootedOzoneFileSystemWithFSO(boolean setDefaultFs,
@@ -60,6 +58,48 @@ public class TestRootedOzoneFileSystemWithFSO
TestRootedOzoneFileSystem.init();
}
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testTempMount() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testOzoneFsServiceLoader() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testCreateWithInvalidPaths() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testDeleteEmptyVolume() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testMkdirNonExistentVolumeBucket() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
+ @Override
+ @Test
+ @Ignore("HDDS-2939")
+ public void testMkdirNonExistentVolume() {
+ // ignore as this is not relevant to PREFIX layout changes
+ }
+
/**
* OFS: Test recursive listStatus on root and volume.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
index 9457bb8..51a35ee 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
@@ -19,35 +19,22 @@
package org.apache.hadoop.fs.ozone.contract;
import java.io.IOException;
-import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.junit.AfterClass;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+import org.junit.BeforeClass;
/**
* Ozone contract tests covering file seek.
*/
-@RunWith(Parameterized.class)
public class ITestOzoneContractSeek extends AbstractContractSeekTest {
- private static boolean fsOptimizedServer;
-
- public ITestOzoneContractSeek(boolean fsoServer)
- throws IOException {
- if (fsOptimizedServer != fsoServer) {
- setFsOptimizedServer(fsoServer);
- ITestOzoneContractUtils.restartCluster(
- fsOptimizedServer);
- }
- }
-
- public static void setFsOptimizedServer(boolean fsOptimizedServer) {
- ITestOzoneContractSeek.fsOptimizedServer = fsOptimizedServer;
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
}
@AfterClass
@@ -59,9 +46,4 @@ public class ITestOzoneContractSeek extends AbstractContractSeekTest {
protected AbstractFSContract createContract(Configuration conf) {
return new OzoneContract(conf);
}
-
- @Parameterized.Parameters
- public static Collection data() {
- return ITestOzoneContractUtils.getFsoCombinations();
- }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java
index 7f55774..e40b22e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java
@@ -21,31 +21,18 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractUnbufferTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.junit.AfterClass;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+import org.junit.BeforeClass;
import java.io.IOException;
-import java.util.Collection;
/**
* Ozone contract tests for {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer}.
*/
-@RunWith(Parameterized.class)
public class ITestOzoneContractUnbuffer extends AbstractContractUnbufferTest {
- private static boolean fsOptimizedServer;
-
- public ITestOzoneContractUnbuffer(boolean fsoServer)
- throws IOException {
- if (fsOptimizedServer != fsoServer) {
- setFsOptimizedServer(fsoServer);
- ITestOzoneContractUtils.restartCluster(
- fsOptimizedServer);
- }
- }
-
- public static void setFsOptimizedServer(boolean fsOptimizedServer) {
- ITestOzoneContractUnbuffer.fsOptimizedServer = fsOptimizedServer;
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
}
@AfterClass
@@ -57,9 +44,4 @@ public class ITestOzoneContractUnbuffer extends AbstractContractUnbufferTest {
protected AbstractFSContract createContract(Configuration conf) {
return new OzoneContract(conf);
}
-
- @Parameterized.Parameters
- public static Collection data() {
- return ITestOzoneContractUtils.getFsoCombinations();
- }
}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 5b92515..1132fe8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -344,6 +344,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private Thread emptier;
+ private static final int MSECS_PER_MINUTE = 60 * 1000;
+
@SuppressWarnings("methodlength")
private OzoneManager(OzoneConfiguration conf) throws IOException,
AuthenticationException {
@@ -1231,13 +1233,14 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* @throws IOException
*/
private void startTrashEmptier(Configuration conf) throws IOException {
- long hadoopTrashInterval =
- conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT);
+ float hadoopTrashInterval =
+ conf.getFloat(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT);
// check whether user has configured ozone specific trash-interval
// if not fall back to hadoop configuration
long trashInterval =
- conf.getLong(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY,
- hadoopTrashInterval);
+ (long)(conf.getFloat(
+ OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, hadoopTrashInterval)
+ * MSECS_PER_MINUTE);
if (trashInterval == 0) {
LOG.info("Trash Interval set to 0. Files deleted will not move to trash");
return;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org