You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "adoroszlai (via GitHub)" <gi...@apache.org> on 2023/12/20 08:02:28 UTC
[PR] HDDS-9922. Migrate TestOzoneFileInterfaces to JUnit5 [ozone]
adoroszlai opened a new pull request, #5838:
URL: https://github.com/apache/ozone/pull/5838
## What changes were proposed in this pull request?
Merge `TestOzoneFileInterfaces` test cases into `AbstractOzoneFileSystemTest`. The latter is already on JUnit5, and is run with various parameters via subclasses.
Tweaked assertions related to statistics to check increase in ops. count instead of assuming the stats are all 0 at the start of the test.
https://issues.apache.org/jira/browse/HDDS-9922
## How was this patch tested?
CI:
https://github.com/adoroszlai/ozone/actions/runs/7266811043
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-9922. Migrate TestOzoneFileInterfaces to JUnit5 [ozone]
Posted by "adoroszlai (via GitHub)" <gi...@apache.org>.
adoroszlai commented on code in PR #5838:
URL: https://github.com/apache/ozone/pull/5838#discussion_r1434821161
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java:
##########
@@ -1755,4 +1778,369 @@ public void testProcessingDetails() throws IOException, InterruptedException {
GenericTestUtils.setLogLevel(log, Level.INFO);
assertNotEquals(nonZeroLines, 0);
}
+
+ @Test
+ public void testOzFsReadWrite() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ long currentTime = Time.now();
+ int stringLen = 20;
+ OMMetadataManager metadataManager = cluster.getOzoneManager()
+ .getMetadataManager();
+ String lev1dir = "l1dir";
+ Path lev1path = createPath("/" + lev1dir);
+ String lev1key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev1path));
+ String lev2dir = "l2dir";
+ Path lev2path = createPath("/" + lev1dir + "/" + lev2dir);
+ String lev2key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev2path));
+
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path path = createPath("/" + lev1dir + "/" + lev2dir + "/" + filePath);
+ String fileKey = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(path));
+
+ // verify prefix directories and the file, do not already exist
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev1key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev2key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(fileKey));
+
+ Map<String, Long> statsBefore = statistics.snapshot();
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+
+ assertChange(statsBefore, statistics, OP_CREATE, 1);
+ assertChange(statsBefore, statistics, "objects_created", 1);
+
+ FileStatus status = fs.getFileStatus(path);
+
+ assertChange(statsBefore, statistics, OP_GET_FILE_STATUS, 1);
+ assertChange(statsBefore, statistics, "objects_query", 1);
+
+ // The timestamp of the newly created file should always be greater than
+ // the time when the test was started
+ assertTrue(status.getModificationTime() > currentTime);
+
+ assertFalse(status.isDirectory());
+ assertEquals(FsPermission.getFileDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
+ // verify prefix directories got created when creating the file.
+ assertEquals("l1dir/", metadataManager.getKeyTable(getBucketLayout()).get(lev1key).getKeyName());
+ assertEquals("l1dir/l2dir/", metadataManager.getKeyTable(getBucketLayout()).get(lev2key).getKeyName());
+ FileStatus lev1status = getDirectoryStat(lev1path);
+ assertNotNull(lev1status);
+ FileStatus lev2status = getDirectoryStat(lev2path);
+ assertNotNull(lev2status);
+
+ try (FSDataInputStream inputStream = fs.open(path)) {
+ byte[] buffer = new byte[stringLen];
+ // This read will not change the offset inside the file
+ int readBytes = inputStream.read(0, buffer, 0, buffer.length);
+ String out = new String(buffer, 0, buffer.length, UTF_8);
+ assertEquals(data, out);
+ assertEquals(readBytes, buffer.length);
+ assertEquals(0, inputStream.getPos());
+
+ // The following read will change the internal offset
+ readBytes = inputStream.read(buffer, 0, buffer.length);
+ assertEquals(data, out);
+ assertEquals(readBytes, buffer.length);
+ assertEquals(buffer.length, inputStream.getPos());
+ }
+
+ assertChange(statsBefore, statistics, OP_OPEN, 1);
+ assertChange(statsBefore, statistics, "objects_read", 1);
+ }
+
+ private static void assertChange(Map<String, Long> before, OzoneFSStorageStatistics after, String key, long delta) {
+ assertEquals(before.get(key) + delta, after.getLong(key));
+ }
+
+ @Test
+ public void testReplication() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ int stringLen = 20;
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path pathIllegal = createPath("/" + filePath + "illegal");
+ try (FSDataOutputStream streamIllegal = fs.create(pathIllegal, (short)2)) {
+ streamIllegal.writeBytes(data);
+ }
+ assertEquals(3, fs.getFileStatus(pathIllegal).getReplication());
+
+ Path pathLegal = createPath("/" + filePath + "legal");
+ try (FSDataOutputStream streamLegal = fs.create(pathLegal, (short)1)) {
+ streamLegal.writeBytes(data);
+ }
+ assertEquals(1, fs.getFileStatus(pathLegal).getReplication());
+ }
+
+ private void verifyOwnerGroup(FileStatus fileStatus) {
+ String owner = getCurrentUser();
+ assertEquals(owner, fileStatus.getOwner());
+ assertEquals(owner, fileStatus.getGroup());
+ }
+
+
+ @Test
+ public void testDirectory() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ String leafName = RandomStringUtils.randomAlphanumeric(5);
+ OMMetadataManager metadataManager = cluster.getOzoneManager()
+ .getMetadataManager();
+
+ String lev1dir = "abc";
+ Path lev1path = createPath("/" + lev1dir);
+ String lev1key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev1path));
+ String lev2dir = "def";
+ Path lev2path = createPath("/" + lev1dir + "/" + lev2dir);
+ String lev2key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev2path));
+
+ Path leaf = createPath("/" + lev1dir + "/" + lev2dir + "/" + leafName);
+ String leafKey = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(leaf));
+
+ // verify prefix directories and the leaf, do not already exist
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev1key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev2key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(leafKey));
+
+ assertTrue(fs.mkdirs(leaf));
+
+ // verify the leaf directory got created.
+ FileStatus leafstatus = getDirectoryStat(leaf);
+ assertNotNull(leafstatus);
+
+ // verify prefix directories got created when creating the leaf directory.
+ assertEquals("abc/", metadataManager
+ .getKeyTable(getBucketLayout())
+ .get(lev1key)
+ .getKeyName());
+ assertEquals("abc/def/", metadataManager
+ .getKeyTable(getBucketLayout())
+ .get(lev2key)
+ .getKeyName());
+ FileStatus lev1status = getDirectoryStat(lev1path);
+ assertNotNull(lev1status);
+ FileStatus lev2status = getDirectoryStat(lev2path);
+ assertNotNull(lev2status);
+
+ // check the root directory
+ FileStatus rootStatus = getDirectoryStat(createPath("/"));
+ assertNotNull(rootStatus);
+
+ // root directory listing should contain the lev1 prefix directory
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertEquals(lev1status, statusList[0]);
+ }
+
+ @Test
+ void testListStatus2() throws IOException {
+ List<Path> paths = new ArrayList<>();
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + dirPath);
+ paths.add(path);
+
+ final Map<String, Long> initialStats = statistics.snapshot();
+ assertTrue(fs.mkdirs(path));
+ assertChange(initialStats, statistics, OP_MKDIRS, 1);
+
+ final long initialListStatusCount = omMetrics.getNumListStatus();
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertChange(initialStats, statistics, Statistic.OBJECTS_LIST.getSymbol(), 1);
+ assertEquals(initialListStatusCount + 1, omMetrics.getNumListStatus());
+ assertEquals(fs.getFileStatus(path), statusList[0]);
+
+ dirPath = RandomStringUtils.randomAlphanumeric(5);
+ path = createPath("/" + dirPath);
+ paths.add(path);
+ assertTrue(fs.mkdirs(path));
+ assertChange(initialStats, statistics, OP_MKDIRS, 2);
+
+ statusList = fs.listStatus(createPath("/"));
+ assertEquals(2, statusList.length);
+ assertChange(initialStats, statistics, Statistic.OBJECTS_LIST.getSymbol(), 2);
+ assertEquals(initialListStatusCount + 2, omMetrics.getNumListStatus());
+ for (Path p : paths) {
+ assertTrue(Arrays.asList(statusList).contains(fs.getFileStatus(p)));
+ }
+ }
+
+ @Test
+ void testOzoneManagerFileSystemInterface() throws IOException {
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path path = createPath("/" + dirPath);
+ assertTrue(fs.mkdirs(path));
+
+ long numFileStatus =
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus();
+ FileStatus status = fs.getFileStatus(path);
+
+ assertEquals(numFileStatus + 1,
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus());
+ assertTrue(status.isDirectory());
+ assertEquals(FsPermission.getDirDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
+ long currentTime = System.currentTimeMillis();
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(o3fs.pathToKey(path))
+ .build();
+ OzoneFileStatus omStatus =
+ cluster.getOzoneManager().getFileStatus(keyArgs);
+ //Another get file status here, incremented the counter.
+ assertEquals(numFileStatus + 2,
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus());
+ assertTrue(omStatus.isDirectory());
+
+ // For directories, the time returned is the current time when the dir key
+ // doesn't actually exist on server; if it exists, it will be a fixed value.
+ // In this case, the dir key exists.
+ assertEquals(0, omStatus.getKeyInfo().getDataSize());
+ assertTrue(omStatus.getKeyInfo().getModificationTime() <= currentTime);
+ assertEquals(new Path(omStatus.getPath()).getName(),
+ o3fs.pathToKey(path));
+ }
+
+ @Test
+ public void testOzoneManagerLocatedFileStatus() throws IOException {
+ String data = RandomStringUtils.randomAlphanumeric(20);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue(status instanceof LocatedFileStatus);
+ LocatedFileStatus locatedFileStatus = (LocatedFileStatus) status;
+ assertTrue(locatedFileStatus.getBlockLocations().length >= 1);
+
+ for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) {
+ assertTrue(blockLocation.getNames().length >= 1);
+ assertTrue(blockLocation.getHosts().length >= 1);
+ }
+ }
+
+ @Test
+ void testBlockOffsetsWithMultiBlockFile() throws Exception {
+ // naive assumption: MiniOzoneCluster will not have larger than ~1GB
+ // block size when running this test.
+ int blockSize = (int) fs.getConf().getStorageSize(
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT,
+ StorageUnit.BYTES
+ );
+ String data = RandomStringUtils.randomAlphanumeric(2 * blockSize + 837);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue(status instanceof LocatedFileStatus);
Review Comment:
Thanks, added this kind of conversion to HDDS-9951.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-9922. Migrate TestOzoneFileInterfaces to JUnit5 [ozone]
Posted by "adoroszlai (via GitHub)" <gi...@apache.org>.
adoroszlai commented on PR #5838:
URL: https://github.com/apache/ozone/pull/5838#issuecomment-1867390936
Thanks @hemantk-12 for the review.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-9922. Migrate TestOzoneFileInterfaces to JUnit5 [ozone]
Posted by "hemantk-12 (via GitHub)" <gi...@apache.org>.
hemantk-12 commented on code in PR #5838:
URL: https://github.com/apache/ozone/pull/5838#discussion_r1434775868
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java:
##########
@@ -1755,4 +1778,369 @@ public void testProcessingDetails() throws IOException, InterruptedException {
GenericTestUtils.setLogLevel(log, Level.INFO);
assertNotEquals(nonZeroLines, 0);
}
+
+ @Test
+ public void testOzFsReadWrite() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ long currentTime = Time.now();
+ int stringLen = 20;
+ OMMetadataManager metadataManager = cluster.getOzoneManager()
+ .getMetadataManager();
+ String lev1dir = "l1dir";
+ Path lev1path = createPath("/" + lev1dir);
+ String lev1key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev1path));
+ String lev2dir = "l2dir";
+ Path lev2path = createPath("/" + lev1dir + "/" + lev2dir);
+ String lev2key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev2path));
+
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path path = createPath("/" + lev1dir + "/" + lev2dir + "/" + filePath);
+ String fileKey = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(path));
+
+ // verify prefix directories and the file, do not already exist
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev1key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev2key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(fileKey));
+
+ Map<String, Long> statsBefore = statistics.snapshot();
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+
+ assertChange(statsBefore, statistics, OP_CREATE, 1);
+ assertChange(statsBefore, statistics, "objects_created", 1);
+
+ FileStatus status = fs.getFileStatus(path);
+
+ assertChange(statsBefore, statistics, OP_GET_FILE_STATUS, 1);
+ assertChange(statsBefore, statistics, "objects_query", 1);
+
+ // The timestamp of the newly created file should always be greater than
+ // the time when the test was started
+ assertTrue(status.getModificationTime() > currentTime);
+
+ assertFalse(status.isDirectory());
+ assertEquals(FsPermission.getFileDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
+ // verify prefix directories got created when creating the file.
+ assertEquals("l1dir/", metadataManager.getKeyTable(getBucketLayout()).get(lev1key).getKeyName());
+ assertEquals("l1dir/l2dir/", metadataManager.getKeyTable(getBucketLayout()).get(lev2key).getKeyName());
+ FileStatus lev1status = getDirectoryStat(lev1path);
+ assertNotNull(lev1status);
+ FileStatus lev2status = getDirectoryStat(lev2path);
+ assertNotNull(lev2status);
+
+ try (FSDataInputStream inputStream = fs.open(path)) {
+ byte[] buffer = new byte[stringLen];
+ // This read will not change the offset inside the file
+ int readBytes = inputStream.read(0, buffer, 0, buffer.length);
+ String out = new String(buffer, 0, buffer.length, UTF_8);
+ assertEquals(data, out);
+ assertEquals(readBytes, buffer.length);
+ assertEquals(0, inputStream.getPos());
+
+ // The following read will change the internal offset
+ readBytes = inputStream.read(buffer, 0, buffer.length);
+ assertEquals(data, out);
+ assertEquals(readBytes, buffer.length);
+ assertEquals(buffer.length, inputStream.getPos());
+ }
+
+ assertChange(statsBefore, statistics, OP_OPEN, 1);
+ assertChange(statsBefore, statistics, "objects_read", 1);
+ }
+
+ private static void assertChange(Map<String, Long> before, OzoneFSStorageStatistics after, String key, long delta) {
+ assertEquals(before.get(key) + delta, after.getLong(key));
+ }
+
+ @Test
+ public void testReplication() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ int stringLen = 20;
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path pathIllegal = createPath("/" + filePath + "illegal");
+ try (FSDataOutputStream streamIllegal = fs.create(pathIllegal, (short)2)) {
+ streamIllegal.writeBytes(data);
+ }
+ assertEquals(3, fs.getFileStatus(pathIllegal).getReplication());
+
+ Path pathLegal = createPath("/" + filePath + "legal");
+ try (FSDataOutputStream streamLegal = fs.create(pathLegal, (short)1)) {
+ streamLegal.writeBytes(data);
+ }
+ assertEquals(1, fs.getFileStatus(pathLegal).getReplication());
+ }
+
+ private void verifyOwnerGroup(FileStatus fileStatus) {
+ String owner = getCurrentUser();
+ assertEquals(owner, fileStatus.getOwner());
+ assertEquals(owner, fileStatus.getGroup());
+ }
+
+
+ @Test
+ public void testDirectory() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ String leafName = RandomStringUtils.randomAlphanumeric(5);
+ OMMetadataManager metadataManager = cluster.getOzoneManager()
+ .getMetadataManager();
+
+ String lev1dir = "abc";
+ Path lev1path = createPath("/" + lev1dir);
+ String lev1key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev1path));
+ String lev2dir = "def";
+ Path lev2path = createPath("/" + lev1dir + "/" + lev2dir);
+ String lev2key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev2path));
+
+ Path leaf = createPath("/" + lev1dir + "/" + lev2dir + "/" + leafName);
+ String leafKey = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(leaf));
+
+ // verify prefix directories and the leaf, do not already exist
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev1key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev2key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(leafKey));
+
+ assertTrue(fs.mkdirs(leaf));
+
+ // verify the leaf directory got created.
+ FileStatus leafstatus = getDirectoryStat(leaf);
+ assertNotNull(leafstatus);
+
+ // verify prefix directories got created when creating the leaf directory.
+ assertEquals("abc/", metadataManager
+ .getKeyTable(getBucketLayout())
+ .get(lev1key)
+ .getKeyName());
+ assertEquals("abc/def/", metadataManager
+ .getKeyTable(getBucketLayout())
+ .get(lev2key)
+ .getKeyName());
+ FileStatus lev1status = getDirectoryStat(lev1path);
+ assertNotNull(lev1status);
+ FileStatus lev2status = getDirectoryStat(lev2path);
+ assertNotNull(lev2status);
+
+ // check the root directory
+ FileStatus rootStatus = getDirectoryStat(createPath("/"));
+ assertNotNull(rootStatus);
+
+ // root directory listing should contain the lev1 prefix directory
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertEquals(lev1status, statusList[0]);
+ }
+
+ @Test
+ void testListStatus2() throws IOException {
+ List<Path> paths = new ArrayList<>();
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + dirPath);
+ paths.add(path);
+
+ final Map<String, Long> initialStats = statistics.snapshot();
+ assertTrue(fs.mkdirs(path));
+ assertChange(initialStats, statistics, OP_MKDIRS, 1);
+
+ final long initialListStatusCount = omMetrics.getNumListStatus();
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertChange(initialStats, statistics, Statistic.OBJECTS_LIST.getSymbol(), 1);
+ assertEquals(initialListStatusCount + 1, omMetrics.getNumListStatus());
+ assertEquals(fs.getFileStatus(path), statusList[0]);
+
+ dirPath = RandomStringUtils.randomAlphanumeric(5);
+ path = createPath("/" + dirPath);
+ paths.add(path);
+ assertTrue(fs.mkdirs(path));
+ assertChange(initialStats, statistics, OP_MKDIRS, 2);
+
+ statusList = fs.listStatus(createPath("/"));
+ assertEquals(2, statusList.length);
+ assertChange(initialStats, statistics, Statistic.OBJECTS_LIST.getSymbol(), 2);
+ assertEquals(initialListStatusCount + 2, omMetrics.getNumListStatus());
+ for (Path p : paths) {
+ assertTrue(Arrays.asList(statusList).contains(fs.getFileStatus(p)));
+ }
+ }
+
+ @Test
+ void testOzoneManagerFileSystemInterface() throws IOException {
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path path = createPath("/" + dirPath);
+ assertTrue(fs.mkdirs(path));
+
+ long numFileStatus =
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus();
+ FileStatus status = fs.getFileStatus(path);
+
+ assertEquals(numFileStatus + 1,
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus());
+ assertTrue(status.isDirectory());
+ assertEquals(FsPermission.getDirDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
+ long currentTime = System.currentTimeMillis();
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(o3fs.pathToKey(path))
+ .build();
+ OzoneFileStatus omStatus =
+ cluster.getOzoneManager().getFileStatus(keyArgs);
+ //Another get file status here, incremented the counter.
+ assertEquals(numFileStatus + 2,
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus());
+ assertTrue(omStatus.isDirectory());
+
+ // For directories, the time returned is the current time when the dir key
+ // doesn't actually exist on server; if it exists, it will be a fixed value.
+ // In this case, the dir key exists.
+ assertEquals(0, omStatus.getKeyInfo().getDataSize());
+ assertTrue(omStatus.getKeyInfo().getModificationTime() <= currentTime);
+ assertEquals(new Path(omStatus.getPath()).getName(),
+ o3fs.pathToKey(path));
+ }
+
+ @Test
+ public void testOzoneManagerLocatedFileStatus() throws IOException {
+ String data = RandomStringUtils.randomAlphanumeric(20);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue(status instanceof LocatedFileStatus);
+ LocatedFileStatus locatedFileStatus = (LocatedFileStatus) status;
+ assertTrue(locatedFileStatus.getBlockLocations().length >= 1);
+
+ for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) {
+ assertTrue(blockLocation.getNames().length >= 1);
+ assertTrue(blockLocation.getHosts().length >= 1);
+ }
+ }
+
+ @Test
+ void testBlockOffsetsWithMultiBlockFile() throws Exception {
+ // naive assumption: MiniOzoneCluster will not have larger than ~1GB
+ // block size when running this test.
+ int blockSize = (int) fs.getConf().getStorageSize(
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT,
+ StorageUnit.BYTES
+ );
+ String data = RandomStringUtils.randomAlphanumeric(2 * blockSize + 837);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue(status instanceof LocatedFileStatus);
Review Comment:
nit: we can change it to `assertInstanceOf()`. Not sure if it will be done as part of HDDS-9978 or other jira.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-9922. Migrate TestOzoneFileInterfaces to JUnit5 [ozone]
Posted by "adoroszlai (via GitHub)" <gi...@apache.org>.
adoroszlai merged PR #5838:
URL: https://github.com/apache/ozone/pull/5838
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
Re: [PR] HDDS-9922. Migrate TestOzoneFileInterfaces to JUnit5 [ozone]
Posted by "hemantk-12 (via GitHub)" <gi...@apache.org>.
hemantk-12 commented on code in PR #5838:
URL: https://github.com/apache/ozone/pull/5838#discussion_r1434775868
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java:
##########
@@ -1755,4 +1778,369 @@ public void testProcessingDetails() throws IOException, InterruptedException {
GenericTestUtils.setLogLevel(log, Level.INFO);
assertNotEquals(nonZeroLines, 0);
}
+
+ @Test
+ public void testOzFsReadWrite() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ long currentTime = Time.now();
+ int stringLen = 20;
+ OMMetadataManager metadataManager = cluster.getOzoneManager()
+ .getMetadataManager();
+ String lev1dir = "l1dir";
+ Path lev1path = createPath("/" + lev1dir);
+ String lev1key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev1path));
+ String lev2dir = "l2dir";
+ Path lev2path = createPath("/" + lev1dir + "/" + lev2dir);
+ String lev2key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev2path));
+
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path path = createPath("/" + lev1dir + "/" + lev2dir + "/" + filePath);
+ String fileKey = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(path));
+
+ // verify prefix directories and the file, do not already exist
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev1key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev2key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(fileKey));
+
+ Map<String, Long> statsBefore = statistics.snapshot();
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+
+ assertChange(statsBefore, statistics, OP_CREATE, 1);
+ assertChange(statsBefore, statistics, "objects_created", 1);
+
+ FileStatus status = fs.getFileStatus(path);
+
+ assertChange(statsBefore, statistics, OP_GET_FILE_STATUS, 1);
+ assertChange(statsBefore, statistics, "objects_query", 1);
+
+ // The timestamp of the newly created file should always be greater than
+ // the time when the test was started
+ assertTrue(status.getModificationTime() > currentTime);
+
+ assertFalse(status.isDirectory());
+ assertEquals(FsPermission.getFileDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
+ // verify prefix directories got created when creating the file.
+ assertEquals("l1dir/", metadataManager.getKeyTable(getBucketLayout()).get(lev1key).getKeyName());
+ assertEquals("l1dir/l2dir/", metadataManager.getKeyTable(getBucketLayout()).get(lev2key).getKeyName());
+ FileStatus lev1status = getDirectoryStat(lev1path);
+ assertNotNull(lev1status);
+ FileStatus lev2status = getDirectoryStat(lev2path);
+ assertNotNull(lev2status);
+
+ try (FSDataInputStream inputStream = fs.open(path)) {
+ byte[] buffer = new byte[stringLen];
+ // This read will not change the offset inside the file
+ int readBytes = inputStream.read(0, buffer, 0, buffer.length);
+ String out = new String(buffer, 0, buffer.length, UTF_8);
+ assertEquals(data, out);
+ assertEquals(readBytes, buffer.length);
+ assertEquals(0, inputStream.getPos());
+
+ // The following read will change the internal offset
+ readBytes = inputStream.read(buffer, 0, buffer.length);
+ assertEquals(data, out);
+ assertEquals(readBytes, buffer.length);
+ assertEquals(buffer.length, inputStream.getPos());
+ }
+
+ assertChange(statsBefore, statistics, OP_OPEN, 1);
+ assertChange(statsBefore, statistics, "objects_read", 1);
+ }
+
+ private static void assertChange(Map<String, Long> before, OzoneFSStorageStatistics after, String key, long delta) {
+ assertEquals(before.get(key) + delta, after.getLong(key));
+ }
+
+ @Test
+ public void testReplication() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ int stringLen = 20;
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path pathIllegal = createPath("/" + filePath + "illegal");
+ try (FSDataOutputStream streamIllegal = fs.create(pathIllegal, (short)2)) {
+ streamIllegal.writeBytes(data);
+ }
+ assertEquals(3, fs.getFileStatus(pathIllegal).getReplication());
+
+ Path pathLegal = createPath("/" + filePath + "legal");
+ try (FSDataOutputStream streamLegal = fs.create(pathLegal, (short)1)) {
+ streamLegal.writeBytes(data);
+ }
+ assertEquals(1, fs.getFileStatus(pathLegal).getReplication());
+ }
+
+ private void verifyOwnerGroup(FileStatus fileStatus) {
+ String owner = getCurrentUser();
+ assertEquals(owner, fileStatus.getOwner());
+ assertEquals(owner, fileStatus.getGroup());
+ }
+
+
+ @Test
+ public void testDirectory() throws IOException {
+ assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));
+
+ String leafName = RandomStringUtils.randomAlphanumeric(5);
+ OMMetadataManager metadataManager = cluster.getOzoneManager()
+ .getMetadataManager();
+
+ String lev1dir = "abc";
+ Path lev1path = createPath("/" + lev1dir);
+ String lev1key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev1path));
+ String lev2dir = "def";
+ Path lev2path = createPath("/" + lev1dir + "/" + lev2dir);
+ String lev2key = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(lev2path));
+
+ Path leaf = createPath("/" + lev1dir + "/" + lev2dir + "/" + leafName);
+ String leafKey = metadataManager.getOzoneDirKey(volumeName, bucketName,
+ o3fs.pathToKey(leaf));
+
+ // verify prefix directories and the leaf, do not already exist
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev1key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(lev2key));
+ assertNull(metadataManager.getKeyTable(getBucketLayout()).get(leafKey));
+
+ assertTrue(fs.mkdirs(leaf));
+
+ // verify the leaf directory got created.
+ FileStatus leafstatus = getDirectoryStat(leaf);
+ assertNotNull(leafstatus);
+
+ // verify prefix directories got created when creating the leaf directory.
+ assertEquals("abc/", metadataManager
+ .getKeyTable(getBucketLayout())
+ .get(lev1key)
+ .getKeyName());
+ assertEquals("abc/def/", metadataManager
+ .getKeyTable(getBucketLayout())
+ .get(lev2key)
+ .getKeyName());
+ FileStatus lev1status = getDirectoryStat(lev1path);
+ assertNotNull(lev1status);
+ FileStatus lev2status = getDirectoryStat(lev2path);
+ assertNotNull(lev2status);
+
+ // check the root directory
+ FileStatus rootStatus = getDirectoryStat(createPath("/"));
+ assertNotNull(rootStatus);
+
+ // root directory listing should contain the lev1 prefix directory
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertEquals(lev1status, statusList[0]);
+ }
+
+ @Test
+ void testListStatus2() throws IOException {
+ List<Path> paths = new ArrayList<>();
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + dirPath);
+ paths.add(path);
+
+ final Map<String, Long> initialStats = statistics.snapshot();
+ assertTrue(fs.mkdirs(path));
+ assertChange(initialStats, statistics, OP_MKDIRS, 1);
+
+ final long initialListStatusCount = omMetrics.getNumListStatus();
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertChange(initialStats, statistics, Statistic.OBJECTS_LIST.getSymbol(), 1);
+ assertEquals(initialListStatusCount + 1, omMetrics.getNumListStatus());
+ assertEquals(fs.getFileStatus(path), statusList[0]);
+
+ dirPath = RandomStringUtils.randomAlphanumeric(5);
+ path = createPath("/" + dirPath);
+ paths.add(path);
+ assertTrue(fs.mkdirs(path));
+ assertChange(initialStats, statistics, OP_MKDIRS, 2);
+
+ statusList = fs.listStatus(createPath("/"));
+ assertEquals(2, statusList.length);
+ assertChange(initialStats, statistics, Statistic.OBJECTS_LIST.getSymbol(), 2);
+ assertEquals(initialListStatusCount + 2, omMetrics.getNumListStatus());
+ for (Path p : paths) {
+ assertTrue(Arrays.asList(statusList).contains(fs.getFileStatus(p)));
+ }
+ }
+
+ @Test
+ void testOzoneManagerFileSystemInterface() throws IOException {
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+
+ Path path = createPath("/" + dirPath);
+ assertTrue(fs.mkdirs(path));
+
+ long numFileStatus =
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus();
+ FileStatus status = fs.getFileStatus(path);
+
+ assertEquals(numFileStatus + 1,
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus());
+ assertTrue(status.isDirectory());
+ assertEquals(FsPermission.getDirDefault(), status.getPermission());
+ verifyOwnerGroup(status);
+
+ long currentTime = System.currentTimeMillis();
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(o3fs.pathToKey(path))
+ .build();
+ OzoneFileStatus omStatus =
+ cluster.getOzoneManager().getFileStatus(keyArgs);
+ //Another get file status here, incremented the counter.
+ assertEquals(numFileStatus + 2,
+ cluster.getOzoneManager().getMetrics().getNumGetFileStatus());
+ assertTrue(omStatus.isDirectory());
+
+ // For directories, the time returned is the current time when the dir key
+ // doesn't actually exist on server; if it exists, it will be a fixed value.
+ // In this case, the dir key exists.
+ assertEquals(0, omStatus.getKeyInfo().getDataSize());
+ assertTrue(omStatus.getKeyInfo().getModificationTime() <= currentTime);
+ assertEquals(new Path(omStatus.getPath()).getName(),
+ o3fs.pathToKey(path));
+ }
+
+ @Test
+ public void testOzoneManagerLocatedFileStatus() throws IOException {
+ String data = RandomStringUtils.randomAlphanumeric(20);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue(status instanceof LocatedFileStatus);
+ LocatedFileStatus locatedFileStatus = (LocatedFileStatus) status;
+ assertTrue(locatedFileStatus.getBlockLocations().length >= 1);
+
+ for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) {
+ assertTrue(blockLocation.getNames().length >= 1);
+ assertTrue(blockLocation.getHosts().length >= 1);
+ }
+ }
+
+ @Test
+ void testBlockOffsetsWithMultiBlockFile() throws Exception {
+ // naive assumption: MiniOzoneCluster will not have larger than ~1GB
+ // block size when running this test.
+ int blockSize = (int) fs.getConf().getStorageSize(
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT,
+ StorageUnit.BYTES
+ );
+ String data = RandomStringUtils.randomAlphanumeric(2 * blockSize + 837);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue(status instanceof LocatedFileStatus);
Review Comment:
nit: we can change it to `assertInstanceOf()`. Not sure if it will be done as part of HDDS-9951.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org