You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2020/06/03 14:39:21 UTC
[hadoop-ozone] branch master updated: HDDS-3518: Add a freon
generator to create directory tree with files (#895)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2e14e40 HDDS-3518: Add a freon generator to create directory tree with files (#895)
2e14e40 is described below
commit 2e14e40677b86a62ab3c065dd0f68fc096ce747a
Author: Rakesh Radhakrishnan <ra...@apache.org>
AuthorDate: Wed Jun 3 20:09:09 2020 +0530
HDDS-3518: Add a freon generator to create directory tree with files (#895)
---
.../ozone/freon/TestHadoopDirTreeGenerator.java | 180 +++++++++++++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 1 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 221 +++++++++++++++++++++
3 files changed, 402 insertions(+)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
new file mode 100644
index 0000000..6f5b113
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.ratis.server.impl.RaftServerImpl;
+import org.apache.ratis.server.raftlog.RaftLog;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.event.Level;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * Test for HadoopDirTreeGenerator.
+ */
+public class TestHadoopDirTreeGenerator {
+
+ private String path;
+ private OzoneConfiguration conf = null;
+ private MiniOzoneCluster cluster = null;
+ private ObjectStore store = null;
+
+ @Before
+ public void setup() {
+ path = GenericTestUtils
+ .getTempPath(TestOzoneClientKeyGenerator.class.getSimpleName());
+ GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
+ GenericTestUtils.setLogLevel(RaftServerImpl.LOG, Level.DEBUG);
+ File baseDir = new File(path);
+ baseDir.mkdirs();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ private void shutdown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ FileUtils.deleteDirectory(new File(path));
+ }
+ }
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ *
+ * @throws IOException
+ */
+ private void startCluster() throws Exception {
+ conf = new OzoneConfiguration();
+
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
+ cluster.waitForClusterToBeReady();
+ cluster.waitTobeOutOfSafeMode();
+
+ store = OzoneClientFactory.getRpcClient(conf).getObjectStore();
+ }
+
+ @Test
+ public void testNestedDirTreeGeneration() throws Exception {
+ try {
+ startCluster();
+ FileOutputStream out = FileUtils.openOutputStream(new File(path,
+ "conf"));
+ cluster.getConf().writeXml(out);
+ out.getFD().sync();
+ out.close();
+
+ verifyDirTree("vol1", "bucket1", 1,
+ 1, 1, 0);
+ verifyDirTree("vol2", "bucket1", 1,
+ 5, 1, 5);
+ verifyDirTree("vol3", "bucket1", 2,
+ 5, 3, 1);
+ verifyDirTree("vol4", "bucket1", 3,
+ 2, 4, 2);
+ verifyDirTree("vol5", "bucket1", 5,
+ 4, 1, 0);
+ } finally {
+ shutdown();
+ }
+ }
+
+ private void verifyDirTree(String volumeName, String bucketName, int depth,
+ int span, int fileCount, int perFileSizeInBytes)
+ throws IOException {
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ String rootPath = "o3fs://" + bucketName + "." + volumeName;
+ String confPath = new File(path, "conf").getAbsolutePath();
+ new Freon().execute(
+ new String[]{"-conf", confPath, "dtsg", "-d", depth + "", "-c",
+ fileCount + "", "-s", span + "", "-n", "1", "-r", rootPath,
+ "-g", perFileSizeInBytes + ""});
+ // verify the directory structure
+ FileSystem fileSystem = FileSystem.get(URI.create(rootPath),
+ conf);
+ Path rootDir = new Path(rootPath.concat("/"));
+ // verify root path details
+ FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
+ for (FileStatus fileStatus : fileStatuses) {
+ // verify the num of peer directories, expected span count is 1
+ // as it has only one dir at root.
+ verifyActualSpan(1, fileStatuses);
+ int actualDepth = traverseToLeaf(fileSystem, fileStatus.getPath(),
+ 1, depth, span, fileCount, perFileSizeInBytes);
+ Assert.assertEquals("Mismatch depth in a path",
+ depth, actualDepth);
+ }
+ }
+
+ private int traverseToLeaf(FileSystem fs, Path dirPath, int depth,
+ int expectedDepth, int expectedSpanCnt,
+ int expectedFileCnt, int perFileSizeInBytes)
+ throws IOException {
+ FileStatus[] fileStatuses = fs.listStatus(dirPath);
+ // check the num of peer directories except root and leaf as both
+ // has less dirs.
+ if (depth < expectedDepth - 1) {
+ verifyActualSpan(expectedSpanCnt, fileStatuses);
+ }
+ int actualNumFiles = 0;
+ for (FileStatus fileStatus : fileStatuses) {
+ if (fileStatus.isDirectory()) {
+ ++depth;
+ return traverseToLeaf(fs, fileStatus.getPath(), depth, expectedDepth,
+ expectedSpanCnt, expectedFileCnt, perFileSizeInBytes);
+ } else {
+ Assert.assertEquals("Mismatches file len",
+ perFileSizeInBytes, fileStatus.getLen());
+ actualNumFiles++;
+ }
+ }
+ Assert.assertEquals("Mismatches files count in a directory",
+ expectedFileCnt, actualNumFiles);
+ return depth;
+ }
+
+ private int verifyActualSpan(int expectedSpanCnt,
+ FileStatus[] fileStatuses) {
+ int actualSpan = 0;
+ for (FileStatus fileStatus : fileStatuses) {
+ if (fileStatus.isDirectory()) {
+ ++actualSpan;
+ }
+ }
+ Assert.assertEquals("Mismatches subdirs count in a directory",
+ expectedSpanCnt, actualSpan);
+ return actualSpan;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index cc78332..1eb1b68 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -43,6 +43,7 @@ import picocli.CommandLine.Option;
OmBucketGenerator.class,
HadoopFsGenerator.class,
HadoopNestedDirGenerator.class,
+ HadoopDirTreeGenerator.class,
HadoopFsValidator.class,
SameKeyReader.class,
S3KeyGenerator.class,
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java
new file mode 100644
index 0000000..62a4965
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.net.URI;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Directory & File Generator tool to test OM performance.
+ */
+@Command(name = "dtsg",
+ aliases = "dfs-tree-generator",
+ description =
+ "Create nested directories and create given number of files in each " +
+ "dir in any dfs compatible file system.",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class HadoopDirTreeGenerator extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HadoopDirTreeGenerator.class);
+
+ @Option(names = {"-r", "--rpath"},
+ description = "Hadoop FS root path",
+ defaultValue = "o3fs://bucket2.vol2")
+ private String rootPath;
+
+ @Option(names = {"-d", "--depth"},
+ description = "Number of directories to be generated recursively",
+ defaultValue = "5")
+ private int depth;
+
+ @Option(names = {"-c", "--fileCount"},
+ description = "Number of files to be written in each directory",
+ defaultValue = "2")
+ private int fileCount;
+
+ @Option(names = {"-g", "--fileSize"},
+ description = "Generated data size(in bytes) of each file to be " +
+ "written in each directory",
+ defaultValue = "4096")
+ private int fileSizeInBytes;
+
+ @Option(names = {"-b", "--buffer"},
+ description = "Size of buffer used to generated the file content.",
+ defaultValue = "1024")
+ private int bufferSize;
+
+ @Option(names = {"-s", "--span"},
+ description =
+ "Number of child directories to be created in each directory.",
+ defaultValue = "10")
+ private int span;
+
+ @Option(names = {"-l", "--nameLen"},
+ description =
+ "Length of the random name of directory you want to create.",
+ defaultValue = "10")
+ private int length;
+
+ private AtomicLong totalDirsCnt = new AtomicLong();
+
+ private Timer timer;
+
+ private ContentGenerator contentGenerator;
+
+ private FileSystem fileSystem;
+
+ @Override
+ public Void call() throws Exception {
+
+ init();
+ OzoneConfiguration configuration = createOzoneConfiguration();
+ fileSystem = FileSystem.get(URI.create(rootPath), configuration);
+
+ contentGenerator = new ContentGenerator(fileSizeInBytes, bufferSize);
+ timer = getMetrics().timer("file-create");
+
+ runTests(this::createDir);
+ return null;
+
+ }
+
+ /*
+ Nested directories will be created like this,
+ suppose you pass depth=3, span=3 and number of tests=1
+
+ Directory Structure:-
+ |-- Dir111
+ |
+ |-- Dir11 --|-- Dir112
+ | |
+ | |-- Dir113
+ |
+ |
+ | |-- Dir121
+ | |
+ Dir1 --|-- Dir12 --|-- Dir122
+ | |
+ | |-- Dir123
+ |
+ |
+ | |-- Dir131
+ | |
+ |-- Dir13 --|-- Dir132
+ |
+ |-- Dir133
+
+ In each directory 'c' number of files with file size in KBs 'g' will be
+ created.
+ */
+ private void createDir(long counter) throws Exception {
+ if (depth <= 0) {
+ LOG.info("Invalid depth value, at least one depth should be passed!");
+ return;
+ }
+ if (span <= 0) {
+ LOG.info("Invalid span value, at least one span should be passed!");
+ return;
+ }
+ String dir = makeDirWithGivenNumberOfFiles(rootPath);
+ if (depth > 1) {
+ createSubDirRecursively(dir, 1, 1);
+ }
+ System.out.println("Successfully created directories & files. Total Dirs " +
+ "Count=" + totalDirsCnt.get() + ", Total Files Count=" +
+ timer.getCount());
+ }
+
+ private void createSubDirRecursively(String parent, int depthIndex,
+ int spanIndex)
+ throws Exception {
+ if (depthIndex < depth) {
+ String depthSubDir = makeDirWithGivenNumberOfFiles(parent);
+ ++depthIndex;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SubDir:{}, depthIndex:{} +", depthSubDir, depthIndex);
+ }
+ // only non-leaf nodes will be iterated recursively..
+ if (depthIndex < depth) {
+ createSubDirRecursively(depthSubDir, depthIndex, spanIndex);
+ }
+ }
+
+ while(spanIndex < span) {
+ String levelSubDir = makeDirWithGivenNumberOfFiles(parent);
+ ++spanIndex;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SpanSubDir:{}, depthIndex:{}, spanIndex:{} +", levelSubDir,
+ depthIndex, spanIndex);
+ }
+ // only non-leaf nodes will be iterated recursively..
+ if (depthIndex < depth) {
+ createSubDirRecursively(levelSubDir, depthIndex, 1);
+ }
+ }
+ }
+
+ private String makeDirWithGivenNumberOfFiles(String parent)
+ throws Exception {
+ String dir = RandomStringUtils.randomAlphanumeric(length);
+ dir = parent.toString().concat("/").concat(dir);
+ fileSystem.mkdirs(new Path(dir));
+ totalDirsCnt.incrementAndGet();
+ // Add given number of files into the created directory.
+ createFiles(dir);
+ return dir;
+ }
+
+ private void createFile(String dir, long counter) throws Exception {
+ String fileName = dir.concat("/").concat(RandomStringUtils.
+ randomAlphanumeric(length));
+ Path file = new Path(fileName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FilePath:{}", file);
+ }
+ timer.time(() -> {
+ try (FSDataOutputStream output = fileSystem.create(file)) {
+ contentGenerator.write(output);
+ }
+ return null;
+ });
+ }
+
+ private void createFiles(String dir) throws Exception {
+ for (int i = 0; i < fileCount; i++) {
+ createFile(dir, i);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org