You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2022/04/13 07:00:18 UTC
[GitHub] [ozone] adoroszlai commented on a diff in pull request #3192: HDDS-1726. Integrate FsPerfTest into Ozone freon performance test suite.
adoroszlai commented on code in PR #3192:
URL: https://github.com/apache/ozone/pull/3192#discussion_r849140812
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/WriteFileThroughputBenchmark.java:
##########
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Option;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.concurrent.Callable;
+
+/**
+ * File Generator for populating the cluster with random files.
+ */
+@CommandLine.Command(name = "wtb",
+ aliases = "write-throughput-benchmark",
+ description = "Benchmark for creating a file",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class WriteFileThroughputBenchmark extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ @Option(names = {"-P", "--path"},
+ description = "Ozone filesystem path OFS scheme",
+ defaultValue = "ofs://ozone1/volume1/bucket1")
+ private String rootPath;
+
+ @Option(names = {"-s", "--size"},
+ description = "Size of each generated files (in GB)",
+ defaultValue = "1")
+ private long fileSize;
+
+ @Option(names = {"-b", "--block"},
+ description = "Specify the Block Size in MB",
+ defaultValue = "128")
+ private long blockSize;
+
+ @Option(names = {"-B", "--buffer"},
+ description = "Size of buffer used store the generated " +
+ "key content",
+ defaultValue = "10240")
+ private int bufferSize;
+
+ @Option(names = {"-T", "--throttle"},
+ description = "Specify the Max Write throughput in bytes/second " +
+ "Should not be used while benchmarking",
+ defaultValue = "0")
+ private int throttle;
+
+ @Option(names = {"-r", "--replication"},
+ description = "Specify the Replication factor",
+ defaultValue = "1")
+ private short replication;
+
+ @Option(names = {"--flags"},
Review Comment:
`flags` is a rather vague name. I think `flush-mode` or something similar would be better.
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/WriteFileThroughputBenchmark.java:
##########
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Option;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.concurrent.Callable;
+
+/**
+ * File Generator for populating the cluster with random files.
+ */
+@CommandLine.Command(name = "wtb",
+ aliases = "write-throughput-benchmark",
+ description = "Benchmark for creating a file",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class WriteFileThroughputBenchmark extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ @Option(names = {"-P", "--path"},
+ description = "Ozone filesystem path OFS scheme",
+ defaultValue = "ofs://ozone1/volume1/bucket1")
+ private String rootPath;
+
+ @Option(names = {"-s", "--size"},
+ description = "Size of each generated files (in GB)",
+ defaultValue = "1")
+ private long fileSize;
+
+ @Option(names = {"-b", "--block"},
+ description = "Specify the Block Size in MB",
+ defaultValue = "128")
+ private long blockSize;
+
+ @Option(names = {"-B", "--buffer"},
+ description = "Size of buffer used store the generated " +
+ "key content",
+ defaultValue = "10240")
+ private int bufferSize;
+
+ @Option(names = {"-T", "--throttle"},
+ description = "Specify the Max Write throughput in bytes/second " +
+ "Should not be used while benchmarking",
+ defaultValue = "0")
+ private int throttle;
+
+ @Option(names = {"-r", "--replication"},
+ description = "Specify the Replication factor",
+ defaultValue = "1")
+ private short replication;
+
+ @Option(names = {"--flags"},
+ description = "Optionally issue hSync or hFlush after every write" +
+ "Cannot be used with hflush",
+ defaultValue = "None"
+ )
+ private String flag;
+
+ /**
+ * Type of flags.
+ */
+ public enum Flags {
+ hSync,
+ hFlush,
+ None,
+ }
+
+ // For Generating the content of the files
+ private ContentGenerator contentGenerator;
+ // For Creating the required configurations for the file system
+ private OzoneConfiguration configuration;
+
+ private URI uri;
+
+ // variable to check if the user wants a delay
+ private boolean isThrottled = false;
+
+ private long expectedIoTimeNs;
+
+ private Timer timer;
+
+ private final ThreadLocal<FileSystem> threadLocalFileSystem =
+ ThreadLocal.withInitial(this::createFS);
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(WriteFileThroughputBenchmark.class);
+
+
+ public Void call() throws Exception {
+
+ init();
+
+ LOG.info("NumFiles=" + getTestNo());
+ LOG.info("Total FileSize=" + fileSize);
+ LOG.info("BlockSize=" + blockSize);
+ LOG.info("BufferSize=" + bufferSize);
+ LOG.info("Replication=" + replication);
+ LOG.info("Threads=" + getThreadNo());
+ LOG.info("URI Scheme Used=" + uri.getScheme());
+ LOG.info("Flag Chosen=" + flag);
+
+ // Choosing which flag is to be set
+ boolean flush = false;
+ boolean sync = false;
+ Flags type = Flags.valueOf(flag);
+ switch (type) {
+ case hSync:
+ sync = true;
+ break;
+ case hFlush:
+ flush = true;
+ break;
+ case None:
+ break;
Review Comment:
Content generator also has the problem of mutually exclusive boolean flags. Can we pass `type` to `ContentGenerator` and let it use it directly in `flushOrSync`?
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/WriteFileThroughputBenchmark.java:
##########
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Option;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.concurrent.Callable;
+
+/**
+ * File Generator for populating the cluster with random files.
+ */
+@CommandLine.Command(name = "wtb",
+ aliases = "write-throughput-benchmark",
+ description = "Benchmark for creating a file",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class WriteFileThroughputBenchmark extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ @Option(names = {"-P", "--path"},
+ description = "Ozone filesystem path OFS scheme",
+ defaultValue = "ofs://ozone1/volume1/bucket1")
+ private String rootPath;
+
+ @Option(names = {"-s", "--size"},
+ description = "Size of each generated files (in GB)",
+ defaultValue = "1")
+ private long fileSize;
+
+ @Option(names = {"-b", "--block"},
+ description = "Specify the Block Size in MB",
+ defaultValue = "128")
+ private long blockSize;
+
+ @Option(names = {"-B", "--buffer"},
+ description = "Size of buffer used store the generated " +
+ "key content",
+ defaultValue = "10240")
+ private int bufferSize;
+
+ @Option(names = {"-T", "--throttle"},
+ description = "Specify the Max Write throughput in bytes/second " +
+ "Should not be used while benchmarking",
+ defaultValue = "0")
+ private int throttle;
+
+ @Option(names = {"-r", "--replication"},
+ description = "Specify the Replication factor",
+ defaultValue = "1")
+ private short replication;
+
+ @Option(names = {"--flags"},
+ description = "Optionally issue hSync or hFlush after every write" +
+ "Cannot be used with hflush",
+ defaultValue = "None"
+ )
+ private String flag;
+
+ /**
+ * Type of flags.
+ */
+ public enum Flags {
+ hSync,
+ hFlush,
+ None,
+ }
+
+ // For Generating the content of the files
+ private ContentGenerator contentGenerator;
+ // For Creating the required configurations for the file system
+ private OzoneConfiguration configuration;
+
+ private URI uri;
+
+ // variable to check if the user wants a delay
+ private boolean isThrottled = false;
+
+ private long expectedIoTimeNs;
+
+ private Timer timer;
+
+ private final ThreadLocal<FileSystem> threadLocalFileSystem =
+ ThreadLocal.withInitial(this::createFS);
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(WriteFileThroughputBenchmark.class);
+
+
+ public Void call() throws Exception {
+
+ init();
+
+ LOG.info("NumFiles=" + getTestNo());
+ LOG.info("Total FileSize=" + fileSize);
+ LOG.info("BlockSize=" + blockSize);
+ LOG.info("BufferSize=" + bufferSize);
+ LOG.info("Replication=" + replication);
+ LOG.info("Threads=" + getThreadNo());
+ LOG.info("URI Scheme Used=" + uri.getScheme());
+ LOG.info("Flag Chosen=" + flag);
+
+ // Choosing which flag is to be set
+ boolean flush = false;
+ boolean sync = false;
+ Flags type = Flags.valueOf(flag);
Review Comment:
We can avoid the need for this by changing `flag` from `String` to `Flags`. picocli supports enums.
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/WriteFileThroughputBenchmark.java:
##########
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Option;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.concurrent.Callable;
+
+/**
+ * File Generator for populating the cluster with random files.
+ */
+@CommandLine.Command(name = "wtb",
+ aliases = "write-throughput-benchmark",
+ description = "Benchmark for creating a file",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class WriteFileThroughputBenchmark extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ @Option(names = {"-P", "--path"},
+ description = "Ozone filesystem path OFS scheme",
+ defaultValue = "ofs://ozone1/volume1/bucket1")
+ private String rootPath;
+
+ @Option(names = {"-s", "--size"},
+ description = "Size of each generated files (in GB)",
+ defaultValue = "1")
+ private long fileSize;
+
+ @Option(names = {"-b", "--block"},
+ description = "Specify the Block Size in MB",
+ defaultValue = "128")
+ private long blockSize;
+
+ @Option(names = {"-B", "--buffer"},
+ description = "Size of buffer used store the generated " +
+ "key content",
+ defaultValue = "10240")
+ private int bufferSize;
+
+ @Option(names = {"-T", "--throttle"},
+ description = "Specify the Max Write throughput in bytes/second " +
+ "Should not be used while benchmarking",
+ defaultValue = "0")
+ private int throttle;
+
+ @Option(names = {"-r", "--replication"},
+ description = "Specify the Replication factor",
+ defaultValue = "1")
+ private short replication;
+
+ @Option(names = {"--flags"},
+ description = "Optionally issue hSync or hFlush after every write" +
+ "Cannot be used with hflush",
Review Comment:
This line is leftover from previous version (with booleans).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org