You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/10/22 16:12:12 UTC
[ozone] branch master updated: HDDS-7199. Implement new mix workload Read/Write Freon command (#3872)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ecdfc20756 HDDS-7199. Implement new mix workload Read/Write Freon command (#3872)
ecdfc20756 is described below
commit ecdfc207562e352bd1bcff52ca55b5c25456907f
Author: DaveTeng0 <10...@users.noreply.github.com>
AuthorDate: Sat Oct 22 09:12:05 2022 -0700
HDDS-7199. Implement new mix workload Read/Write Freon command (#3872)
---
.../src/main/smoketest/freon/read-write-key.robot | 53 +++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 2 +
.../hadoop/ozone/freon/KeyGeneratorUtil.java | 50 +++++
.../ozone/freon/OzoneClientKeyReadWriteOps.java | 245 +++++++++++++++++++++
.../hadoop/ozone/freon/RangeKeysGenerator.java | 164 ++++++++++++++
5 files changed, 514 insertions(+)
diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
new file mode 100644
index 0000000000..0995f757f3
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test freon read/write key commands
+Resource ../ozone-lib/freon.robot
+Test Timeout 5 minutes
+
+*** Variables ***
+${PREFIX} ${EMPTY}
+
+
+*** Test Cases ***
+Pre-generate 100 keys of size 1 byte each to Ozone
+ ${result} = Execute ozone freon ork -n 1 -t 10 -r 100 --size 1 -v voltest -b buckettest -p performanceTest
+
+Read 10 keys from pre-generated keys
+ ${keysCount} = BuiltIn.Set Variable 10
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -r 100 -v voltest -b buckettest -p performanceTest
+ Should contain ${result} Successful executions: ${keysCount}
+
+Read 10 keys' metadata from pre-generated keys
+ ${keysCount} = BuiltIn.Set Variable 10
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -m -r 100 -v voltest -b buckettest -p performanceTest
+ Should contain ${result} Successful executions: ${keysCount}
+
+Write 10 keys of size 1 byte each from key index 0 to 99
+ ${keysCount} = BuiltIn.Set Variable 10
+ ${size} = BuiltIn.Set Variable 1
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 0 --size ${size} -r 100 -v voltest -b buckettest -p performanceTest
+ Should contain ${result} Successful executions: ${keysCount}
+ ${keyName} = Execute echo -n '1' | md5sum | head -c 7
+ ${result} = Execute ozone sh key info /voltest/buckettest/performanceTest/${keyName}
+ Should contain ${result} \"dataSize\" : 1
+
+
+Run 90 % of read-key tasks and 10 % of write-key tasks for 10 keys from pre-generated keys
+ ${keysCount} = BuiltIn.Set Variable 10
+ ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 90 -r 100 -v voltest -b buckettest -p performanceTest
+ Should contain ${result} Successful executions: ${keysCount}
+
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index b6bb6055ee..dfb52ca9ad 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -68,6 +68,8 @@ import picocli.CommandLine.Option;
OmBucketReadWriteFileOps.class,
OmBucketReadWriteKeyOps.class,
OmRPCLoadGenerator.class,
+ OzoneClientKeyReadWriteOps.class,
+ RangeKeysGenerator.class
},
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true)
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
new file mode 100644
index 0000000000..9773caa440
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.util.function.Function;
+
+/**
+ * Utility class to generate key name from a given key index.
+ */
+public class KeyGeneratorUtil {
+ public static final String PURE_INDEX = "pureIndex";
+ public static final String MD5 = "md5";
+ public static final String FILE_DIR_SEPARATOR = "/";
+
+ public String generatePureIndexKeyName(int number) {
+ return String.valueOf(number);
+ }
+ public Function<Integer, String> pureIndexKeyNameFunc() {
+ return number -> String.valueOf(number);
+ }
+
+ public String generateMd5KeyName(int number) {
+ String encodedStr = DigestUtils.md5Hex(String.valueOf(number));
+ return encodedStr.substring(0, 7);
+ }
+
+ public Function<Integer, String> md5KeyNameFunc() {
+ return number -> DigestUtils.md5Hex(String.valueOf(number)).substring(0, 7);
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
new file mode 100644
index 0000000000..88aec0763c
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+
+import com.codahale.metrics.Timer;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.HashMap;
+
+import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR;
+
+/**
+ * Ozone key generator/reader for performance test.
+ */
+
+@CommandLine.Command(name = "ockrw",
+ aliases = "ozone-client-key-read-write-ops",
+ description = "Generate keys with a fixed name and ranges that can"
+ + " be written and read as sub-ranges from multiple clients.",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ @CommandLine.Option(names = {"-v", "--volume"},
+ description = "Name of the volume which contains the test data. " +
+ "Will be created if missing.",
+ defaultValue = "ockrwvolume")
+ private String volumeName;
+
+ @CommandLine.Option(names = {"-b", "--bucket"},
+ description = "Name of the bucket which contains the test data.",
+ defaultValue = "ockrwbucket")
+ private String bucketName;
+
+ @CommandLine.Option(names = {"-m", "--read-metadata-only"},
+ description = "If only read key's metadata.",
+ defaultValue = "false")
+ private boolean readMetadataOnly;
+
+ @CommandLine.Option(names = {"-s", "--start-index"},
+ description = "Start index of keys of read/write operation." +
+ "This can allow adding keys incrementally or parallel from multiple"
+ + " clients. Example: Write keys 0-1000000 followed by "
+ + "keys 1000001-2000000.",
+ defaultValue = "0")
+ private int startIndex;
+
+ @CommandLine.Option(names = {"-r", "--range"},
+ description = "Range of read/write operations. This in co-ordination"
+ + " with --start-index can specify the range to read. "
+ + "Example: Read from --start-index 1000 and read --range 1000 keys.",
+ defaultValue = "1")
+ private int range;
+
+ @CommandLine.Option(names = {"--size"},
+ description = "Object size (in bytes) " +
+ "to read/write. If user sets a read size which is larger"
+ + " than the key size, it only reads bytes up to key size.",
+ defaultValue = "1")
+ private int objectSizeInBytes;
+
+ @CommandLine.Option(names = {"--contiguous"},
+ description = "By default, the keys are randomized lexically"
+ + " by calculating the md5 of the key name. If this option is set,"
+ + " the keys are written lexically contiguously.",
+ defaultValue = "false")
+ private boolean keySorted;
+
+ @CommandLine.Option(names = {"--percentage-read"},
+ description = "Percentage of read tasks in mix workload."
+ + " The remainder of the percentage will writes to keys."
+ + " Example --percentage-read 90 will result in 10% writes.",
+ defaultValue = "100")
+ private int percentageRead;
+
+ @CommandLine.Option(
+ names = "--om-service-id",
+ description = "OM Service ID"
+ )
+ private String omServiceID = null;
+
+ private Timer timer;
+
+ private OzoneClient[] ozoneClients;
+
+ private int clientCount;
+
+ private byte[] keyContent;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class);
+
+ /**
+ * Task type of read task, or write task.
+ */
+ public enum TaskType {
+ READ_TASK,
+ WRITE_TASK
+ }
+ private KeyGeneratorUtil kg;
+
+
+ @Override
+ public Void call() throws Exception {
+ init();
+ OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
+ clientCount = getThreadNo();
+ ozoneClients = new OzoneClient[clientCount];
+ for (int i = 0; i < clientCount; i++) {
+ ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration);
+ }
+
+ ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName);
+
+ timer = getMetrics().timer("key-read-write");
+ if (objectSizeInBytes >= 0) {
+ keyContent = RandomUtils.nextBytes(objectSizeInBytes);
+ }
+ if (kg == null) {
+ kg = new KeyGeneratorUtil();
+ }
+ runTests(this::readWriteKeys);
+
+ for (int i = 0; i < clientCount; i++) {
+ if (ozoneClients[i] != null) {
+ ozoneClients[i].close();
+ }
+ }
+ return null;
+ }
+
+ public void readWriteKeys(long counter) throws RuntimeException, IOException {
+ int clientIndex = (int)((counter) % clientCount);
+ TaskType taskType = decideReadOrWriteTask();
+ String keyName = getKeyName();
+
+ timer.time(() -> {
+ try {
+ switch (taskType) {
+ case READ_TASK:
+ processReadTasks(keyName, ozoneClients[clientIndex]);
+ break;
+ case WRITE_TASK:
+ processWriteTasks(keyName, ozoneClients[clientIndex]);
+ break;
+ default:
+ break;
+ }
+ } catch (RuntimeException ex) {
+ LOG.error(ex.getMessage());
+ throw ex;
+ } catch (IOException ex) {
+ LOG.error(ex.getMessage());
+ throw new RuntimeException(ex.getMessage());
+ }
+
+ });
+ }
+ @SuppressFBWarnings
+ public void processReadTasks(String keyName, OzoneClient client)
+ throws RuntimeException, IOException {
+ OzoneKeyDetails keyDetails = client.getProxy().
+ getKeyDetails(volumeName, bucketName, keyName);
+ if (!readMetadataOnly) {
+ byte[] data = new byte[objectSizeInBytes];
+ try (OzoneInputStream introStream = keyDetails.getContent()) {
+ introStream.read(data);
+ } catch (Exception ex) {
+ throw ex;
+ }
+ }
+ }
+
+ public void processWriteTasks(String keyName, OzoneClient ozoneClient)
+ throws RuntimeException, IOException {
+ try (OzoneOutputStream out =
+ ozoneClient.getProxy().createKey(volumeName, bucketName,
+ keyName, objectSizeInBytes, null, new HashMap())) {
+ out.write(keyContent);
+ } catch (Exception ex) {
+ throw ex;
+ }
+ }
+
+ public TaskType decideReadOrWriteTask() {
+ if (percentageRead == 100) {
+ return TaskType.READ_TASK;
+ } else if (percentageRead == 0) {
+ return TaskType.WRITE_TASK;
+ }
+ //mix workload
+ int tmp = ThreadLocalRandom.current().nextInt(1, 101);
+ if (tmp <= percentageRead) {
+ return TaskType.READ_TASK;
+ } else {
+ return TaskType.WRITE_TASK;
+ }
+ }
+
+ public String getKeyName() {
+ StringBuilder keyNameSb = new StringBuilder();
+ int randomIdxWithinRange = ThreadLocalRandom.current().
+ nextInt(startIndex, startIndex + range);
+
+ if (keySorted) {
+ keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
+ append(randomIdxWithinRange);
+ } else {
+ keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
+ append(kg.generateMd5KeyName(randomIdxWithinRange));
+ }
+ return keyNameSb.toString();
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java
new file mode 100644
index 0000000000..e149410281
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+import java.util.function.Function;
+import java.util.HashMap;
+
+import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.PURE_INDEX;
+import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.MD5;
+import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR;
+
+/**
+ * Ozone range keys generator for performance test.
+ */
+@CommandLine.Command(name = "ork",
+ description = "write range keys with the help of the ozone clients.",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class RangeKeysGenerator extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RangeKeysGenerator.class);
+
+ @CommandLine.Option(names = {"-v", "--volume"},
+ description = "Name of the volume which contains the test data. " +
+ "Will be created if missing.",
+ defaultValue = "ockrwvolume")
+ private String volumeName;
+
+ @CommandLine.Option(names = {"-b", "--bucket"},
+ description = "Name of the bucket which contains the test data.",
+ defaultValue = "ockrwbucket")
+ private String bucketName;
+
+ @CommandLine.Option(names = {"-r", "--range-each-client-write"},
+ description = "Write range for each client.",
+ defaultValue = "0")
+ private int range;
+
+ @CommandLine.Option(names = {"-s", "--key-start-index"},
+ description = "Start index of key.",
+ defaultValue = "0")
+ private int startIndex;
+
+
+ @CommandLine.Option(names = {"-k", "--key-encode"},
+ description = "The algorithm to generate key names. " +
+ "Options are pureIndex, md5",
+ defaultValue = "md5")
+ private String encodeFormat;
+
+ @CommandLine.Option(names = {"-g", "--size"},
+ description = "Generated object size (in bytes) " +
+ "to be written.",
+ defaultValue = "1")
+ private int objectSizeInBytes;
+
+ @CommandLine.Option(
+ names = "--om-service-id",
+ description = "OM Service ID"
+ )
+ private String omServiceID = null;
+ private KeyGeneratorUtil kg;
+ private int clientCount;
+ private OzoneClient[] ozoneClients;
+ private byte[] keyContent;
+ private Timer timer;
+
+
+ @Override
+ public Void call() throws Exception {
+ init();
+ OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
+ clientCount = getThreadNo();
+ ozoneClients = new OzoneClient[clientCount];
+ for (int i = 0; i < clientCount; i++) {
+ ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration);
+ }
+
+ ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName);
+ if (objectSizeInBytes >= 0) {
+ keyContent = RandomUtils.nextBytes(objectSizeInBytes);
+ }
+ timer = getMetrics().timer("key-read-write");
+
+ kg = new KeyGeneratorUtil();
+ runTests(this::generateRangeKeys);
+ for (int i = 0; i < clientCount; i++) {
+ if (ozoneClients[i] != null) {
+ ozoneClients[i].close();
+ }
+ }
+
+ return null;
+ }
+
+ public void generateRangeKeys(long count) throws Exception {
+ int clientIndex = (int)(count % clientCount);
+ OzoneClient client = ozoneClients[clientIndex];
+ int start = startIndex + (int)count * range;
+ int end = start + range;
+
+ timer.time(() -> {
+ switch (encodeFormat) {
+ case PURE_INDEX:
+ loopRunner(kg.pureIndexKeyNameFunc(), client, start, end);
+ break;
+ case MD5:
+ loopRunner(kg.md5KeyNameFunc(), client, start, end);
+ break;
+ default:
+ loopRunner(kg.md5KeyNameFunc(), client, start, end);
+ break;
+ }
+ return null;
+ });
+ }
+
+
+ public void loopRunner(Function<Integer, String> keyNameGeneratorfunc,
+ OzoneClient client, int start, int end)
+ throws Exception {
+ String keyName;
+ for (int i = start; i < end + 1; i++) {
+ keyName = getPrefix() + FILE_DIR_SEPARATOR +
+ keyNameGeneratorfunc.apply(i);
+ try (OzoneOutputStream out = client.getProxy().
+ createKey(volumeName, bucketName, keyName,
+ objectSizeInBytes, null, new HashMap())) {
+ out.write(keyContent);
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org