You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/10/20 02:27:15 UTC
[ozone] branch master updated: Revert "HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements (#3754)"
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3fd7cd0896 Revert "HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements (#3754)"
3fd7cd0896 is described below
commit 3fd7cd08962a4eac375e9025c8a90eeb281feb00
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Thu Oct 20 04:26:50 2022 +0200
Revert "HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements (#3754)"
This reverts commit e45f9b8333a0ccd605bdc3d18aa36282d4ba5859.
---
.../src/main/smoketest/freon/read-write-key.robot | 53 -----
.../java/org/apache/hadoop/ozone/freon/Freon.java | 2 -
.../hadoop/ozone/freon/KeyGeneratorUtil.java | 50 -----
.../ozone/freon/OzoneClientKeyReadWriteOps.java | 243 ---------------------
.../hadoop/ozone/freon/RangeKeysGenerator.java | 176 ---------------
5 files changed, 524 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
deleted file mode 100644
index 25a3c4b9e9..0000000000
--- a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Test freon read/write key commands
-Resource ../ozone-lib/freon.robot
-Test Timeout 5 minutes
-
-*** Variables ***
-${PREFIX} ${EMPTY}
-
-
-*** Test Cases ***
-Pre-generate 100 keys of size 1 byte each to Ozone
- ${result} = Execute ozone freon ork -n 1 -t 10 -r 100 --size 1 -v voltest -b buckettest -p performanceTest
-
-Read 10 keys from pre-generated keys
- ${keysCount} = BuiltIn.Set Variable 10
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -r 100 -v voltest -b buckettest -p performanceTest
- Should contain ${result} Successful executions: ${keysCount}
-
-Read 10 keys' metadata from pre-generated keys
- ${keysCount} = BuiltIn.Set Variable 10
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -m -r 100 -v voltest -b buckettest -p performanceTest
- Should contain ${result} Successful executions: ${keysCount}
-
-Write 10 keys of size 1 byte each from key index 0 to 99
- ${keysCount} = BuiltIn.Set Variable 10
- ${size} = BuiltIn.Set Variable 1
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 0 --size ${size} -r 100 -v voltest -b buckettest -p performanceTest2
- Should contain ${result} Successful executions: ${keysCount}
- ${keyName} = Execute echo -n '1' | md5sum | head -c 7
- ${result} = Execute ozone sh key info /voltest/buckettest/performanceTest2/${keyName}
- Should contain ${result} \"dataSize\" : 1
-
-
-Run 90 % of read-key tasks and 10 % of write-key tasks for 10 keys from pre-generated keys
- ${keysCount} = BuiltIn.Set Variable 10
- ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 90 -r 100 -v voltest -b buckettest -p performanceTest
- Should contain ${result} Successful executions: ${keysCount}
-
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index dfb52ca9ad..b6bb6055ee 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -68,8 +68,6 @@ import picocli.CommandLine.Option;
OmBucketReadWriteFileOps.class,
OmBucketReadWriteKeyOps.class,
OmRPCLoadGenerator.class,
- OzoneClientKeyReadWriteOps.class,
- RangeKeysGenerator.class
},
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true)
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
deleted file mode 100644
index 9773caa440..0000000000
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.freon;
-
-import org.apache.commons.codec.digest.DigestUtils;
-
-import java.util.function.Function;
-
-/**
- * Utility class to generate key name from a given key index.
- */
-public class KeyGeneratorUtil {
- public static final String PURE_INDEX = "pureIndex";
- public static final String MD5 = "md5";
- public static final String FILE_DIR_SEPARATOR = "/";
-
- public String generatePureIndexKeyName(int number) {
- return String.valueOf(number);
- }
- public Function<Integer, String> pureIndexKeyNameFunc() {
- return number -> String.valueOf(number);
- }
-
- public String generateMd5KeyName(int number) {
- String encodedStr = DigestUtils.md5Hex(String.valueOf(number));
- return encodedStr.substring(0, 7);
- }
-
- public Function<Integer, String> md5KeyNameFunc() {
- return number -> DigestUtils.md5Hex(String.valueOf(number)).substring(0, 7);
- }
-
-}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
deleted file mode 100644
index c4374f4ead..0000000000
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.freon;
-
-
-import com.codahale.metrics.Timer;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneKeyDetails;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import picocli.CommandLine;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.HashMap;
-
-import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR;
-
-/**
- * Ozone key generator/reader for performance test.
- */
-
-@CommandLine.Command(name = "ockrw",
- aliases = "ozone-client-key-read-write-ops",
- description = "Generate keys with a fixed name and ranges that can"
- + " be written and read as sub-ranges from multiple clients.",
- versionProvider = HddsVersionProvider.class,
- mixinStandardHelpOptions = true,
- showDefaultValues = true)
-public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator
- implements Callable<Void> {
-
- @CommandLine.Option(names = {"-v", "--volume"},
- description = "Name of the volume which contains the test data. " +
- "Will be created if missing.",
- defaultValue = "ockrwvolume")
- private String volumeName;
-
- @CommandLine.Option(names = {"-b", "--bucket"},
- description = "Name of the bucket which contains the test data.",
- defaultValue = "ockrwbucket")
- private String bucketName;
-
- @CommandLine.Option(names = {"-m", "--read-metadata-only"},
- description = "If only read key's metadata.",
- defaultValue = "false")
- private boolean readMetadataOnly;
-
- @CommandLine.Option(names = {"-s", "--start-index"},
- description = "Start index of keys of read/write operation." +
- "This can allow adding keys incrementally or parallel from multiple"
- + " clients. Example: Write keys 0-1000000 followed by "
- + "keys 1000001-2000000.",
- defaultValue = "0")
- private int startIndex;
-
- @CommandLine.Option(names = {"-r", "--range"},
- description = "Range of read/write operations. This in co-ordination"
- + " with --start-index can specify the range to read. "
- + "Example: Read from --start-index 1000 and read --range 1000 keys.",
- defaultValue = "1")
- private int range;
-
- @CommandLine.Option(names = {"--size"},
- description = "Object size (in bytes) " +
- "to read/write. If user sets a read size which is larger"
- + " than the key size, it only reads bytes up to key size.",
- defaultValue = "1")
- private int objectSizeInBytes;
-
- @CommandLine.Option(names = {"--contiguous"},
- description = "By default, the keys are randomized lexically"
- + " by calculating the md5 of the key name. If this option is set,"
- + " the keys are written lexically contiguously.",
- defaultValue = "false")
- private boolean keySorted;
-
- @CommandLine.Option(names = {"--percentage-read"},
- description = "Percentage of read tasks in mix workload."
- + " The remainder of the percentage will writes to keys."
- + " Example --percentage-read 90 will result in 10% writes.",
- defaultValue = "100")
- private int percentageRead;
-
- @CommandLine.Option(
- names = "--om-service-id",
- description = "OM Service ID"
- )
- private String omServiceID = null;
-
- private Timer timer;
-
- private OzoneClient[] ozoneClients;
-
- private int clientCount;
-
- private byte[] keyContent;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class);
-
- /**
- * Task type of read task, or write task.
- */
- public enum TaskType {
- READ_TASK,
- WRITE_TASK
- }
- private KeyGeneratorUtil kg;
-
-
- @Override
- public Void call() throws Exception {
- init();
- OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
- clientCount = getThreadNo();
- ozoneClients = new OzoneClient[clientCount];
- for (int i = 0; i < clientCount; i++) {
- ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration);
- }
-
- ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName);
-
- timer = getMetrics().timer("key-read-write");
- if (objectSizeInBytes >= 0) {
- keyContent = RandomUtils.nextBytes(objectSizeInBytes);
- }
- if (kg == null) {
- kg = new KeyGeneratorUtil();
- }
- runTests(this::readWriteKeys);
-
- for (int i = 0; i < clientCount; i++) {
- if (ozoneClients[i] != null) {
- ozoneClients[i].close();
- }
- }
- return null;
- }
-
- public void readWriteKeys(long counter) throws RuntimeException, IOException {
- int clientIndex = (int)((counter) % clientCount);
- TaskType taskType = decideReadOrWriteTask();
- String keyName = getKeyName();
-
- timer.time(() -> {
- try {
- switch (taskType) {
- case READ_TASK:
- processReadTasks(keyName, ozoneClients[clientIndex]);
- break;
- case WRITE_TASK:
- processWriteTasks(keyName, ozoneClients[clientIndex]);
- break;
- default:
- break;
- }
- } catch (RuntimeException ex) {
- LOG.error(ex.getMessage());
- throw ex;
- } catch (IOException ex) {
- LOG.error(ex.getMessage());
- throw new RuntimeException(ex.getMessage());
- }
-
- });
- }
-
- public void processReadTasks(String keyName, OzoneClient client)
- throws RuntimeException, IOException {
- OzoneKeyDetails keyDetails = client.getProxy().getKeyDetails(volumeName, bucketName, keyName);
- if (!readMetadataOnly) {
- byte[] data = new byte[objectSizeInBytes];
- try (OzoneInputStream introStream = keyDetails.getContent()) {
- introStream.read(data);
- } catch (Exception ex) {
- throw ex;
- }
- }
- }
-
- public void processWriteTasks(String keyName, OzoneClient ozoneClient)
- throws RuntimeException, IOException {
- try (OzoneOutputStream out =
- ozoneClient.getProxy().createKey(volumeName, bucketName, keyName, objectSizeInBytes, null, new HashMap())) {
- out.write(keyContent);
- } catch (Exception ex) {
- throw ex;
- }
- }
-
- public TaskType decideReadOrWriteTask() {
- if (percentageRead == 100) {
- return TaskType.READ_TASK;
- } else if (percentageRead == 0) {
- return TaskType.WRITE_TASK;
- }
- //mix workload
- int tmp = ThreadLocalRandom.current().nextInt(1,101);
- if (tmp <= percentageRead) {
- return TaskType.READ_TASK;
- } else {
- return TaskType.WRITE_TASK;
- }
- }
-
- public String getKeyName() {
- StringBuilder keyNameSb = new StringBuilder();
- int randomIdxWithinRange = ThreadLocalRandom.current().
- nextInt(startIndex, startIndex + range);
-
- if (keySorted) {
- keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
- append(randomIdxWithinRange);
- } else {
- keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
- append(kg.generateMd5KeyName(randomIdxWithinRange));
- }
- return keyNameSb.toString();
- }
-
-}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java
deleted file mode 100644
index bfe23f1781..0000000000
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.freon;
-
-import com.codahale.metrics.Timer;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import picocli.CommandLine;
-
-import java.util.concurrent.Callable;
-import java.util.function.Function;
-import java.util.HashMap;
-
-import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.PURE_INDEX;
-import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.MD5;
-import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR;
-
-// import com.fasterxml.jackson.databind.ObjectMapper;
-// import java.util.HashMap;
-// import java.util.Map;
-// import java.io.File;
-
-/**
- * Ozone range keys generator for performance test.
- */
-@CommandLine.Command(name = "ork",
- description = "write range keys with the help of the ozone clients.",
- versionProvider = HddsVersionProvider.class,
- mixinStandardHelpOptions = true,
- showDefaultValues = true)
-public class RangeKeysGenerator extends BaseFreonGenerator
- implements Callable<Void> {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RangeKeysGenerator.class);
-
- @CommandLine.Option(names = {"-v", "--volume"},
- description = "Name of the volume which contains the test data. " +
- "Will be created if missing.",
- defaultValue = "ockrwvolume")
- private String volumeName;
-
- @CommandLine.Option(names = {"-b", "--bucket"},
- description = "Name of the bucket which contains the test data.",
- defaultValue = "ockrwbucket")
- private String bucketName;
-
- @CommandLine.Option(names = {"-r", "--range-each-client-write"},
- description = "Write range for each client.",
- defaultValue = "0")
- private int range;
-
- @CommandLine.Option(names = {"-s", "--key-start-index"},
- description = "Start index of key.",
- defaultValue = "0")
- private int startIndex;
-
-
- @CommandLine.Option(names = {"-k", "--key-encode"},
- description = "The algorithm to generate key names. " +
- "Options are pureIndex, md5",
- defaultValue = "md5")
- private String encodeFormat;
-
- @CommandLine.Option(names = {"-g", "--size"},
- description = "Generated object size (in bytes) " +
- "to be written.",
- defaultValue = "1")
- private int objectSizeInBytes;
-
- private int clientCount;
-
- @CommandLine.Option(
- names = "--debug",
- description = "Enable debugging message.",
- defaultValue = "false"
- )
- private boolean debug;
-
-
- @CommandLine.Option(
- names = "--om-service-id",
- description = "OM Service ID"
- )
- private String omServiceID = null;
- private KeyGeneratorUtil kg;
- private OzoneClient[] ozoneClients;
- private byte[] keyContent;
- private Timer timer;
-
-
- @Override
- public Void call() throws Exception {
- init();
- OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
- clientCount = getThreadNo();
- ozoneClients = new OzoneClient[clientCount];
- for (int i = 0; i < clientCount; i++) {
- ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration);
- }
-
- ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName);
- if (objectSizeInBytes >= 0) {
- keyContent = RandomUtils.nextBytes(objectSizeInBytes);
- }
- timer = getMetrics().timer("key-read-write");
-
- kg = new KeyGeneratorUtil();
- runTests(this::generateRangeKeys);
- for (int i = 0; i < clientCount; i++) {
- if (ozoneClients[i] != null) {
- ozoneClients[i].close();
- }
- }
-
- return null;
- }
-
- public void generateRangeKeys(long count) throws Exception {
- int clientIndex = (int)(count % clientCount);
- OzoneClient client = ozoneClients[clientIndex];
- int start = startIndex + (int)count * range;
- int end = start + range;
-
- timer.time(() -> {
- switch (encodeFormat) {
- case PURE_INDEX:
- loopRunner(kg.pureIndexKeyNameFunc(), client, start, end);
- break;
- case MD5:
- loopRunner(kg.md5KeyNameFunc(), client, start, end);
- break;
- default:
- loopRunner(kg.md5KeyNameFunc(), client, start, end);
- break;
- }
- return null;
- });
- }
-
-
- public void loopRunner(Function<Integer, String> fn, OzoneClient client,
- int start, int end) throws Exception {
- String keyName;
- for (int i = start; i < end + 1; i++) {
- keyName = getPrefix() + FILE_DIR_SEPARATOR + fn.apply(i);
- try (OzoneOutputStream out = client.getProxy().
- createKey(volumeName, bucketName, keyName, objectSizeInBytes, null, new HashMap())) {
- out.write(keyContent);
- }
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org