You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by te...@apache.org on 2019/02/27 18:29:27 UTC
[hadoop] branch trunk updated: YARN-9318.
Resources#multiplyAndRoundUp does not consider Resource Types (Contributed
by Szilard Nemeth via Daniel Templeton)
This is an automated email from the ASF dual-hosted git repository.
templedf pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 29e27fa YARN-9318. Resources#multiplyAndRoundUp does not consider Resource Types (Contributed by Szilard Nemeth via Daniel Templeton)
29e27fa is described below
commit 29e27faf96b6e5b0ff3bca98f43a662d9d689f0a
Author: Szilard Nemeth <sn...@cloudera.com>
AuthorDate: Wed Feb 27 10:04:55 2019 -0800
YARN-9318. Resources#multiplyAndRoundUp does not consider Resource Types
(Contributed by Szilard Nemeth via Daniel Templeton)
Change-Id: Ia45f528574c2b054f6f764d1d140e592bdb7e217
---
.../hadoop/yarn/util/resource/Resources.java | 68 ++++++++++++--------
.../hadoop/yarn/util/resource/TestResources.java | 72 ++++++++++++++--------
2 files changed, 89 insertions(+), 51 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 4764147..40d8d38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
@Unstable
public class Resources {
+ private enum RoundingDirection { UP, DOWN }
+
private static final Log LOG =
LogFactory.getLog(Resources.class);
@@ -305,17 +307,7 @@ public class Resources {
}
public static Resource multiplyTo(Resource lhs, double by) {
- int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
- for (int i = 0; i < maxLength; i++) {
- try {
- ResourceInformation lhsValue = lhs.getResourceInformation(i);
- lhs.setResourceValue(i, (long) (lhsValue.getValue() * by));
- } catch (ResourceNotFoundException ye) {
- LOG.warn("Resource is missing:" + ye.getMessage());
- continue;
- }
- }
- return lhs;
+ return multiplyAndRound(lhs, by, RoundingDirection.DOWN);
}
public static Resource multiply(Resource lhs, double by) {
@@ -338,7 +330,6 @@ public class Resources {
lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs);
} catch (ResourceNotFoundException ye) {
LOG.warn("Resource is missing:" + ye.getMessage());
- continue;
}
}
return lhs;
@@ -358,29 +349,58 @@ public class Resources {
ResourceCalculator calculator,Resource lhs, double by, Resource factor) {
return calculator.multiplyAndNormalizeDown(lhs, by, factor);
}
-
+
+ /**
+ * Multiply {@code lhs} by {@code by}, and set the result rounded down into a
+ * cloned version of {@code lhs} Resource object.
+ * @param lhs Resource object
+ * @param by Multiply values by this value
+ * @return A cloned version of {@code lhs} with updated values
+ */
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
- Resource out = clone(lhs);
+ return multiplyAndRound(clone(lhs), by, RoundingDirection.DOWN);
+ }
+
+ /**
+ * Multiply {@code lhs} by {@code by}, and set the result rounded up into a
+ * cloned version of {@code lhs} Resource object.
+ * @param lhs Resource object
+ * @param by Multiply values by this value
+ * @return A cloned version of {@code lhs} with updated values
+ */
+ public static Resource multiplyAndRoundUp(Resource lhs, double by) {
+ return multiplyAndRound(clone(lhs), by, RoundingDirection.UP);
+ }
+
+ /**
+ * Multiply {@code lhs} by {@code by}, and set the result according to
+ * the rounding direction to {@code lhs}
+ * without creating any new {@link Resource} object.
+ * @param lhs Resource object
+ * @param by Multiply values by this value
+ * @return Returns {@code lhs} itself (without cloning) with updated values
+ */
+ private static Resource multiplyAndRound(Resource lhs, double by,
+ RoundingDirection roundingDirection) {
int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation lhsValue = lhs.getResourceInformation(i);
- out.setResourceValue(i, (long) (lhsValue.getValue() * by));
+
+ final long value;
+ if (roundingDirection == RoundingDirection.DOWN) {
+ value = (long) (lhsValue.getValue() * by);
+ } else {
+ value = (long) Math.ceil(lhsValue.getValue() * by);
+ }
+ lhs.setResourceValue(i, value);
} catch (ResourceNotFoundException ye) {
LOG.warn("Resource is missing:" + ye.getMessage());
- continue;
}
}
- return out;
+ return lhs;
}
- public static Resource multiplyAndRoundUp(Resource lhs, double by) {
- Resource out = clone(lhs);
- out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by));
- out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by));
- return out;
- }
-
public static Resource normalize(
ResourceCalculator calculator, Resource lhs, Resource min,
Resource max, Resource increment) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index ecd940e..307e0d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -31,6 +31,7 @@ import java.io.File;
import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMin;
import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMax;
import static org.apache.hadoop.yarn.util.resource.Resources.add;
+import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndRoundUp;
import static org.apache.hadoop.yarn.util.resource.Resources.subtract;
import static org.apache.hadoop.yarn.util.resource.Resources.multiply;
import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndAddTo;
@@ -41,6 +42,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class TestResources {
+ private static final String INVALID_RESOURCE_MSG = "Invalid resource value";
static class ExtendedResources extends Resources {
public static Resource unbounded() {
@@ -116,27 +118,6 @@ public class TestResources {
assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
}
- @Test(timeout=10000)
- public void testMultipleRoundUp() {
- final double by = 0.5;
- final String memoryErrorMsg = "Invalid memory size.";
- final String vcoreErrorMsg = "Invalid virtual core number.";
- Resource resource = Resources.createResource(1, 1);
- Resource result = Resources.multiplyAndRoundUp(resource, by);
- assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
- assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
-
- resource = Resources.createResource(2, 2);
- result = Resources.multiplyAndRoundUp(resource, by);
- assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
- assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
-
- resource = Resources.createResource(0, 0);
- result = Resources.multiplyAndRoundUp(resource, by);
- assertEquals(memoryErrorMsg, result.getMemorySize(), 0);
- assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0);
- }
-
@Test(timeout = 1000)
public void testFitsIn() {
assertTrue(fitsIn(createResource(1, 1), createResource(2, 2)));
@@ -228,19 +209,56 @@ public class TestResources {
assertEquals(createResource(4, 4, 6), multiply(createResource(2, 2, 3), 2));
}
+ @Test(timeout=10000)
+ public void testMultiplyRoundUp() {
+ final double by = 0.5;
+ final String memoryErrorMsg = "Invalid memory size.";
+ final String vcoreErrorMsg = "Invalid virtual core number.";
+ Resource resource = Resources.createResource(1, 1);
+ Resource result = Resources.multiplyAndRoundUp(resource, by);
+ assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
+ assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
+
+ resource = Resources.createResource(2, 2);
+ result = Resources.multiplyAndRoundUp(resource, by);
+ assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
+ assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
+
+ resource = Resources.createResource(0, 0);
+ result = Resources.multiplyAndRoundUp(resource, by);
+ assertEquals(memoryErrorMsg, result.getMemorySize(), 0);
+ assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0);
+ }
+
+ @Test
+ public void testMultiplyAndRoundUpCustomResources() {
+ assertEquals(INVALID_RESOURCE_MSG, createResource(5, 2, 8),
+ multiplyAndRoundUp(createResource(3, 1, 5), 1.5));
+ assertEquals(INVALID_RESOURCE_MSG, createResource(5, 2, 0),
+ multiplyAndRoundUp(createResource(3, 1, 0), 1.5));
+ assertEquals(INVALID_RESOURCE_MSG, createResource(5, 5, 0),
+ multiplyAndRoundUp(createResource(3, 3, 0), 1.5));
+ assertEquals(INVALID_RESOURCE_MSG, createResource(8, 3, 13),
+ multiplyAndRoundUp(createResource(3, 1, 5), 2.5));
+ assertEquals(INVALID_RESOURCE_MSG, createResource(8, 3, 0),
+ multiplyAndRoundUp(createResource(3, 1, 0), 2.5));
+ assertEquals(INVALID_RESOURCE_MSG, createResource(8, 8, 0),
+ multiplyAndRoundUp(createResource(3, 3, 0), 2.5));
+ }
+
@Test
public void testMultiplyAndRoundDown() {
- assertEquals(createResource(4, 1),
+ assertEquals(INVALID_RESOURCE_MSG, createResource(4, 1),
multiplyAndRoundDown(createResource(3, 1), 1.5));
- assertEquals(createResource(4, 1, 0),
+ assertEquals(INVALID_RESOURCE_MSG, createResource(4, 1, 0),
multiplyAndRoundDown(createResource(3, 1), 1.5));
- assertEquals(createResource(1, 4),
+ assertEquals(INVALID_RESOURCE_MSG, createResource(1, 4),
multiplyAndRoundDown(createResource(1, 3), 1.5));
- assertEquals(createResource(1, 4, 0),
+ assertEquals(INVALID_RESOURCE_MSG, createResource(1, 4, 0),
multiplyAndRoundDown(createResource(1, 3), 1.5));
- assertEquals(createResource(7, 7, 0),
+ assertEquals(INVALID_RESOURCE_MSG, createResource(7, 7, 0),
multiplyAndRoundDown(createResource(3, 3, 0), 2.5));
- assertEquals(createResource(2, 2, 7),
+ assertEquals(INVALID_RESOURCE_MSG, createResource(2, 2, 7),
multiplyAndRoundDown(createResource(1, 1, 3), 2.5));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org