You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dr...@apache.org on 2016/10/08 06:10:14 UTC
[18/50] [abbrv] hadoop git commit: YARN-5702. Refactor
TestPBImplRecords for reuse in other YARN modules. Contributed by Subru
Krishnan
YARN-5702. Refactor TestPBImplRecords for reuse in other YARN modules. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d65b9577
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d65b9577
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d65b9577
Branch: refs/heads/HADOOP-12756
Commit: d65b957776c4f055f82549a610fd1e8494580fe6
Parents: 8867762
Author: Chris Douglas <cd...@apache.org>
Authored: Wed Oct 5 12:16:49 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed Oct 5 12:16:49 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/api/BasePBImplRecordsTest.java | 264 +++++++++++++++++++
.../hadoop/yarn/api/TestPBImplRecords.java | 259 +-----------------
2 files changed, 271 insertions(+), 252 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d65b9577/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
new file mode 100644
index 0000000..21a737d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Assert;
+
+import java.lang.reflect.*;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+/**
+ * Generic helper class to validate protocol records.
+ */
+public class BasePBImplRecordsTest {
+ static final Log LOG = LogFactory.getLog(BasePBImplRecordsTest.class);
+
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected static HashMap<Type, Object> typeValueCache =
+ new HashMap<Type, Object>();
+ private static Random rand = new Random();
+ private static byte [] bytes = new byte[] {'1', '2', '3', '4'};
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static Object genTypeValue(Type type) {
+ Object ret = typeValueCache.get(type);
+ if (ret != null) {
+ return ret;
+ }
+ // only use positive primitive values
+ if (type.equals(boolean.class)) {
+ return rand.nextBoolean();
+ } else if (type.equals(byte.class)) {
+ return bytes[rand.nextInt(4)];
+ } else if (type.equals(int.class) || type.equals(Integer.class)) {
+ return rand.nextInt(1000000);
+ } else if (type.equals(long.class) || type.equals(Long.class)) {
+ return Long.valueOf(rand.nextInt(1000000));
+ } else if (type.equals(float.class)) {
+ return rand.nextFloat();
+ } else if (type.equals(double.class)) {
+ return rand.nextDouble();
+ } else if (type.equals(String.class)) {
+ return String.format("%c%c%c",
+ 'a' + rand.nextInt(26),
+ 'a' + rand.nextInt(26),
+ 'a' + rand.nextInt(26));
+ } else if (type instanceof Class) {
+ Class clazz = (Class)type;
+ if (clazz.isArray()) {
+ Class compClass = clazz.getComponentType();
+ if (compClass != null) {
+ ret = Array.newInstance(compClass, 2);
+ Array.set(ret, 0, genTypeValue(compClass));
+ Array.set(ret, 1, genTypeValue(compClass));
+ }
+ } else if (clazz.isEnum()) {
+ Object [] values = clazz.getEnumConstants();
+ ret = values[rand.nextInt(values.length)];
+ } else if (clazz.equals(ByteBuffer.class)) {
+ // return new ByteBuffer every time
+ // to prevent potential side effects
+ ByteBuffer buff = ByteBuffer.allocate(4);
+ rand.nextBytes(buff.array());
+ return buff;
+ }
+ } else if (type instanceof ParameterizedType) {
+ ParameterizedType pt = (ParameterizedType)type;
+ Type rawType = pt.getRawType();
+ Type [] params = pt.getActualTypeArguments();
+ // only support EnumSet<T>, List<T>, Set<T>, Map<K,V>
+ if (rawType.equals(EnumSet.class)) {
+ if (params[0] instanceof Class) {
+ Class c = (Class)(params[0]);
+ return EnumSet.allOf(c);
+ }
+ } if (rawType.equals(List.class)) {
+ ret = Lists.newArrayList(genTypeValue(params[0]));
+ } else if (rawType.equals(Set.class)) {
+ ret = Sets.newHashSet(genTypeValue(params[0]));
+ } else if (rawType.equals(Map.class)) {
+ Map<Object, Object> map = Maps.newHashMap();
+ map.put(genTypeValue(params[0]), genTypeValue(params[1]));
+ ret = map;
+ }
+ }
+ if (ret == null) {
+ throw new IllegalArgumentException("type " + type + " is not supported");
+ }
+ typeValueCache.put(type, ret);
+ return ret;
+ }
+
+ /**
+ * this method generate record instance by calling newIntance
+ * using reflection, add register the generated value to typeValueCache
+ */
+ @SuppressWarnings("rawtypes")
+ protected static Object generateByNewInstance(Class clazz) throws Exception {
+ Object ret = typeValueCache.get(clazz);
+ if (ret != null) {
+ return ret;
+ }
+ Method newInstance = null;
+ Type [] paramTypes = new Type[0];
+ // get newInstance method with most parameters
+ for (Method m : clazz.getMethods()) {
+ int mod = m.getModifiers();
+ if (m.getDeclaringClass().equals(clazz) &&
+ Modifier.isPublic(mod) &&
+ Modifier.isStatic(mod) &&
+ m.getName().equals("newInstance")) {
+ Type [] pts = m.getGenericParameterTypes();
+ if (newInstance == null
+ || (pts.length > paramTypes.length)) {
+ newInstance = m;
+ paramTypes = pts;
+ }
+ }
+ }
+ if (newInstance == null) {
+ throw new IllegalArgumentException("type " + clazz.getName() +
+ " does not have newInstance method");
+ }
+ Object [] args = new Object[paramTypes.length];
+ for (int i=0;i<args.length;i++) {
+ args[i] = genTypeValue(paramTypes[i]);
+ }
+ ret = newInstance.invoke(null, args);
+ typeValueCache.put(clazz, ret);
+ return ret;
+ }
+
+ private class GetSetPair {
+ public String propertyName;
+ public Method getMethod;
+ public Method setMethod;
+ public Type type;
+ public Object testValue;
+
+ @Override
+ public String toString() {
+ return String.format("{ name=%s, class=%s, value=%s }", propertyName,
+ type, testValue);
+ }
+ }
+
+ private <R> Map<String, GetSetPair> getGetSetPairs(Class<R> recordClass)
+ throws Exception {
+ Map<String, GetSetPair> ret = new HashMap<String, GetSetPair>();
+ Method [] methods = recordClass.getDeclaredMethods();
+ // get all get methods
+ for (int i = 0; i < methods.length; i++) {
+ Method m = methods[i];
+ int mod = m.getModifiers();
+ if (m.getDeclaringClass().equals(recordClass) &&
+ Modifier.isPublic(mod) &&
+ (!Modifier.isStatic(mod))) {
+ String name = m.getName();
+ if (name.equals("getProto")) {
+ continue;
+ }
+ if ((name.length() > 3) && name.startsWith("get") &&
+ (m.getParameterTypes().length == 0)) {
+ String propertyName = name.substring(3);
+ Type valueType = m.getGenericReturnType();
+ GetSetPair p = ret.get(propertyName);
+ if (p == null) {
+ p = new GetSetPair();
+ p.propertyName = propertyName;
+ p.type = valueType;
+ p.getMethod = m;
+ ret.put(propertyName, p);
+ } else {
+ Assert.fail("Multiple get method with same name: " + recordClass
+ + p.propertyName);
+ }
+ }
+ }
+ }
+ // match get methods with set methods
+ for (int i = 0; i < methods.length; i++) {
+ Method m = methods[i];
+ int mod = m.getModifiers();
+ if (m.getDeclaringClass().equals(recordClass) &&
+ Modifier.isPublic(mod) &&
+ (!Modifier.isStatic(mod))) {
+ String name = m.getName();
+ if (name.startsWith("set") && (m.getParameterTypes().length == 1)) {
+ String propertyName = name.substring(3);
+ Type valueType = m.getGenericParameterTypes()[0];
+ GetSetPair p = ret.get(propertyName);
+ if (p != null && p.type.equals(valueType)) {
+ p.setMethod = m;
+ }
+ }
+ }
+ }
+ // exclude incomplete get/set pair, and generate test value
+ Iterator<Map.Entry<String, GetSetPair>> itr = ret.entrySet().iterator();
+ while (itr.hasNext()) {
+ Map.Entry<String, GetSetPair> cur = itr.next();
+ GetSetPair gsp = cur.getValue();
+ if ((gsp.getMethod == null) ||
+ (gsp.setMethod == null)) {
+ LOG.info(String.format("Exclude protential property: %s\n", gsp.propertyName));
+ itr.remove();
+ } else {
+ LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type));
+ gsp.testValue = genTypeValue(gsp.type);
+ LOG.info(String.format(" testValue: %s\n", gsp.testValue));
+ }
+ }
+ return ret;
+ }
+
+ protected <R, P> void validatePBImplRecord(Class<R> recordClass,
+ Class<P> protoClass)
+ throws Exception {
+ LOG.info(String.format("Validate %s %s\n", recordClass.getName(),
+ protoClass.getName()));
+ Constructor<R> emptyConstructor = recordClass.getConstructor();
+ Constructor<R> pbConstructor = recordClass.getConstructor(protoClass);
+ Method getProto = recordClass.getDeclaredMethod("getProto");
+ Map<String, GetSetPair> getSetPairs = getGetSetPairs(recordClass);
+ R origRecord = emptyConstructor.newInstance();
+ for (GetSetPair gsp : getSetPairs.values()) {
+ gsp.setMethod.invoke(origRecord, gsp.testValue);
+ }
+ Object ret = getProto.invoke(origRecord);
+ Assert.assertNotNull(recordClass.getName() + "#getProto returns null", ret);
+ if (!(protoClass.isAssignableFrom(ret.getClass()))) {
+ Assert.fail("Illegal getProto method return type: " + ret.getClass());
+ }
+ R deserRecord = pbConstructor.newInstance(ret);
+ Assert.assertEquals("whole " + recordClass + " records should be equal",
+ origRecord, deserRecord);
+ for (GetSetPair gsp : getSetPairs.values()) {
+ Object origValue = gsp.getMethod.invoke(origRecord);
+ Object deserValue = gsp.getMethod.invoke(deserRecord);
+ Assert.assertEquals("property " + recordClass.getName() + "#"
+ + gsp.propertyName + " should be equal", origValue, deserValue);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d65b9577/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 5270486..4b71282 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -16,26 +16,8 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.api;
-import java.io.IOException;
-import java.lang.reflect.Array;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-import java.nio.ByteBuffer;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-
+import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang.math.LongRange;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
@@ -343,127 +325,12 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-public class TestPBImplRecords {
- static final Log LOG = LogFactory.getLog(TestPBImplRecords.class);
-
- private static HashMap<Type, Object> typeValueCache = new HashMap<Type, Object>();
- private static Random rand = new Random();
- private static byte [] bytes = new byte[] {'1', '2', '3', '4'};
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private static Object genTypeValue(Type type) {
- Object ret = typeValueCache.get(type);
- if (ret != null) {
- return ret;
- }
- // only use positive primitive values
- if (type.equals(boolean.class)) {
- return rand.nextBoolean();
- } else if (type.equals(byte.class)) {
- return bytes[rand.nextInt(4)];
- } else if (type.equals(int.class) || type.equals(Integer.class)) {
- return rand.nextInt(1000000);
- } else if (type.equals(long.class) || type.equals(Long.class)) {
- return Long.valueOf(rand.nextInt(1000000));
- } else if (type.equals(float.class)) {
- return rand.nextFloat();
- } else if (type.equals(double.class)) {
- return rand.nextDouble();
- } else if (type.equals(String.class)) {
- return String.format("%c%c%c",
- 'a' + rand.nextInt(26),
- 'a' + rand.nextInt(26),
- 'a' + rand.nextInt(26));
- } else if (type instanceof Class) {
- Class clazz = (Class)type;
- if (clazz.isArray()) {
- Class compClass = clazz.getComponentType();
- if (compClass != null) {
- ret = Array.newInstance(compClass, 2);
- Array.set(ret, 0, genTypeValue(compClass));
- Array.set(ret, 1, genTypeValue(compClass));
- }
- } else if (clazz.isEnum()) {
- Object [] values = clazz.getEnumConstants();
- ret = values[rand.nextInt(values.length)];
- } else if (clazz.equals(ByteBuffer.class)) {
- // return new ByteBuffer every time
- // to prevent potential side effects
- ByteBuffer buff = ByteBuffer.allocate(4);
- rand.nextBytes(buff.array());
- return buff;
- }
- } else if (type instanceof ParameterizedType) {
- ParameterizedType pt = (ParameterizedType)type;
- Type rawType = pt.getRawType();
- Type [] params = pt.getActualTypeArguments();
- // only support EnumSet<T>, List<T>, Set<T>, Map<K,V>
- if (rawType.equals(EnumSet.class)) {
- if (params[0] instanceof Class) {
- Class c = (Class)(params[0]);
- return EnumSet.allOf(c);
- }
- } if (rawType.equals(List.class)) {
- ret = Lists.newArrayList(genTypeValue(params[0]));
- } else if (rawType.equals(Set.class)) {
- ret = Sets.newHashSet(genTypeValue(params[0]));
- } else if (rawType.equals(Map.class)) {
- Map<Object, Object> map = Maps.newHashMap();
- map.put(genTypeValue(params[0]), genTypeValue(params[1]));
- ret = map;
- }
- }
- if (ret == null) {
- throw new IllegalArgumentException("type " + type + " is not supported");
- }
- typeValueCache.put(type, ret);
- return ret;
- }
-
- /**
- * this method generate record instance by calling newIntance
- * using reflection, add register the generated value to typeValueCache
- */
- @SuppressWarnings("rawtypes")
- private static Object generateByNewInstance(Class clazz) throws Exception {
- Object ret = typeValueCache.get(clazz);
- if (ret != null) {
- return ret;
- }
- Method newInstance = null;
- Type [] paramTypes = new Type[0];
- // get newInstance method with most parameters
- for (Method m : clazz.getMethods()) {
- int mod = m.getModifiers();
- if (m.getDeclaringClass().equals(clazz) &&
- Modifier.isPublic(mod) &&
- Modifier.isStatic(mod) &&
- m.getName().equals("newInstance")) {
- Type [] pts = m.getGenericParameterTypes();
- if (newInstance == null
- || (pts.length > paramTypes.length)) {
- newInstance = m;
- paramTypes = pts;
- }
- }
- }
- if (newInstance == null) {
- throw new IllegalArgumentException("type " + clazz.getName() +
- " does not have newInstance method");
- }
- Object [] args = new Object[paramTypes.length];
- for (int i=0;i<args.length;i++) {
- args[i] = genTypeValue(paramTypes[i]);
- }
- ret = newInstance.invoke(null, args);
- typeValueCache.put(clazz, ret);
- return ret;
- }
+import java.io.IOException;
+
+/**
+ * Test class for YARN API protocol records.
+ */
+public class TestPBImplRecords extends BasePBImplRecordsTest {
@BeforeClass
public static void setup() throws Exception {
@@ -532,118 +399,6 @@ public class TestPBImplRecords {
generateByNewInstance(CommitResponse.class);
}
- private class GetSetPair {
- public String propertyName;
- public Method getMethod;
- public Method setMethod;
- public Type type;
- public Object testValue;
-
- @Override
- public String toString() {
- return String.format("{ name=%s, class=%s, value=%s }", propertyName,
- type, testValue);
- }
- }
-
- private <R> Map<String, GetSetPair> getGetSetPairs(Class<R> recordClass)
- throws Exception {
- Map<String, GetSetPair> ret = new HashMap<String, GetSetPair>();
- Method [] methods = recordClass.getDeclaredMethods();
- // get all get methods
- for (int i = 0; i < methods.length; i++) {
- Method m = methods[i];
- int mod = m.getModifiers();
- if (m.getDeclaringClass().equals(recordClass) &&
- Modifier.isPublic(mod) &&
- (!Modifier.isStatic(mod))) {
- String name = m.getName();
- if (name.equals("getProto")) {
- continue;
- }
- if ((name.length() > 3) && name.startsWith("get") &&
- (m.getParameterTypes().length == 0)) {
- String propertyName = name.substring(3);
- Type valueType = m.getGenericReturnType();
- GetSetPair p = ret.get(propertyName);
- if (p == null) {
- p = new GetSetPair();
- p.propertyName = propertyName;
- p.type = valueType;
- p.getMethod = m;
- ret.put(propertyName, p);
- } else {
- Assert.fail("Multiple get method with same name: " + recordClass
- + p.propertyName);
- }
- }
- }
- }
- // match get methods with set methods
- for (int i = 0; i < methods.length; i++) {
- Method m = methods[i];
- int mod = m.getModifiers();
- if (m.getDeclaringClass().equals(recordClass) &&
- Modifier.isPublic(mod) &&
- (!Modifier.isStatic(mod))) {
- String name = m.getName();
- if (name.startsWith("set") && (m.getParameterTypes().length == 1)) {
- String propertyName = name.substring(3);
- Type valueType = m.getGenericParameterTypes()[0];
- GetSetPair p = ret.get(propertyName);
- if (p != null && p.type.equals(valueType)) {
- p.setMethod = m;
- }
- }
- }
- }
- // exclude incomplete get/set pair, and generate test value
- Iterator<Entry<String, GetSetPair>> itr = ret.entrySet().iterator();
- while (itr.hasNext()) {
- Entry<String, GetSetPair> cur = itr.next();
- GetSetPair gsp = cur.getValue();
- if ((gsp.getMethod == null) ||
- (gsp.setMethod == null)) {
- LOG.info(String.format("Exclude protential property: %s\n", gsp.propertyName));
- itr.remove();
- } else {
- LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type));
- gsp.testValue = genTypeValue(gsp.type);
- LOG.info(String.format(" testValue: %s\n", gsp.testValue));
- }
- }
- return ret;
- }
-
- private <R, P> void validatePBImplRecord(Class<R> recordClass,
- Class<P> protoClass)
- throws Exception {
- LOG.info(String.format("Validate %s %s\n", recordClass.getName(),
- protoClass.getName()));
- Constructor<R> emptyConstructor = recordClass.getConstructor();
- Constructor<R> pbConstructor = recordClass.getConstructor(protoClass);
- Method getProto = recordClass.getDeclaredMethod("getProto");
- Map<String, GetSetPair> getSetPairs = getGetSetPairs(recordClass);
- R origRecord = emptyConstructor.newInstance();
- for (GetSetPair gsp : getSetPairs.values()) {
- gsp.setMethod.invoke(origRecord, gsp.testValue);
- }
- Object ret = getProto.invoke(origRecord);
- Assert.assertNotNull(recordClass.getName() + "#getProto returns null", ret);
- if (!(protoClass.isAssignableFrom(ret.getClass()))) {
- Assert.fail("Illegal getProto method return type: " + ret.getClass());
- }
- R deserRecord = pbConstructor.newInstance(ret);
- Assert.assertEquals("whole " + recordClass + " records should be equal",
- origRecord, deserRecord);
- for (GetSetPair gsp : getSetPairs.values()) {
- Object origValue = gsp.getMethod.invoke(origRecord);
- Object deserValue = gsp.getMethod.invoke(deserRecord);
- Assert.assertEquals("property " + recordClass.getName() + "#"
- + gsp.propertyName + " should be equal", origValue, deserValue);
- }
- }
-
@Test
public void testAllocateRequestPBImpl() throws Exception {
validatePBImplRecord(AllocateRequestPBImpl.class, AllocateRequestProto.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org