You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/10/09 11:27:09 UTC
[hadoop] branch trunk updated: YARN-9128. Use SerializationUtils
from apache commons to serialize / deserialize ResourceMappings.
Contributed by Zoltan Siegl
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6f1ab95 YARN-9128. Use SerializationUtils from apache commons to serialize / deserialize ResourceMappings. Contributed by Zoltan Siegl
6f1ab95 is described below
commit 6f1ab9516869dbfdee034f4a17bccc379a39fc31
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Oct 9 13:26:26 2019 +0200
YARN-9128. Use SerializationUtils from apache commons to serialize / deserialize ResourceMappings. Contributed by Zoltan Siegl
---
.../container/ResourceMappings.java | 32 ++----
.../container/TestResourceMappings.java | 118 +++++++++++++++++++++
2 files changed, 128 insertions(+), 22 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
index d673341..c1c3b5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
@@ -18,11 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import org.apache.commons.lang3.SerializationException;
+import org.apache.commons.lang3.SerializationUtils;
+
import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
@@ -30,8 +29,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.IOUtils;
-
/**
* This class is used to store assigned resource to a single container by
* resource types.
@@ -91,16 +88,11 @@ public class ResourceMappings {
@SuppressWarnings("unchecked")
public static AssignedResources fromBytes(byte[] bytes)
throws IOException {
- ObjectInputStream ois = null;
- List<Serializable> resources;
+ final List<Serializable> resources;
try {
- ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
- ois = new ObjectInputStream(bis);
- resources = (List<Serializable>) ois.readObject();
- } catch (ClassNotFoundException e) {
+ resources = SerializationUtils.deserialize(bytes);
+ } catch (SerializationException e) {
throw new IOException(e);
- } finally {
- IOUtils.closeQuietly(ois);
}
AssignedResources ar = new AssignedResources();
ar.updateAssignedResources(resources);
@@ -108,15 +100,11 @@ public class ResourceMappings {
}
public byte[] toBytes() throws IOException {
- ObjectOutputStream oos = null;
- byte[] bytes;
+ final byte[] bytes;
try {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- oos = new ObjectOutputStream(bos);
- oos.writeObject(resources);
- bytes = bos.toByteArray();
- } finally {
- IOUtils.closeQuietly(oos);
+ bytes = SerializationUtils.serialize((Serializable) resources);
+ } catch (SerializationException e) {
+ throw new IOException(e);
}
return bytes;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestResourceMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestResourceMappings.java
new file mode 100644
index 0000000..561ce0c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestResourceMappings.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.List;
+
+public class TestResourceMappings {
+
+ private static final ResourceMappings.AssignedResources testResources =
+ new ResourceMappings.AssignedResources();
+
+ @BeforeClass
+ public static void setup() {
+ testResources.updateAssignedResources(ImmutableList.of(
+ Device.Builder.newInstance()
+ .setId(0)
+ .setDevPath("/dev/hdwA0")
+ .setMajorNumber(256)
+ .setMinorNumber(0)
+ .setBusID("0000:80:00.0")
+ .setHealthy(true)
+ .build(),
+ Device.Builder.newInstance()
+ .setId(1)
+ .setDevPath("/dev/hdwA1")
+ .setMajorNumber(256)
+ .setMinorNumber(0)
+ .setBusID("0000:80:00.1")
+ .setHealthy(true)
+ .build()
+ ));
+ }
+
+ @Test
+ public void testSerializeAssignedResourcesWithSerializationUtils() {
+ try {
+ byte[] serializedString = testResources.toBytes();
+
+ ResourceMappings.AssignedResources deserialized =
+ ResourceMappings.AssignedResources.fromBytes(serializedString);
+
+ Assert.assertEquals(testResources.getAssignedResources(),
+ deserialized.getAssignedResources());
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail(String.format("Serialization of test AssignedResources " +
+ "failed with %s", e.getMessage()));
+ }
+ }
+
+ @Test
+ public void testAssignedResourcesCanDeserializePreviouslySerializedValues() {
+ try {
+ byte[] serializedString = toBytes(testResources.getAssignedResources());
+
+ ResourceMappings.AssignedResources deserialized =
+ ResourceMappings.AssignedResources.fromBytes(serializedString);
+
+ Assert.assertEquals(testResources.getAssignedResources(),
+ deserialized.getAssignedResources());
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail(String.format("Deserialization of test AssignedResources " +
+ "failed with %s", e.getMessage()));
+ }
+ }
+
+ /**
+ * This was the legacy way to serialize resources. This is here for
+ * backward compatibility to ensure that after YARN-9128 we can still
+ * deserialize previously serialized resources.
+ *
+ * @param resources the list of resources
+ * @return byte array representation of the resource
+ * @throws IOException
+ */
+ private byte[] toBytes(List<Serializable> resources) throws IOException {
+ ObjectOutputStream oos = null;
+ byte[] bytes;
+ try {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ oos = new ObjectOutputStream(bos);
+ oos.writeObject(resources);
+ bytes = bos.toByteArray();
+ } finally {
+ IOUtils.closeQuietly(oos);
+ }
+ return bytes;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org