You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2026/09/09 21:53:37 UTC
(hadoop) branch branch-3.4 updated: HADOOP-19930. ObjectWritable to validate class on load (#8640) (#8728)
This is an automated email from the ASF dual-hosted git repository.
steveloughran pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 536b278f3ea HADOOP-19930. ObjectWritable to validate class on load (#8640) (#8728)
536b278f3ea is described below
commit 536b278f3eae32707dae7f9d73b55c254d58b535
Author: Steve Loughran <st...@apache.org>
AuthorDate: Wed Sep 9 22:53:30 2026 +0100
HADOOP-19930. ObjectWritable to validate class on load (#8640) (#8728)
New reflection validation in ReflectionUtils
-Pulls up class type checking to before actual class load; fails fast
-Add check of array length.
-Limit recursive depth to 100.
This is used in SequenceFile, InputSplits and elsewhere that
Writable subclasses instantiate classes.
Contributed by Steve Loughran
Assisted-by: Claude.ai
"This is Ripley, last survivor of the Nostromo, signing off."
9589683f-369f-4a30-8c93-e9f5d04ceb5a
---
.../org/apache/hadoop/io/AbstractMapWritable.java | 5 +-
.../java/org/apache/hadoop/io/EnumSetWritable.java | 10 +-
.../java/org/apache/hadoop/io/GenericWritable.java | 14 +-
.../java/org/apache/hadoop/io/ObjectWritable.java | 80 +++++--
.../java/org/apache/hadoop/io/SequenceFile.java | 4 +-
.../java/org/apache/hadoop/io/WritableName.java | 9 +-
.../org/apache/hadoop/io/file/tfile/TFile.java | 14 +-
.../org/apache/hadoop/util/ReflectionUtils.java | 55 +++++
.../org/apache/hadoop/io/TestGenericWritable.java | 24 ++-
.../hadoop/io/TestSequenceFileClassValidation.java | 207 ++++++++++++++++++
.../hadoop/io/TestWritableClassValidation.java | 233 +++++++++++++++++++++
.../hadoop/mapred/join/CompositeInputSplit.java | 8 +-
.../apache/hadoop/mapred/lib/TaggedInputSplit.java | 12 +-
.../mapreduce/lib/input/TaggedInputSplit.java | 13 +-
.../mapreduce/lib/join/CompositeInputSplit.java | 4 +-
.../hadoop/mapreduce/lib/join/TupleWritable.java | 6 +-
.../mapred/lib/TestTaggedInputSplitValidation.java | 99 +++++++++
.../lib/input/TestTaggedInputSplitValidation.java | 106 ++++++++++
.../lib/join/TestJoinClassValidation.java | 113 ++++++++++
.../tools/fedbalance/procedure/BalanceJob.java | 4 +-
20 files changed, 965 insertions(+), 55 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
index 8cf82f42509..e166cb40822 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.ReflectionUtils;
/**
* Abstract base class for MapWritable and SortedMapWritable
@@ -208,7 +209,9 @@ public void readFields(DataInput in) throws IOException {
byte id = in.readByte();
String className = in.readUTF();
try {
- addToMap(classLoader.loadClass(className), id);
+ addToMap(
+ ReflectionUtils.loadUninitedClass(classLoader, className, Writable.class),
+ id);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
index f2c8b76e2ab..22a3b61c403 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
/** A Writable wrapper for EnumSet. */
@InterfaceAudience.Public
@@ -121,8 +122,13 @@ public void readFields(DataInput in) throws IOException {
if (length == -1)
this.value = null;
else if (length == 0) {
- this.elementType = (Class<E>) ObjectWritable.loadClass(conf,
- WritableUtils.readString(in));
+ try {
+ this.elementType = (Class<E>) ReflectionUtils.loadUninitedClass(conf,
+ WritableUtils.readString(in),
+ Enum.class);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
this.value = EnumSet.noneOf(this.elementType);
} else {
E first = (E) ObjectWritable.readObject(in, conf);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java
index 6de927467e4..7419d5d32b2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java
@@ -123,13 +123,19 @@ public String toString() {
@Override
public void readFields(DataInput in) throws IOException {
- type = in.readByte();
- Class<? extends Writable> clazz = getTypes()[type & 0xff];
+ byte t = in.readByte();
+ Class<? extends Writable>[] types = getTypes();
+ int index = t & 0xff;
+ if (index >= types.length) {
+ throw new IOException("Type index " + index
+ + " is out of range [0, " + types.length + ")");
+ }
+ type = t;
+ Class<? extends Writable> clazz = types[index];
try {
instance = ReflectionUtils.newInstance(clazz, conf);
} catch (Exception e) {
- e.printStackTrace();
- throw new IOException("Cannot initialize the class: " + clazz);
+ throw new IOException("Cannot initialize the class: " + clazz, e);
}
instance.readFields(in);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
index 831931bdace..57f8f9ae178 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
@@ -27,10 +27,12 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.ProtoUtil;
import org.apache.hadoop.thirdparty.protobuf.Message;
+import org.apache.hadoop.util.ReflectionUtils;
/** A polymorphic Writable that writes an instance with it's class name.
* Handles arrays, strings and primitive types without a Writable wrapper.
@@ -39,6 +41,20 @@
@InterfaceStability.Stable
public class ObjectWritable implements Writable, Configurable {
+
+ /**
+ * Maximum object nesting depth permitted when reading, to bound recursion
+ * from a stream of deeply nested arrays.
+ * This is also used as the max size of array list to allocate even if the header declares
+ * a larger value.
+ */
+ static final int MAX_NESTING_DEPTH = 100;
+
+ @VisibleForTesting
+ static final String E_MAX_DEPTH =
+ "Maximum object nesting depth exceeded: " + MAX_NESTING_DEPTH;
+
+
private Class declaredClass;
private Object instance;
private Configuration conf;
@@ -245,22 +261,43 @@ public static void writeObject(DataOutput out, Object instance,
*/
public static Object readObject(DataInput in, Configuration conf)
throws IOException {
- return readObject(in, null, conf);
+ return readObject(in, null, conf, 0);
}
-
+
/**
* Read a {@link Writable}, {@link String}, primitive type, or an array of
- * the preceding.
+ * these.
*
* @param in DataInput.
- * @param objectWritable objectWritable.
+ * @param objectWritable object Writable.
* @param conf configuration.
- * @return Object.
- * @throws IOException raised on errors performing I/O.
+ * @throws IOException raised on errors performing I/O, or if the contents is rejected
+ * as the depth is exceeded or the array size is negative
*/
- @SuppressWarnings("unchecked")
public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
throws IOException {
+ return readObject(in, objectWritable, conf, 0);
+ }
+
+
+ /**
+ * Read a {@link Writable}, {@link String}, primitive type, or an array of
+ * these.
+ * @param in DataInput.
+ * @param objectWritable object Writable.
+ * @param conf configuration.
+ * @param depth depth of recursion
+ * @return Object.
+ * @throws IOException raised on errors performing I/O, or if the contents is rejected
+ * as the depth is exceeded or the array size is negative
+ */
+ @SuppressWarnings("unchecked")
+ private static Object readObject(DataInput in, ObjectWritable objectWritable,
+ Configuration conf, int depth) throws IOException {
+
+ if (depth > MAX_NESTING_DEPTH) {
+ throw new IOException(E_MAX_DEPTH);
+ }
String className = UTF8.readString(in);
Class<?> declaredClass = PRIMITIVE_NAMES.get(className);
if (declaredClass == null) {
@@ -295,11 +332,20 @@ public static Object readObject(DataInput in, ObjectWritable objectWritable, Con
} else if (declaredClass.isArray()) { // array
int length = in.readInt();
- instance = Array.newInstance(declaredClass.getComponentType(), length);
+ if (length < 0) {
+ throw new IOException("Negative array length: " + length);
+ }
+ // Read elements into a growable list first to avoid over-allocating
+ List<Object> elements = new ArrayList<>(Math.min(length, MAX_NESTING_DEPTH));
for (int i = 0; i < length; i++) {
- Array.set(instance, i, readObject(in, conf));
+ elements.add(readObject(in, null, conf, depth + 1));
}
-
+ instance = Array.newInstance(declaredClass.getComponentType(),
+ elements.size());
+ for (int i = 0; i < elements.size(); i++) {
+ Array.set(instance, i, elements.get(i));
+ }
+
} else if (declaredClass == ArrayPrimitiveWritable.Internal.class) {
// Read and unwrap ArrayPrimitiveWritable$Internal array.
// Always allow the read, even if write is disabled by allowCompactArrays.
@@ -315,11 +361,17 @@ public static Object readObject(DataInput in, ObjectWritable objectWritable, Con
instance = Enum.valueOf((Class<? extends Enum>) declaredClass, UTF8.readString(in));
} else if (Message.class.isAssignableFrom(declaredClass)) {
instance = tryInstantiateProtobuf(declaredClass, in);
- } else { // Writable
- Class instanceClass = null;
+ } else {
+ // Writable
+ // the next field is read to identify the writable to load.
+ Class<? extends Writable> instanceClass;
String str = UTF8.readString(in);
- instanceClass = loadClass(conf, str);
-
+ try {
+ instanceClass = ReflectionUtils.loadUninitedClass(conf, str, Writable.class);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("readObject can't find class " + str, e);
+ }
+
Writable writable = WritableFactories.newInstance(instanceClass, conf);
writable.readFields(in);
instance = writable;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index 325820d11cc..3f955cf5ca2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -2073,8 +2073,8 @@ private void init(boolean tempReader) throws IOException {
if (version >= CUSTOM_COMPRESS_VERSION) {
String codecClassname = Text.readString(in);
try {
- Class<? extends CompressionCodec> codecClass
- = conf.getClassByName(codecClassname).asSubclass(CompressionCodec.class);
+ Class<? extends CompressionCodec> codecClass =
+ ReflectionUtils.loadUninitedClass(conf, codecClassname, CompressionCodec.class);
this.codec = ReflectionUtils.newInstance(codecClass, conf);
} catch (ClassNotFoundException cnfe) {
throw new IllegalArgumentException("Unknown codec: " +
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java
index 683d6c099b5..904803fb953 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
/** Utility to permit renaming of Writable implementation classes without
* invalidiating files that contain their class name.
@@ -86,7 +87,7 @@ public static synchronized String getName(Class<?> writableClass) {
* @param name input name.
* @param conf input configuration.
* @return class for a name.
- * @throws IOException raised on errors performing I/O.
+ * @throws IOException raised on errors loading the class.
*/
public static synchronized Class<?> getClass(String name, Configuration conf
) throws IOException {
@@ -94,11 +95,9 @@ public static synchronized Class<?> getClass(String name, Configuration conf
if (writableClass != null)
return writableClass;
try {
- return conf.getClassByName(name);
+ return ReflectionUtils.loadUninitedClass(conf, name, null);
} catch (ClassNotFoundException e) {
- IOException newE = new IOException("WritableName can't load class: " + name);
- newE.initCause(e);
- throw newE;
+ throw new IOException("WritableName can't load class: " + name, e);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
index 09dd10a46ef..102f67df81f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
@@ -48,6 +48,7 @@
import org.apache.hadoop.io.file.tfile.CompareUtils.BytesComparator;
import org.apache.hadoop.io.file.tfile.CompareUtils.MemcmpRawComparator;
import org.apache.hadoop.io.file.tfile.Utils.Version;
+import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2105,14 +2106,11 @@ static BytesComparator makeComparator(String comparator,
String compClassName =
comparator.substring(COMPARATOR_JCLASS.length()).trim();
try {
- // Resolve without running the class initializer, confirm it really
- // is a RawComparator, and only then load and construct it.
- Class<?> compClass =
- Class.forName(compClassName, false, conf.getClassLoader());
- RawComparator<Object> rawComparator =
- (RawComparator<Object>) compClass.asSubclass(RawComparator.class)
- .getDeclaredConstructor().newInstance();
- return new BytesComparator(rawComparator);
+ Class<? extends RawComparator> compClass = ReflectionUtils.loadUninitedClass(
+ TFile.class.getClassLoader(), compClassName, RawComparator.class);
+ // use its default ctor to create an instance
+ return new BytesComparator((RawComparator<Object>) compClass
+ .newInstance());
} catch (Exception e) {
throw new IllegalArgumentException(
"Failed to instantiate comparator: " + comparator + "("
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
index 26bcd4a41cf..d4fc6186fba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
@@ -302,6 +302,60 @@ static void clearCache() {
static int getCacheSize() {
return CONSTRUCTOR_CACHE.size();
}
+
+ /**
+ * Find and load the class with given name <code>className</code> using the
+ * classloader of the specified <code>conf</code> (or this class's classloader
+ * when <code>conf</code> is null) and, when <code>requiredSuperType</code> is
+ * non-null, verify the loaded class is assignable to it before returning.
+ * <p>The class is not initialized.
+ *
+ * @param conf configuration supplying the classloader; may be null.
+ * @param className the name of the class to load.
+ * @param requiredSuperType required supertype of the class, or null to skip
+ * the check.
+ * @param <T> type of the interface.
+ * @return the loaded class.
+ * @throws ClassNotFoundException if the class cannot be found.
+ * @throws ClassCastException if <code>requiredSuperType</code> is
+ * non-null and the class is not assignable to it.
+ */
+ public static<T> Class<? extends T> loadUninitedClass(Configuration conf, String className,
+ Class<T> requiredSuperType) throws ClassNotFoundException {
+ ClassLoader cl = (conf != null)
+ ? conf.getClassLoader()
+ : ReflectionUtils.class.getClassLoader();
+ return loadUninitedClass(cl, className, requiredSuperType);
+ }
+
+ /**
+ * Find and load the class with given name <code>className</code> using the
+ * specified classloader <code>cl</code> and, when
+ * <code>requiredSuperType</code> is non-null, verify the loaded class is
+ * assignable to it before returning.
+ * <p>The class is not initialized.
+ *
+ * @param cl the classloader to use.
+ * @param className the name of the class to load.
+ * @param requiredSuperType required supertype of the class, or null to skip
+ * the check.
+ * @param <T> type of the interface.
+ * @return the loaded class.
+ * @throws ClassNotFoundException if the class cannot be found.
+ * @throws ClassCastException if <code>requiredSuperType</code> is
+ * non-null and the class is not assignable to it.
+ */
+ @SuppressWarnings("unchecked")
+ public static<T> Class<? extends T> loadUninitedClass(ClassLoader cl, String className,
+ Class<T> requiredSuperType) throws ClassNotFoundException, ClassCastException {
+ Class<?> clazz = Class.forName(className, false, cl);
+ if (requiredSuperType != null && !requiredSuperType.isAssignableFrom(clazz)) {
+ throw new ClassCastException(
+ className + " is not a subtype of " + requiredSuperType.getName());
+ }
+ return (Class<? extends T>) clazz;
+ }
+
/**
* A pair of input/output buffers that we use to clone writables.
*/
@@ -413,4 +467,5 @@ public static List<Method> getDeclaredMethodsIncludingInherited(Class<?> clazz)
return methods;
}
+
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestGenericWritable.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestGenericWritable.java
index 2f576441645..e2c903db0cc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestGenericWritable.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestGenericWritable.java
@@ -26,6 +26,8 @@
import org.apache.hadoop.conf.Configuration;
import org.junit.Before;
import org.junit.Test;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;
@@ -100,8 +102,8 @@ public static class Baz extends Bar {
public void readFields(DataInput in) throws IOException {
super.readFields(in);
//needs a configuration parameter
- assertEquals("Configuration is not set for the wrapped object",
- CONF_TEST_VALUE, getConf().get(CONF_TEST_KEY));
+ assertEquals("Configuration is not set for the wrapped object",
+ CONF_TEST_VALUE, getConf().get(CONF_TEST_KEY));
}
@Override
public void write(DataOutput out) throws IOException {
@@ -189,4 +191,22 @@ public void testGet() throws Exception {
assertEquals(foo, generic.get());
}
+ /**
+ * A type index that falls outside the registered types array is reported as
+ * an {@link IOException}.
+ */
+ @Test
+ public void testReadFieldsRejectsOutOfRangeType() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ out.writeByte(0xff);
+ out.close();
+ try (DataInputBuffer in = new DataInputBuffer()) {
+ in.reset(out.getData(), out.getLength());
+ FooGenericWritable generic = new FooGenericWritable();
+ generic.setConf(conf);
+ intercept(IOException.class, "out of range", () ->
+ generic.readFields(in));
+ }
+ }
+
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileClassValidation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileClassValidation.java
new file mode 100644
index 00000000000..8eb4d4e01a0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileClassValidation.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests that {@link SequenceFile.Reader} resolves the key, value and codec
+ * class names from a file header without loading them eagerly, and rejects a
+ * class that no configured serializer (or, for the codec, the codec type) can
+ * handle.
+ */
+public class TestSequenceFileClassValidation extends AbstractHadoopTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestSequenceFileClassValidation.class);
+
+ /** Name of {@link Sentinel}, referenced as a string so it is never loaded. */
+ private static final String SENTINEL =
+ TestSequenceFileClassValidation.class.getName() + "$Sentinel";
+
+ private Configuration conf;
+ private FileSystem fs;
+ private File testDir;
+
+ @Before
+ public void setup() throws IOException {
+ LoadFlag.reset();
+ conf = new Configuration();
+ fs = FileSystem.getLocal(conf);
+ testDir = GenericTestUtils.getTestDir("TestSequenceFileClassValidation");
+ }
+
+ /**
+ * Write a minimal SequenceFile header naming the given key, value and (when
+ * compressed) codec classes; no records follow.
+ */
+ private Path writeHeader(String name, String keyClass, String valClass,
+ boolean compressed, String codecClass) throws IOException {
+ Path path = new Path(new File(testDir, name).toURI());
+ try (FSDataOutputStream out = fs.create(path, true)) {
+ out.write(new byte[] {'S', 'E', 'Q', 6}); // magic + version
+ Text.writeString(out, keyClass);
+ Text.writeString(out, valClass);
+ out.writeBoolean(compressed); // compressed?
+ out.writeBoolean(false); // block compressed?
+ if (compressed) {
+ Text.writeString(out, codecClass);
+ }
+ out.writeInt(0); // empty metadata
+ out.write(new byte[16]); // sync marker
+ }
+ return path;
+ }
+
+ @Test
+ public void testResolveDoesNotLoad() throws Exception {
+ Class<?> clazz = WritableName.getClass(SENTINEL, conf);
+ assertThat(clazz.getName()).isEqualTo(SENTINEL);
+ assertSentinelNotLoaded();
+ }
+
+ @Test
+ public void testResolveUnknownClassThrows() throws Exception {
+ intercept(IOException.class,
+ () -> WritableName.getClass("no.such.Class", conf));
+ }
+
+ @Test
+ public void testReaderRejectsUnacceptedKeyClass() throws Exception {
+ Path path = writeHeader("badkey.seq", SENTINEL, Text.class.getName(),
+ false, null);
+ intercept(IOException.class,
+ () -> new SequenceFile.Reader(fs, path, conf).close());
+ assertSentinelNotLoaded();
+ }
+
+ @Test
+ public void testReaderRejectsUnacceptedValueClass() throws Exception {
+ Path path = writeHeader("badvalue.seq", Text.class.getName(), SENTINEL,
+ false, null);
+ intercept(IOException.class,
+ () -> new SequenceFile.Reader(fs, path, conf).close());
+ assertSentinelNotLoaded();
+ }
+
+ @Test
+ public void testReaderRejectsNonCodec() throws Exception {
+ Path path = writeHeader("badcodec.seq", Text.class.getName(),
+ Text.class.getName(), true, SENTINEL);
+ assertSentinelNotLoaded();
+ intercept(ClassCastException.class,
+ () -> new SequenceFile.Reader(fs, path, conf).close());
+ assertSentinelNotLoaded();
+ }
+
+ @Test
+ public void testRoundTrip() throws Exception {
+ Path path = new Path(new File(testDir, "roundtrip.seq").toURI());
+ try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path,
+ Text.class, IntWritable.class)) {
+ writer.append(new Text("k"), new IntWritable(7));
+ }
+ assertRead(path);
+ }
+
+ @Test
+ public void testRoundTripCompressed() throws Exception {
+ Path path = new Path(new File(testDir, "roundtrip-codec.seq").toURI());
+ try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path,
+ Text.class, IntWritable.class, CompressionType.RECORD,
+ new DefaultCodec())) {
+ writer.append(new Text("k"), new IntWritable(7));
+ }
+ assertRead(path);
+ }
+
+ private void assertRead(Path path) throws IOException {
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) {
+ Text key = new Text();
+ IntWritable value = new IntWritable();
+ assertThat(reader.next(key, value)).isTrue();
+ assertThat(key).isEqualTo(new Text("k"));
+ assertThat(value).isEqualTo(new IntWritable(7));
+ }
+ }
+
+
+ /**
+ * Assert the sentinel was not loaded.
+ */
+ private static void assertSentinelNotLoaded() {
+ assertThat(LoadFlag.isLoaded())
+ .describedAs("sentinel must not be loaded")
+ .isFalse();
+ }
+
+
+ /**
+ * Records whether {@link Sentinel} has been loaded. Kept separate from
+ * {@link Sentinel} so that reading the flag does not load the sentinel.
+ */
+ public static final class LoadFlag {
+
+ private static volatile boolean loaded = false;
+
+ static boolean isLoaded() {
+ return loaded;
+ }
+
+ static void set() {
+ loaded = true;
+ }
+
+ static void reset() {
+ LOG.info("Reset Sentinel");
+ loaded = false;
+ }
+ }
+
+ /** Neither a Writable nor a CompressionCodec; records that it was loaded. */
+ @SuppressWarnings("unused")
+ public static final class Sentinel {
+
+ static {
+ LOG.info("Initialized Sentinel");
+ LoadFlag.set();
+ }
+
+ public Sentinel() {
+ }
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableClassValidation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableClassValidation.java
new file mode 100644
index 00000000000..0d370ce7fe9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableClassValidation.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import static org.apache.hadoop.io.ObjectWritable.E_MAX_DEPTH;
+import static org.apache.hadoop.io.ObjectWritable.MAX_NESTING_DEPTH;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests that the polymorphic Writable classes reject a class named on the wire
+ * that does not implement the required type, and do not load that class.
+ */
+public class TestWritableClassValidation extends AbstractHadoopTestBase {
+
+ /** Name of {@link Sentinel}, referenced as a string so it is never loaded. */
+ private static final String SENTINEL =
+ TestWritableClassValidation.class.getName() + "$Sentinel";
+
+ @Before
+ public void resetSentinel() {
+ LoadFlag.setLoaded(false);
+ }
+
+ @Test
+ public void testLoadClassRejectsWrongType() throws Exception {
+ Configuration conf = new Configuration();
+ intercept(ClassCastException.class, () -> {
+ ReflectionUtils.loadUninitedClass(conf, SENTINEL, Writable.class);
+ });
+ assertSentinelNotLoaded();
+ }
+
+ private static void assertSentinelNotLoaded() {
+ assertThat(LoadFlag.isLoaded()).
+ describedAs("sentinel must not be loaded")
+ .isFalse();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testObjectWritableRejectsNonWritable() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ // A benign declaredClass routes readObject down the Writable branch; the
+ // instanceClass then names the non-Writable sentinel, which is the value
+ // checked before instantiation.
+ UTF8.writeString(out, Text.class.getName());
+ UTF8.writeString(out, SENTINEL);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ assertWrongKind(intercept(RuntimeException.class,
+ () -> ObjectWritable.readObject(in, null)));
+ assertSentinelNotLoaded();
+ }
+
+ /** Assert a ClassCastException is somewhere in the cause chain. */
+ private static void assertWrongKind(Throwable t) {
+ for (Throwable c = t; c != null; c = c.getCause()) {
+ if (c instanceof ClassCastException) {
+ return;
+ }
+ }
+ fail("expected ClassCastException in cause chain of " + t);
+ }
+
+ @Test
+ public void testMapWritableRejectsNonWritable() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ out.writeByte(1); // one "new" class in the table
+ out.writeByte(1); // its id
+ out.writeUTF(SENTINEL); // its name
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ MapWritable map = new MapWritable();
+ intercept(ClassCastException.class, () ->
+ map.readFields(in));
+ assertSentinelNotLoaded();
+ }
+
+ @Test
+ public void testEnumSetWritableRejectsNonEnum() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ out.writeInt(0); // empty set
+ WritableUtils.writeString(out, SENTINEL); // element type
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ EnumSetWritable<?> set = new EnumSetWritable<>();
+ assertWrongKind(intercept(RuntimeException.class, () -> set.readFields(in)));
+ assertSentinelNotLoaded();
+ }
+
+ @Test
+ public void testNegativeArrayLengthRejected() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ UTF8.writeString(out, Text[].class.getName());
+ out.writeInt(-1);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ intercept(IOException.class, () -> ObjectWritable.readObject(in, null));
+ }
+
+ @Test
+ public void testHugeArrayLengthDoesNotPreallocate() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ UTF8.writeString(out, Text[].class.getName());
+ out.writeInt(Integer.MAX_VALUE); // no element data follows
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ intercept(IOException.class, () -> ObjectWritable.readObject(in, null));
+ }
+
+ @Test
+ public void testDeepNestingRejected() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ String arrayClass = Object[].class.getName();
+ for (int i = 0; i < MAX_NESTING_DEPTH + 10; i++) { // each level: an Object[] of length 1
+ UTF8.writeString(out, arrayClass);
+ out.writeInt(1);
+ }
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ intercept(IOException.class, E_MAX_DEPTH,
+ () -> ObjectWritable.readObject(in, null));
+ }
+
+ @Test
+ public void testArrayRoundTrip() throws Exception {
+ Text[] src = {new Text("a"), new Text("b"), new Text("c")};
+ DataOutputBuffer out = new DataOutputBuffer();
+ ObjectWritable.writeObject(out, src, Text[].class, null);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ Text[] read = (Text[]) ObjectWritable.readObject(in, null);
+ assertThat(read)
+ .containsExactly(new Text("a"), new Text("b"), new Text("c"));
+ }
+
+ @Test
+ public void testObjectWritableRoundTrip() throws Exception {
+ Text src = new Text("hello");
+ DataOutputBuffer out = new DataOutputBuffer();
+ ObjectWritable.writeObject(out, src, Text.class, null);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ assertThat(ObjectWritable.readObject(in, null))
+ .isEqualTo(src);
+ }
+
+ @Test
+ public void testMapWritableRoundTrip() throws Exception {
+ MapWritable src = new MapWritable();
+ src.put(new Text("k"), new IntWritable(7));
+ DataOutputBuffer out = new DataOutputBuffer();
+ src.write(out);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ MapWritable dst = new MapWritable();
+ dst.readFields(in);
+ assertThat(dst.get(new Text("k"))).isEqualTo(new IntWritable(7));
+ }
+
+ @Test
+ public void testEnumSetWritableRoundTrip() throws Exception {
+ EnumSetWritable<Colour> src =
+ new EnumSetWritable<>(EnumSet.of(Colour.RED, Colour.BLUE));
+ DataOutputBuffer out = new DataOutputBuffer();
+ src.write(out);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ EnumSetWritable<Colour> dst = new EnumSetWritable<>();
+ dst.readFields(in);
+ assertThat(dst.get())
+ .contains(Colour.RED, Colour.BLUE)
+ .doesNotContain(Colour.GREEN);
+ }
+
+ enum Colour { RED, GREEN, BLUE }
+
+ /**
+ * Records whether {@link Sentinel} has been loaded. Kept separate from
+ * {@link Sentinel} so that reading the flag does not load the sentinel.
+ */
+ public static final class LoadFlag {
+
+ private static volatile boolean loaded = false;
+
+ static boolean isLoaded() {
+ return loaded;
+ }
+
+ static void setLoaded(boolean loaded) {
+ LoadFlag.loaded = loaded;
+ }
+ }
+
+ /** Not a Writable and not an enum; records that it was loaded. */
+ public static final class Sentinel {
+
+ static {
+ LoadFlag.setLoaded(true);
+ }
+
+ public Sentinel() {
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputSplit.java
index 9f5336f3ebc..01ec5ccb72f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/join/CompositeInputSplit.java
@@ -138,15 +138,17 @@ public void readFields(DataInput in) throws IOException {
Class<? extends InputSplit>[] cls = new Class[card];
try {
for (int i = 0; i < card; ++i) {
- cls[i] =
- Class.forName(Text.readString(in)).asSubclass(InputSplit.class);
+ cls[i] = ReflectionUtils.loadUninitedClass(
+ getClass().getClassLoader(),
+ Text.readString(in),
+ InputSplit.class);
}
for (int i = 0; i < card; ++i) {
splits[i] = ReflectionUtils.newInstance(cls[i], null);
splits[i].readFields(in);
}
} catch (ClassNotFoundException e) {
- throw (IOException)new IOException("Failed split init").initCause(e);
+ throw new IOException("Failed split init", e);
}
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java
index 494e1c10611..6b21b39a6db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java
@@ -106,18 +106,20 @@ public String[] getLocations() throws IOException {
@SuppressWarnings("unchecked")
public void readFields(DataInput in) throws IOException {
- inputSplitClass = (Class<? extends InputSplit>) readClass(in);
+ inputSplitClass = (Class<? extends InputSplit>) readClass(in,
+ InputSplit.class);
inputSplit = (InputSplit) ReflectionUtils
.newInstance(inputSplitClass, conf);
inputSplit.readFields(in);
- inputFormatClass = (Class<? extends InputFormat>) readClass(in);
- mapperClass = (Class<? extends Mapper>) readClass(in);
+ inputFormatClass = (Class<? extends InputFormat>) readClass(in,
+ InputFormat.class);
+ mapperClass = (Class<? extends Mapper>) readClass(in, Mapper.class);
}
- private Class<?> readClass(DataInput in) throws IOException {
+ private Class<?> readClass(DataInput in, Class<?> xface) throws IOException {
String className = StringInterner.weakIntern(Text.readString(in));
try {
- return conf.getClassByName(className);
+ return ReflectionUtils.loadUninitedClass(conf, className, xface);
} catch (ClassNotFoundException e) {
throw new RuntimeException("readObject can't find class", e);
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java
index fd07d00fa7d..85c0aa8840e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java
@@ -117,9 +117,12 @@ public String[] getLocations() throws IOException, InterruptedException {
@SuppressWarnings("unchecked")
public void readFields(DataInput in) throws IOException {
- inputSplitClass = (Class<? extends InputSplit>) readClass(in);
- inputFormatClass = (Class<? extends InputFormat<?, ?>>) readClass(in);
- mapperClass = (Class<? extends Mapper<?, ?, ?, ?>>) readClass(in);
+ inputSplitClass = (Class<? extends InputSplit>) readClass(in,
+ InputSplit.class);
+ inputFormatClass = (Class<? extends InputFormat<?, ?>>) readClass(in,
+ InputFormat.class);
+ mapperClass = (Class<? extends Mapper<?, ?, ?, ?>>) readClass(in,
+ Mapper.class);
inputSplit = (InputSplit) ReflectionUtils
.newInstance(inputSplitClass, conf);
SerializationFactory factory = new SerializationFactory(conf);
@@ -128,10 +131,10 @@ public void readFields(DataInput in) throws IOException {
inputSplit = (InputSplit)deserializer.deserialize(inputSplit);
}
- private Class<?> readClass(DataInput in) throws IOException {
+ private Class<?> readClass(DataInput in, Class<?> xface) throws IOException {
String className = StringInterner.weakIntern(Text.readString(in));
try {
- return conf.getClassByName(className);
+ return ReflectionUtils.loadUninitedClass(conf, className, xface);
} catch (ClassNotFoundException e) {
throw new RuntimeException("readObject can't find class", e);
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputSplit.java
index 676209de118..3eed3a3be30 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/CompositeInputSplit.java
@@ -149,8 +149,8 @@ public void readFields(DataInput in) throws IOException {
Class<? extends InputSplit>[] cls = new Class[card];
try {
for (int i = 0; i < card; ++i) {
- cls[i] =
- Class.forName(Text.readString(in)).asSubclass(InputSplit.class);
+ cls[i] = ReflectionUtils.loadUninitedClass(
+ conf, Text.readString(in), InputSplit.class);
}
for (int i = 0; i < card; ++i) {
splits[i] = ReflectionUtils.newInstance(cls[i], null);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/TupleWritable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/TupleWritable.java
index 380363c897a..7477f9eb4fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/TupleWritable.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/TupleWritable.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.ReflectionUtils;
/**
* Writable type storing multiple {@link org.apache.hadoop.io.Writable}s.
@@ -191,7 +192,10 @@ public void readFields(DataInput in) throws IOException {
Class<? extends Writable>[] cls = new Class[card];
try {
for (int i = 0; i < card; ++i) {
- cls[i] = Class.forName(Text.readString(in)).asSubclass(Writable.class);
+ cls[i] = ReflectionUtils.loadUninitedClass(
+ getClass().getClassLoader(),
+ Text.readString(in),
+ Writable.class);
}
for (int i = 0; i < card; ++i) {
if (cls[i].equals(NullWritable.class)) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestTaggedInputSplitValidation.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestTaggedInputSplitValidation.java
new file mode 100644
index 00000000000..5c4b6add961
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/lib/TestTaggedInputSplitValidation.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred.lib;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+import org.apache.hadoop.test.LambdaTestUtils.VoidCallable;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/**
+ * Tests that {@link TaggedInputSplit} rejects a class named on the wire that
+ * does not implement the required type, and does not load that class.
+ */
+public class TestTaggedInputSplitValidation extends AbstractHadoopTestBase {
+
+ public static final class LoadFlag {
+
+ private static volatile boolean loaded = false;
+
+ static boolean isLoaded() {
+ return loaded;
+ }
+
+ static void setLoaded(boolean loaded) {
+ LoadFlag.loaded = loaded;
+ }
+ }
+
+ /** Not an InputSplit; records that it was loaded. */
+ @SuppressWarnings("unused")
+ public static final class Sentinel {
+
+ static {
+ LoadFlag.setLoaded(true);
+ }
+
+ public Sentinel() {
+ }
+ }
+
+ private static final String SENTINEL =
+ TestTaggedInputSplitValidation.class.getName() + "$Sentinel";
+
+ @Before
+ public void resetSentinel() {
+ LoadFlag.setLoaded(false);
+ }
+
+ private static void assertRejectsWrongKind(VoidCallable action) throws Exception {
+ RuntimeException thrown = intercept(RuntimeException.class, action);
+ for (Throwable c = thrown; c != null; c = c.getCause()) {
+ if (c instanceof ClassCastException) {
+ return;
+ }
+ }
+ fail("expected ClassCastException in cause chain of " + thrown, thrown);
+ }
+
+ @Test
+ public void testRejectsNonInputSplit() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ Text.writeString(out, SENTINEL);
+ try (DataInputBuffer in = new DataInputBuffer()) {
+ in.reset(out.getData(), out.getLength());
+ TaggedInputSplit split = new TaggedInputSplit();
+ split.setConf(new Configuration());
+ assertRejectsWrongKind(() -> split.readFields(in));
+ }
+ assertThat(LoadFlag.isLoaded()).
+ describedAs("sentinel must not be loaded")
+ .isFalse();
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestTaggedInputSplitValidation.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestTaggedInputSplitValidation.java
new file mode 100644
index 00000000000..841b33a6b8e
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestTaggedInputSplitValidation.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.input;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.test.LambdaTestUtils.VoidCallable;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+/**
+ * Tests that {@link TaggedInputSplit} rejects a class named on the wire that
+ * does not implement the required type, and does not load that class.
+ */
+public class TestTaggedInputSplitValidation {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestTaggedInputSplitValidation.class);
+
+ public static final class LoadFlag {
+
+ private static volatile boolean loaded = false;
+
+ static boolean isLoaded() {
+ return loaded;
+ }
+
+ static void set() {
+ loaded = true;
+ }
+
+ static void reset() {
+ LOG.info("Reset Sentinel");
+ loaded = false;
+ }
+ }
+ /** Not an InputSplit; records that it was loaded. */
+ @SuppressWarnings("unused")
+ public static final class Sentinel {
+ static {
+ LoadFlag.set();
+ }
+
+ public Sentinel() {
+ }
+ }
+
+ private static final String SENTINEL =
+ TestTaggedInputSplitValidation.class.getName() + "$Sentinel";
+
+ @Before
+ public void resetSentinel() {
+ LoadFlag.reset();
+ }
+
+ private static void assertRejectsWrongKind(VoidCallable action) throws Exception {
+ RuntimeException thrown = intercept(RuntimeException.class, action);
+ for (Throwable c = thrown; c != null; c = c.getCause()) {
+ if (c instanceof ClassCastException) {
+ return;
+ }
+ }
+ fail("expected ClassCastException in cause chain of " + thrown, thrown);
+ }
+
+ @Test
+ public void testRejectsNonInputSplit() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ Text.writeString(out, SENTINEL);
+ try (DataInputBuffer in = new DataInputBuffer()) {
+ in.reset(out.getData(), out.getLength());
+ TaggedInputSplit split = new TaggedInputSplit();
+ split.setConf(new Configuration());
+ assertRejectsWrongKind(() -> split.readFields(in));
+ }
+ assertThat(LoadFlag.isLoaded()).
+ describedAs("sentinel must not be loaded")
+ .isFalse();
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinClassValidation.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinClassValidation.java
new file mode 100644
index 00000000000..209c1b56fdc
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinClassValidation.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.join;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests that the join deserializers reject a class named on the wire that does
+ * not implement the required type, and do not load that class.
+ */
+public class TestJoinClassValidation extends AbstractHadoopTestBase {
+
+ private static final class LoadFlag {
+ private static volatile boolean loaded = false;
+
+ static boolean isLoaded() {
+ return loaded;
+ }
+
+ static void setLoaded(boolean loaded) {
+ LoadFlag.loaded = loaded;
+ }
+ }
+
+ /** Implements neither Writable nor InputSplit; records that it was loaded. */
+ @SuppressWarnings("unused")
+ public static final class Sentinel {
+ static {
+ LoadFlag.setLoaded(true);
+ }
+
+ public Sentinel() {
+ }
+ }
+
+ private static final String SENTINEL =
+ TestJoinClassValidation.class.getName() + "$Sentinel";
+
+ @Before
+ public void resetSentinel() {
+ LoadFlag.setLoaded(false);
+ }
+
+ @Test
+ public void testTupleWritableRejectsNonWritable() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ WritableUtils.writeVInt(out, 1); // one element
+ WritableUtils.writeVLong(out, 0L); // empty "written" bitset
+ Text.writeString(out, SENTINEL);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ TupleWritable tuple = new TupleWritable();
+ intercept(ClassCastException.class, () -> tuple.readFields(in));
+ assertSentinelNotLoaded();
+ }
+
+ private static void assertSentinelNotLoaded() {
+ assertThat(LoadFlag.isLoaded()).
+ describedAs("sentinel must not be loaded")
+ .isFalse();
+ }
+
+ @Test
+ public void testCompositeInputSplitRejectsNonSplit() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ WritableUtils.writeVInt(out, 1);
+ Text.writeString(out, SENTINEL);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ CompositeInputSplit split = new CompositeInputSplit();
+ intercept(ClassCastException.class, () -> split.readFields(in));
+ assertSentinelNotLoaded();
+ }
+
+ @Test
+ public void testMapredCompositeInputSplitRejectsNonSplit() throws Exception {
+ DataOutputBuffer out = new DataOutputBuffer();
+ WritableUtils.writeVInt(out, 1);
+ Text.writeString(out, SENTINEL);
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+ org.apache.hadoop.mapred.join.CompositeInputSplit split =
+ new org.apache.hadoop.mapred.join.CompositeInputSplit();
+ intercept(ClassCastException.class, () -> split.readFields(in));
+ assertSentinelNotLoaded();
+ }
+}
diff --git a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceJob.java b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceJob.java
index 7a088ed5ebf..a70cd525787 100644
--- a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceJob.java
+++ b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceJob.java
@@ -266,7 +266,9 @@ public void readFields(DataInput in) throws IOException {
for (int i = 0; i < taskTableSize; i++) {
String className = Text.readString(in);
try {
- T p = (T) ReflectionUtils.newInstance(Class.forName(className), null);
+ Class<? extends BalanceProcedure> clazz = ReflectionUtils.loadUninitedClass(
+ getClass().getClassLoader(), className, BalanceProcedure.class);
+ T p = (T) ReflectionUtils.newInstance(clazz, null);
p.readFields(in);
procedureTable.put(p.name(), p);
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org