You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by za...@apache.org on 2024/05/17 02:27:58 UTC
(hadoop) branch trunk updated: HADOOP-19167 Bug Fix: Change of Codec configuration does not work (#6807)
This is an automated email from the ASF dual-hosted git repository.
zanderxu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3c00093cb554 HADOOP-19167 Bug Fix: Change of Codec configuration does not work (#6807)
3c00093cb554 is described below
commit 3c00093cb554176ffc4bf6cebca6d2953099bfe9
Author: skyskyhu <zk...@yahoo.com>
AuthorDate: Fri May 17 10:27:39 2024 +0800
HADOOP-19167 Bug Fix: Change of Codec configuration does not work (#6807)
---
.../org/apache/hadoop/io/compress/CodecPool.java | 4 +++
.../apache/hadoop/io/compress/TestCodecPool.java | 35 ++++++++++++++++++++++
2 files changed, 39 insertions(+)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
index 5b1826f9e30a..283dcd622ffe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
@@ -152,6 +153,9 @@ public class CodecPool {
compressor = codec.createCompressor();
LOG.info("Got brand-new compressor ["+codec.getDefaultExtension()+"]");
} else {
+ if (conf == null && codec instanceof Configurable) {
+ conf = ((Configurable)codec).getConf();
+ }
compressor.reinit(conf);
if(LOG.isDebugEnabled()) {
LOG.debug("Got recycled compressor");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
index 4b18ee6047ba..ac6aff7427e4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
@@ -22,6 +22,8 @@ import static org.junit.Assert.assertEquals;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.util.List;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
@@ -32,7 +34,10 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.zlib.BuiltInGzipCompressor;
import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor;
+import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionLevel;
+import org.apache.hadoop.io.compress.zlib.ZlibFactory;
import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Before;
import org.junit.Test;
@@ -86,6 +91,36 @@ public class TestCodecPool {
}
}
+ @Test(timeout = 10000)
+ public void testCompressorConf() throws Exception {
+ DefaultCodec codec1 = new DefaultCodec();
+ Configuration conf = new Configuration();
+ ZlibFactory.setCompressionLevel(conf, CompressionLevel.TWO);
+ codec1.setConf(conf);
+ Compressor comp1 = CodecPool.getCompressor(codec1);
+ CodecPool.returnCompressor(comp1);
+
+ DefaultCodec codec2 = new DefaultCodec();
+ Configuration conf2 = new Configuration();
+ CompressionLevel newCompressionLevel = CompressionLevel.THREE;
+ ZlibFactory.setCompressionLevel(conf2, newCompressionLevel);
+ codec2.setConf(conf2);
+ Compressor comp2 = CodecPool.getCompressor(codec2);
+ List<Field> fields = ReflectionUtils.getDeclaredFieldsIncludingInherited(comp2.getClass());
+ for (Field field : fields) {
+ if (field.getName().equals("level")) {
+ field.setAccessible(true);
+ Object levelValue = field.get(comp2);
+ if (levelValue instanceof CompressionLevel) {
+ assertEquals(newCompressionLevel, levelValue);
+ } else {
+ assertEquals(3, levelValue);
+ }
+ }
+ }
+ CodecPool.returnCompressor(comp2);
+ }
+
@Test(timeout = 10000)
public void testDecompressorPoolCounts() {
// Get two decompressors and return them
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org