You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/07/12 02:26:53 UTC
svn commit: r1360493 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
main/java/org/apache/hadoop/hbase/util/CompressionTest.java
test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
Author: mbautin
Date: Thu Jul 12 00:26:53 2012
New Revision: 1360493
URL: http://svn.apache.org/viewvc?rev=1360493&view=rev
Log:
[master] [89-fb] Make HBase compile without Snappy and pull TestCompressionTest from trunk
Author: mbautin
Summary:
In order to be able to compile HBase with our current HDFS release branch that does not have Snappy we need to lazily instantiate SnappyCodec via reflection. Also pulling in TestCompressionTest that tests all compression codecs.
Reviewing this internally because the HBase part of Snappy Hadoop integration requires to be manually ported to hbase-89-fb-apache.
Tagged as [master] but this is really an 89-fb-only change that is not applicable to hbase-apache-trunk.
Test Plan: Run unit tests
Reviewers: pritam, feifei, pkhemani
Reviewed By: pkhemani
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D515178
Added:
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java?rev=1360493&r1=1360492&r2=1360493&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java Thu Jul 12 00:26:53 2012
@@ -34,7 +34,6 @@ import org.apache.hadoop.io.compress.Com
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
import org.apache.hadoop.io.compress.GzipCodec;
-import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.util.ReflectionUtils;
@@ -108,15 +107,31 @@ public final class Compression {
}
},
SNAPPY("snappy") {
- private transient CompressionCodec snappyCodec;
- @Override
- CompressionCodec getCodec(Configuration conf) {
- if (snappyCodec == null) {
- snappyCodec = new SnappyCodec();
- ((Configurable) snappyCodec).setConf(new Configuration(conf));
- }
- return (CompressionCodec) snappyCodec;
- }
+ private transient CompressionCodec snappyCodec;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ CompressionCodec getCodec(Configuration conf) {
+ if (snappyCodec == null) {
+ try {
+ Class<? extends CompressionCodec> snappyCodecClass =
+ (Class<? extends CompressionCodec>)
+ Class.forName(CompressionCodec.class.getPackage().getName() + ".SnappyCodec");
+ snappyCodec = snappyCodecClass.newInstance();
+ } catch (InstantiationException e) {
+ LOG.error(e);
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ LOG.error(e);
+ throw new RuntimeException(e);
+ } catch (ClassNotFoundException e) {
+ LOG.error(e);
+ throw new RuntimeException(e);
+ }
+ ((Configurable) snappyCodec).setConf(new Configuration(conf));
+ }
+ return (CompressionCodec) snappyCodec;
+ }
},
NONE("none") {
@Override
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java?rev=1360493&r1=1360492&r2=1360493&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java Thu Jul 12 00:26:53 2012
@@ -19,77 +19,129 @@
*/
package org.apache.hadoop.hbase.util;
-import java.net.URI;
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.io.compress.Compressor;
+
+import com.google.common.base.Joiner;
/**
* Compression validation test. Checks compression is working. Be sure to run
* on every node in your cluster.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class CompressionTest {
+ static final Log LOG = LogFactory.getLog(CompressionTest.class);
+
+ public static boolean testCompression(String codec) {
+ codec = codec.toLowerCase();
+
+ Compression.Algorithm a;
+
+ try {
+ a = Compression.getCompressionAlgorithmByName(codec);
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Codec type: " + codec + " is not known");
+ return false;
+ }
+
+ try {
+ testCompression(a);
+ return true;
+ } catch (IOException ignored) {
+ LOG.warn("Can't instantiate codec: " + codec, ignored);
+ return false;
+ }
+ }
+
+ private final static Boolean[] compressionTestResults
+ = new Boolean[Compression.Algorithm.values().length];
+ static {
+ for (int i = 0 ; i < compressionTestResults.length ; ++i) {
+ compressionTestResults[i] = null;
+ }
+ }
+
+ public static void testCompression(Compression.Algorithm algo)
+ throws IOException {
+ if (compressionTestResults[algo.ordinal()] != null) {
+ if (compressionTestResults[algo.ordinal()]) {
+ return ; // already passed test, dont do it again.
+ } else {
+ // failed.
+ throw new IOException("Compression algorithm '" + algo.getName() + "'" +
+ " previously failed test.");
+ }
+ }
+
+ try {
+ Compressor c = algo.getCompressor();
+ algo.returnCompressor(c);
+ compressionTestResults[algo.ordinal()] = true; // passes
+ } catch (Throwable t) {
+ compressionTestResults[algo.ordinal()] = false; // failure
+ throw new IOException(t);
+ }
+ }
+
protected static Path path = new Path(".hfile-comp-test");
public static void usage() {
- System.err.println("Usage: CompressionTest HDFS_PATH none|gz|lzo");
+ System.err.println("Usage: CompressionTest <path> "
+ + Joiner.on("|").join(Compression.Algorithm.values()) + "\n\nFor example:\n"
+ + " hbase " + CompressionTest.class + " file:///tmp/testfile gz\n");
System.exit(1);
}
- protected static DistributedFileSystem openConnection(String urlString)
- throws java.net.URISyntaxException, java.io.IOException {
- URI dfsUri = new URI(urlString);
- Configuration dfsConf = new Configuration();
- DistributedFileSystem dfs = new DistributedFileSystem();
- dfs.initialize(dfsUri, dfsConf);
- return dfs;
- }
-
- protected static boolean closeConnection(DistributedFileSystem dfs) {
- if (dfs != null) {
- try {
- dfs.close();
- dfs = null;
- } catch (Exception e) {
- e.printStackTrace();
- }
+ public static void doSmokeTest(FileSystem fs, Path path, String codec)
+ throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
+ .withPath(fs, path)
+ .withCompression(codec.toLowerCase())
+ .create();
+ writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval"));
+ writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval"));
+ writer.close();
+
+ HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf));
+ reader.loadFileInfo();
+ byte[] key = reader.getFirstKey();
+ boolean rc = Bytes.toString(key).equals("testkey");
+ reader.close();
+
+ if (!rc) {
+ throw new Exception("Read back incorrect result: " +
+ Bytes.toStringBinary(key));
}
- return dfs == null;
}
- public static void main(String[] args) {
- if (args.length != 2) usage();
- Configuration conf = HBaseConfiguration.create();
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ usage();
+ System.exit(1);
+ }
+
+ Configuration conf = new Configuration();
+ Path path = new Path(args[0]);
+ FileSystem fs = path.getFileSystem(conf);
try {
- DistributedFileSystem dfs = openConnection(args[0]);
- dfs.delete(path, false);
- HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
- .withPath(dfs, path)
- .withCompression(args[1])
- .create();
- writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval"));
- writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval"));
- writer.close();
-
- HFile.Reader reader =
- HFile.createReader(dfs, path, new CacheConfig(conf));
- reader.loadFileInfo();
- byte[] key = reader.getFirstKey();
- boolean rc = Bytes.toString(key).equals("testkey");
- reader.close();
-
- dfs.delete(path, false);
- closeConnection(dfs);
-
- if (rc) System.exit(0);
- } catch (Exception e) {
- e.printStackTrace();
+ doSmokeTest(fs, path, args[1]);
+ } finally {
+ fs.delete(path, false);
}
- System.out.println("FAILED");
- System.exit(1);
+ System.out.println("SUCCESS");
}
-}
\ No newline at end of file
+}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java?rev=1360493&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java Thu Jul 12 00:26:53 2012
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.junit.Test;
+
+import java.io.BufferedOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+
+public class TestCompressionTest {
+
+ @Test
+ public void testTestCompression() {
+
+ // This test will fail if you run the tests with LZO compression available.
+ try {
+ CompressionTest.testCompression(Compression.Algorithm.LZO);
+ fail(); // always throws
+ } catch (IOException e) {
+ // there should be a 'cause'.
+ assertNotNull(e.getCause());
+ }
+
+ // this is testing the caching of the test results.
+ try {
+ CompressionTest.testCompression(Compression.Algorithm.LZO);
+ fail(); // always throws
+ } catch (IOException e) {
+ // there should be NO cause because it's a direct exception not wrapped
+ assertNull(e.getCause());
+ }
+
+
+ assertFalse(CompressionTest.testCompression("LZO"));
+ assertTrue(CompressionTest.testCompression("NONE"));
+ assertTrue(CompressionTest.testCompression("GZ"));
+
+ if (isCompressionAvailable("org.apache.hadoop.io.compress.SnappyCodec")) {
+ if (NativeCodeLoader.isNativeCodeLoaded()) {
+ try {
+ System.loadLibrary("snappy");
+
+ try {
+ Configuration conf = new Configuration();
+ CompressionCodec codec = (CompressionCodec)
+ ReflectionUtils.newInstance(
+ conf.getClassByName("org.apache.hadoop.io.compress.SnappyCodec"), conf);
+
+ DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
+ CompressionOutputStream deflateFilter =
+ codec.createOutputStream(compressedDataBuffer);
+
+ byte[] data = new byte[1024];
+ DataOutputStream deflateOut = new DataOutputStream(
+ new BufferedOutputStream(deflateFilter));
+ deflateOut.write(data, 0, data.length);
+ deflateOut.flush();
+ deflateFilter.finish();
+
+ // Snappy Codec class, Snappy nativelib and Hadoop nativelib with
+ // Snappy JNIs are present
+ assertTrue(CompressionTest.testCompression("SNAPPY"));
+ }
+ catch (UnsatisfiedLinkError ex) {
+ // Hadoop nativelib does not have Snappy JNIs
+
+ // cannot assert the codec here because the current logic of
+ // CompressionTest checks only classloading, not the codec
+ // usage.
+ }
+ catch (Exception ex) {
+ }
+ }
+ catch (UnsatisfiedLinkError ex) {
+ // Snappy nativelib is not available
+ assertFalse(CompressionTest.testCompression("SNAPPY"));
+ }
+ }
+ else {
+ // Hadoop nativelib is not available
+ assertFalse(CompressionTest.testCompression("SNAPPY"));
+ }
+ }
+ else {
+ // Snappy Codec class is not available
+ assertFalse(CompressionTest.testCompression("SNAPPY"));
+ }
+ }
+
+ private boolean isCompressionAvailable(String codecClassName) {
+ try {
+ Thread.currentThread().getContextClassLoader().loadClass(codecClassName);
+ return true;
+ }
+ catch (Exception ex) {
+ return false;
+ }
+ }
+
+}
+