You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ap...@apache.org on 2014/11/21 20:02:03 UTC
[1/4] phoenix git commit: PHOENIX-1468 Add org.iq80.snappy to client
and server assemblies
Repository: phoenix
Updated Branches:
refs/heads/3.0 b6de62c61 -> ebfebec81
refs/heads/4.0 a58672377 -> 3c1eb3a9f
PHOENIX-1468 Add org.iq80.snappy to client and server assemblies
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3c1eb3a9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3c1eb3a9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3c1eb3a9
Branch: refs/heads/4.0
Commit: 3c1eb3a9faf6d06ab11f5b6ead473e3172a8c068
Parents: 3da2d2b
Author: Andrew Purtell <ap...@apache.org>
Authored: Wed Nov 19 13:35:17 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 21 10:56:15 2014 -0800
----------------------------------------------------------------------
phoenix-assembly/src/build/client.xml | 1 +
phoenix-assembly/src/build/server.xml | 1 +
pom.xml | 18 ++++++++++++++++++
3 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c1eb3a9/phoenix-assembly/src/build/client.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/client.xml b/phoenix-assembly/src/build/client.xml
index f674331..ceaae85 100644
--- a/phoenix-assembly/src/build/client.xml
+++ b/phoenix-assembly/src/build/client.xml
@@ -49,6 +49,7 @@
<include>org.cloudera.htrace:htrace-core</include>
<include>io.netty:netty</include>
<include>commons-codec:commons-codec</include>
+ <include>org.iq80.snappy:snappy</include>
</includes>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c1eb3a9/phoenix-assembly/src/build/server.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server.xml b/phoenix-assembly/src/build/server.xml
index bc656e5..ddfd093 100644
--- a/phoenix-assembly/src/build/server.xml
+++ b/phoenix-assembly/src/build/server.xml
@@ -36,6 +36,7 @@
<includes>
<include>org.apache.phoenix:phoenix-core</include>
<include>org.apache.phoenix:phoenix-hadoop*</include>
+ <include>org.iq80.snappy:snappy</include>
</includes>
</dependencySet>
<dependencySet>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c1eb3a9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 39bb7c8..36ece49 100644
--- a/pom.xml
+++ b/pom.xml
@@ -477,6 +477,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop-two.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -502,6 +508,12 @@
<artifactId>pig</artifactId>
<version>${pig.version}</version>
<classifier>h2</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Make sure we have all the antlr dependencies -->
@@ -534,6 +546,12 @@
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>${flume.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>net.sourceforge.findbugs</groupId>
[3/4] phoenix git commit: PHOENIX-1455 Replace org.xerial.snappy with
org.iq80.snappy pure Java snappy implementation
Posted by ap...@apache.org.
PHOENIX-1455 Replace org.xerial.snappy with org.iq80.snappy pure Java snappy implementation
Conflicts:
phoenix-core/pom.xml
phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
pom.xml
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/39141035
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/39141035
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/39141035
Branch: refs/heads/3.0
Commit: 391410359fca4e111027e0359e1e0bdbd35ac497
Parents: b6de62c
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Nov 17 18:05:33 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 21 10:57:42 2014 -0800
----------------------------------------------------------------------
phoenix-core/pom.xml | 5 ++++
.../DistinctValueWithCountClientAggregator.java | 17 ++++++++++----
.../DistinctValueWithCountServerAggregator.java | 24 ++++++--------------
.../apache/phoenix/join/HashCacheClient.java | 3 ++-
.../apache/phoenix/join/HashCacheFactory.java | 15 ++++++++----
pom.xml | 14 +++++++++---
6 files changed, 47 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/39141035/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index cdf97f6..fceb6c7 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -272,6 +272,11 @@
<artifactId>jackson-xc</artifactId>
</dependency>
<dependency>
+ <groupId>org.iq80.snappy</groupId>
+ <artifactId>snappy</artifactId>
+ <version>${snappy.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/39141035/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
index f29f46a..56ca000 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
@@ -35,6 +35,7 @@ import org.apache.phoenix.schema.PDataType;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.tuple.SingleKeyValueTuple;
import org.apache.phoenix.schema.tuple.Tuple;
+import org.iq80.snappy.Snappy;
/**
* Client side Aggregator which will aggregate data and find distinct values with number of occurrences for each.
@@ -59,14 +60,20 @@ public abstract class DistinctValueWithCountClientAggregator extends BaseAggrega
PDataType resultDataType = getResultDataType();
cachedResult = resultDataType.toObject(ptr, resultDataType, sortOrder);
} else {
- InputStream is = new ByteArrayInputStream(ptr.get(), ptr.getOffset() + 1, ptr.getLength() - 1);
+ InputStream is;
try {
if (Bytes.equals(ptr.get(), ptr.getOffset(), 1, DistinctValueWithCountServerAggregator.COMPRESS_MARKER,
0, 1)) {
- InputStream decompressionStream = DistinctValueWithCountServerAggregator.COMPRESS_ALGO
- .createDecompressionStream(is,
- DistinctValueWithCountServerAggregator.COMPRESS_ALGO.getDecompressor(), 0);
- is = decompressionStream;
+ // This reads the uncompressed length from the front of the compressed input
+ int uncompressedLength = Snappy.getUncompressedLength(ptr.get(), ptr.getOffset() + 1);
+ byte[] uncompressed = new byte[uncompressedLength];
+ // This will throw CorruptionException, a RuntimeException if the snappy data is invalid.
+ // We're making a RuntimeException out of a checked IOException below so assume it's ok
+ // to let any CorruptionException escape.
+ Snappy.uncompress(ptr.get(), ptr.getOffset() + 1, ptr.getLength() - 1, uncompressed, 0);
+ is = new ByteArrayInputStream(uncompressed, 0, uncompressedLength);
+ } else {
+ is = new ByteArrayInputStream(ptr.get(), ptr.getOffset() + 1, ptr.getLength() - 1);
}
DataInputStream in = new DataInputStream(is);
int mapSize = WritableUtils.readVInt(in);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/39141035/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
index f2392d6..6b6ed3e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
@@ -17,16 +17,12 @@
*/
package org.apache.phoenix.expression.aggregator;
-import java.io.ByteArrayOutputStream;
-import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.query.QueryServices;
@@ -40,6 +36,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.iq80.snappy.Snappy;
+
/**
* Server side Aggregator which will aggregate data and find distinct values with number of occurrences for each.
*
@@ -50,7 +48,6 @@ public class DistinctValueWithCountServerAggregator extends BaseAggregator {
private static final Logger LOG = LoggerFactory.getLogger(DistinctValueWithCountServerAggregator.class);
public static final int DEFAULT_ESTIMATED_DISTINCT_VALUES = 10000;
public static final byte[] COMPRESS_MARKER = new byte[] { (byte)1 };
- public static final Algorithm COMPRESS_ALGO = Compression.Algorithm.SNAPPY;
private int compressThreshold;
private byte[] buffer = null;
@@ -101,18 +98,11 @@ public class DistinctValueWithCountServerAggregator extends BaseAggregator {
}
if (serializationSize > compressThreshold) {
// The size for the map serialization is above the threshold. We will do the Snappy compression here.
- ByteArrayOutputStream compressedByteStream = new ByteArrayOutputStream();
- try {
- compressedByteStream.write(COMPRESS_MARKER);
- OutputStream compressionStream = COMPRESS_ALGO.createCompressionStream(compressedByteStream,
- COMPRESS_ALGO.getCompressor(), 0);
- compressionStream.write(buffer, 1, buffer.length - 1);
- compressionStream.flush();
- ptr.set(compressedByteStream.toByteArray(), 0, compressedByteStream.size());
- return true;
- } catch (Exception e) {
- LOG.error("Exception while Snappy compression of data.", e);
- }
+ byte[] compressed = new byte[COMPRESS_MARKER.length + Snappy.maxCompressedLength(buffer.length)];
+ System.arraycopy(COMPRESS_MARKER, 0, compressed, 0, COMPRESS_MARKER.length);
+ int compressedLen = Snappy.compress(buffer, 1, buffer.length - 1, compressed, COMPRESS_MARKER.length);
+ ptr.set(compressed, 0, compressedLen + 1);
+ return true;
}
ptr.set(buffer, 0, offset);
return true;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/39141035/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
index e2f57df..6494603 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
@@ -39,7 +39,8 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
import org.apache.phoenix.util.TupleUtil;
-import org.xerial.snappy.Snappy;
+
+import org.iq80.snappy.Snappy;
/**
*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/39141035/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
index 6c2d7ce..5647155 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;
+
import org.apache.phoenix.cache.HashCache;
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -50,7 +51,9 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.SizedUtil;
import org.apache.phoenix.util.TupleUtil;
-import org.xerial.snappy.Snappy;
+
+import org.iq80.snappy.CorruptionException;
+import org.iq80.snappy.Snappy;
public class HashCacheFactory implements ServerCacheFactory {
@@ -68,11 +71,13 @@ public class HashCacheFactory implements ServerCacheFactory {
@Override
public Closeable newCache(ImmutableBytesWritable cachePtr, MemoryChunk chunk) throws SQLException {
try {
- int size = Snappy.uncompressedLength(cachePtr.get());
- byte[] uncompressed = new byte[size];
- Snappy.uncompress(cachePtr.get(), 0, cachePtr.getLength(), uncompressed, 0);
+ // This reads the uncompressed length from the front of the compressed input
+ int uncompressedLen = Snappy.getUncompressedLength(cachePtr.get(), cachePtr.getOffset());
+ byte[] uncompressed = new byte[uncompressedLen];
+ Snappy.uncompress(cachePtr.get(), cachePtr.getOffset(), cachePtr.getLength(),
+ uncompressed, 0);
return new HashCacheImpl(uncompressed, chunk);
- } catch (IOException e) {
+ } catch (CorruptionException e) {
throw ServerUtil.parseServerException(e);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/39141035/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 141b61a..011bea0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -50,6 +50,14 @@
<id>conjars.org</id>
<url>http://conjars.org/repo</url>
</repository>
+ <repository>
+ <id>sonatype-nexus-snapshots</id>
+ <name>Sonatype Nexus Snapshots</name>
+ <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
</repositories>
<parent>
@@ -96,7 +104,7 @@
<flume.version>1.4.0</flume.version>
<findbugs.version>1.3.2</findbugs.version>
<jline.version>2.11</jline.version>
- <snappy.version>1.1.0.1</snappy.version>
+ <snappy.version>0.3</snappy.version>
<jodatime.version>2.3</jodatime.version>
<!-- Test Dependencies -->
@@ -476,8 +484,8 @@
<version>${findbugs.version}</version>
</dependency>
<dependency>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
+ <groupId>org.iq80.snappy</groupId>
+ <artifactId>snappy</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
[4/4] phoenix git commit: PHOENIX-1468 Add org.iq80.snappy to client
and server assemblies
Posted by ap...@apache.org.
PHOENIX-1468 Add org.iq80.snappy to client and server assemblies
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ebfebec8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ebfebec8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ebfebec8
Branch: refs/heads/3.0
Commit: ebfebec8139a7c61143770d69d57ef2feacf68b3
Parents: 3914103
Author: Andrew Purtell <ap...@apache.org>
Authored: Wed Nov 19 13:35:17 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 21 10:57:42 2014 -0800
----------------------------------------------------------------------
phoenix-assembly/src/build/client.xml | 1 +
phoenix-assembly/src/build/server.xml | 1 +
pom.xml | 26 ++++++++++++++++++++++++++
3 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ebfebec8/phoenix-assembly/src/build/client.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/client.xml b/phoenix-assembly/src/build/client.xml
index 2b71ce2..f2dd389 100644
--- a/phoenix-assembly/src/build/client.xml
+++ b/phoenix-assembly/src/build/client.xml
@@ -46,6 +46,7 @@
<include>jline:jline</include>
<include>sqlline:sqlline</include>
<include>org.apache.hbase:hbase*</include>
+ <include>org.iq80.snappy:snappy</include>
</includes>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ebfebec8/phoenix-assembly/src/build/server.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server.xml b/phoenix-assembly/src/build/server.xml
index 3329975..ecce920 100644
--- a/phoenix-assembly/src/build/server.xml
+++ b/phoenix-assembly/src/build/server.xml
@@ -36,6 +36,7 @@
<includes>
<include>org.apache.phoenix:phoenix-core</include>
<include>org.apache.phoenix:phoenix-hadoop*</include>
+ <include>org.iq80.snappy:snappy</include>
</includes>
</dependencySet>
</dependencySets>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ebfebec8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 011bea0..fd8a415 100644
--- a/pom.xml
+++ b/pom.xml
@@ -442,6 +442,12 @@
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -456,6 +462,10 @@
<groupId>org.jruby</groupId>
<artifactId>jruby-complete</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -477,6 +487,12 @@
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>${flume.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>net.sourceforge.findbugs</groupId>
@@ -617,6 +633,10 @@
<groupId>oro</groupId>
<artifactId>oro</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -661,6 +681,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
[2/4] phoenix git commit: PHOENIX-1455 Replace org.xerial.snappy with
org.iq80.snappy pure Java snappy implementation
Posted by ap...@apache.org.
PHOENIX-1455 Replace org.xerial.snappy with org.iq80.snappy pure Java snappy implementation
Conflicts:
phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3da2d2b3
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3da2d2b3
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3da2d2b3
Branch: refs/heads/4.0
Commit: 3da2d2b395e2147884070518eadcb2b8eb92eab7
Parents: a586723
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Nov 17 18:05:33 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Nov 21 10:56:15 2014 -0800
----------------------------------------------------------------------
phoenix-core/pom.xml | 6 ++---
.../DistinctValueWithCountClientAggregator.java | 17 ++++++++++----
.../DistinctValueWithCountServerAggregator.java | 24 ++++++--------------
.../apache/phoenix/join/HashCacheClient.java | 3 ++-
.../apache/phoenix/join/HashCacheFactory.java | 15 ++++++++----
pom.xml | 14 +++++++++---
6 files changed, 45 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3da2d2b3/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index d9293be..8173f35 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -276,8 +276,8 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
+ <groupId>org.iq80.snappy</groupId>
+ <artifactId>snappy</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
@@ -399,4 +399,4 @@
<artifactId>hadoop-minicluster</artifactId>
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3da2d2b3/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
index f29f46a..56ca000 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java
@@ -35,6 +35,7 @@ import org.apache.phoenix.schema.PDataType;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.tuple.SingleKeyValueTuple;
import org.apache.phoenix.schema.tuple.Tuple;
+import org.iq80.snappy.Snappy;
/**
* Client side Aggregator which will aggregate data and find distinct values with number of occurrences for each.
@@ -59,14 +60,20 @@ public abstract class DistinctValueWithCountClientAggregator extends BaseAggrega
PDataType resultDataType = getResultDataType();
cachedResult = resultDataType.toObject(ptr, resultDataType, sortOrder);
} else {
- InputStream is = new ByteArrayInputStream(ptr.get(), ptr.getOffset() + 1, ptr.getLength() - 1);
+ InputStream is;
try {
if (Bytes.equals(ptr.get(), ptr.getOffset(), 1, DistinctValueWithCountServerAggregator.COMPRESS_MARKER,
0, 1)) {
- InputStream decompressionStream = DistinctValueWithCountServerAggregator.COMPRESS_ALGO
- .createDecompressionStream(is,
- DistinctValueWithCountServerAggregator.COMPRESS_ALGO.getDecompressor(), 0);
- is = decompressionStream;
+ // This reads the uncompressed length from the front of the compressed input
+ int uncompressedLength = Snappy.getUncompressedLength(ptr.get(), ptr.getOffset() + 1);
+ byte[] uncompressed = new byte[uncompressedLength];
+ // This will throw CorruptionException, a RuntimeException if the snappy data is invalid.
+ // We're making a RuntimeException out of a checked IOException below so assume it's ok
+ // to let any CorruptionException escape.
+ Snappy.uncompress(ptr.get(), ptr.getOffset() + 1, ptr.getLength() - 1, uncompressed, 0);
+ is = new ByteArrayInputStream(uncompressed, 0, uncompressedLength);
+ } else {
+ is = new ByteArrayInputStream(ptr.get(), ptr.getOffset() + 1, ptr.getLength() - 1);
}
DataInputStream in = new DataInputStream(is);
int mapSize = WritableUtils.readVInt(in);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3da2d2b3/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
index 05b4abe..70de959 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java
@@ -17,16 +17,12 @@
*/
package org.apache.phoenix.expression.aggregator;
-import java.io.ByteArrayOutputStream;
-import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.compress.Compression;
-import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.query.QueryServices;
@@ -39,6 +35,8 @@ import org.apache.phoenix.util.SizedUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.iq80.snappy.Snappy;
+
/**
* Server side Aggregator which will aggregate data and find distinct values with number of occurrences for each.
*
@@ -49,7 +47,6 @@ public class DistinctValueWithCountServerAggregator extends BaseAggregator {
private static final Logger LOG = LoggerFactory.getLogger(DistinctValueWithCountServerAggregator.class);
public static final int DEFAULT_ESTIMATED_DISTINCT_VALUES = 10000;
public static final byte[] COMPRESS_MARKER = new byte[] { (byte)1 };
- public static final Algorithm COMPRESS_ALGO = Compression.Algorithm.SNAPPY;
private int compressThreshold;
private byte[] buffer = null;
@@ -100,18 +97,11 @@ public class DistinctValueWithCountServerAggregator extends BaseAggregator {
}
if (serializationSize > compressThreshold) {
// The size for the map serialization is above the threshold. We will do the Snappy compression here.
- ByteArrayOutputStream compressedByteStream = new ByteArrayOutputStream();
- try {
- compressedByteStream.write(COMPRESS_MARKER);
- OutputStream compressionStream = COMPRESS_ALGO.createCompressionStream(compressedByteStream,
- COMPRESS_ALGO.getCompressor(), 0);
- compressionStream.write(buffer, 1, buffer.length - 1);
- compressionStream.flush();
- ptr.set(compressedByteStream.toByteArray(), 0, compressedByteStream.size());
- return true;
- } catch (Exception e) {
- LOG.error("Exception while Snappy compression of data.", e);
- }
+ byte[] compressed = new byte[COMPRESS_MARKER.length + Snappy.maxCompressedLength(buffer.length)];
+ System.arraycopy(COMPRESS_MARKER, 0, compressed, 0, COMPRESS_MARKER.length);
+ int compressedLen = Snappy.compress(buffer, 1, buffer.length - 1, compressed, COMPRESS_MARKER.length);
+ ptr.set(compressed, 0, compressedLen + 1);
+ return true;
}
ptr.set(buffer, 0, offset);
return true;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3da2d2b3/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
index e2f57df..6494603 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheClient.java
@@ -39,7 +39,8 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
import org.apache.phoenix.util.TupleUtil;
-import org.xerial.snappy.Snappy;
+
+import org.iq80.snappy.Snappy;
/**
*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3da2d2b3/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
index cfe064c..f1f6e22 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/join/HashCacheFactory.java
@@ -36,6 +36,7 @@ import net.jcip.annotations.Immutable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;
+
import org.apache.phoenix.cache.HashCache;
import org.apache.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -50,7 +51,9 @@ import org.apache.phoenix.util.ResultUtil;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.SizedUtil;
import org.apache.phoenix.util.TupleUtil;
-import org.xerial.snappy.Snappy;
+
+import org.iq80.snappy.CorruptionException;
+import org.iq80.snappy.Snappy;
public class HashCacheFactory implements ServerCacheFactory {
@@ -68,11 +71,13 @@ public class HashCacheFactory implements ServerCacheFactory {
@Override
public Closeable newCache(ImmutableBytesWritable cachePtr, MemoryChunk chunk) throws SQLException {
try {
- int size = Snappy.uncompressedLength(cachePtr.get());
- byte[] uncompressed = new byte[size];
- Snappy.uncompress(cachePtr.get(), 0, cachePtr.getLength(), uncompressed, 0);
+ // This reads the uncompressed length from the front of the compressed input
+ int uncompressedLen = Snappy.getUncompressedLength(cachePtr.get(), cachePtr.getOffset());
+ byte[] uncompressed = new byte[uncompressedLen];
+ Snappy.uncompress(cachePtr.get(), cachePtr.getOffset(), cachePtr.getLength(),
+ uncompressed, 0);
return new HashCacheImpl(uncompressed, chunk);
- } catch (IOException e) {
+ } catch (CorruptionException e) {
throw ServerUtil.parseServerException(e);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3da2d2b3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a9b0118..39bb7c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,6 +45,14 @@
<enabled>true</enabled>
</snapshots>
</repository>
+ <repository>
+ <id>sonatype-nexus-snapshots</id>
+ <name>Sonatype Nexus Snapshots</name>
+ <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
</repositories>
<parent>
@@ -89,7 +97,7 @@
<flume.version>1.4.0</flume.version>
<findbugs.version>1.3.2</findbugs.version>
<jline.version>2.11</jline.version>
- <snappy.version>1.1.0.1</snappy.version>
+ <snappy.version>0.3</snappy.version>
<netty.version>3.6.6.Final</netty.version>
<commons-codec.version>1.7</commons-codec.version>
<htrace.version>2.04</htrace.version>
@@ -533,8 +541,8 @@
<version>${findbugs.version}</version>
</dependency>
<dependency>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
+ <groupId>org.iq80.snappy</groupId>
+ <artifactId>snappy</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>