You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2019/11/18 22:26:27 UTC
[pulsar-client-go] branch master updated: Updated zstd go library
to avoid cgo dependency for compression (#102)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git
The following commit(s) were added to refs/heads/master by this push:
new a0d96c8 Updated zstd go library to avoid cgo dependency for compression (#102)
a0d96c8 is described below
commit a0d96c89ad5dc2091f5e28c4386f73ebddf99a4a
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Nov 18 14:26:20 2019 -0800
Updated zstd go library to avoid cgo dependency for compression (#102)
---
go.mod | 6 +---
go.sum | 15 ++--------
pkg/compression/zstd.go | 45 +++++++++++++++++------------
pkg/compression/zstd_native.go | 64 ------------------------------------------
4 files changed, 30 insertions(+), 100 deletions(-)
diff --git a/go.mod b/go.mod
index dddb428..339748e 100644
--- a/go.mod
+++ b/go.mod
@@ -3,16 +3,12 @@ module github.com/apache/pulsar-client-go
go 1.12
require (
- github.com/DataDog/zstd v1.4.0 // indirect
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b
- github.com/cespare/xxhash v1.1.0 // indirect
- github.com/deckarep/golang-set v1.7.1
github.com/golang/protobuf v1.3.1
github.com/google/go-cmp v0.3.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
- github.com/klauspost/compress v1.6.2
- github.com/klauspost/cpuid v1.2.1 // indirect
+ github.com/klauspost/compress v1.9.2
github.com/pierrec/lz4 v2.0.5+incompatible
github.com/pkg/errors v0.8.1
github.com/sirupsen/logrus v1.4.1
diff --git a/go.sum b/go.sum
index a1ccf2f..c3fdcf6 100644
--- a/go.sum
+++ b/go.sum
@@ -1,28 +1,18 @@
-github.com/DataDog/zstd v1.4.0 h1:vhoV+DUHnRZdKW1i5UMjAk2G4JY8wN4ayRfYDNdEhwo=
-github.com/DataDog/zstd v1.4.0/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttPGEjR0tPH1xaWWoKBEg9Q1THAj2h3I=
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
-github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/klauspost/compress v1.6.2 h1:D9kM6nOc1x+yA/DW/k81uG1xdmwqCMQ/A266P1edQEw=
-github.com/klauspost/compress v1.6.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY=
+github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
@@ -33,7 +23,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
diff --git a/pkg/compression/zstd.go b/pkg/compression/zstd.go
index 529020c..b385c34 100644
--- a/pkg/compression/zstd.go
+++ b/pkg/compression/zstd.go
@@ -15,36 +15,45 @@
// specific language governing permissions and limitations
// under the License.
-// +build cgo
-
-// If CGO is enabled, use ZSTD library that links with official
-// C based zstd. This can perform both compression and decompression.
-
package compression
import (
- zstd "github.com/valyala/gozstd"
+ "bytes"
+ "github.com/klauspost/compress/zstd"
+ "github.com/pkg/errors"
)
-// NewZStdProvider returns a Provider interface.
-func NewZStdProvider() Provider {
- return newCGoZStdProvider()
+type zstdProvider struct {
+ encoder *zstd.Encoder
}
-type zstdProvider struct{}
-
-func newCGoZStdProvider() Provider {
- return &zstdProvider{}
+func NewZStdProvider() Provider {
+ p := &zstdProvider{}
+ p.encoder, _ = zstd.NewWriter(nil)
+ return p
}
-func (zstdProvider) CanCompress() bool {
+func (p *zstdProvider) CanCompress() bool {
return true
}
-func (zstdProvider) Compress(data []byte) []byte {
- return zstd.Compress(nil, data)
+func (p *zstdProvider) Compress(data []byte) []byte {
+ return p.encoder.EncodeAll(data, []byte{})
}
-func (zstdProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) {
- return zstd.Decompress(nil, compressedData)
+func (p* zstdProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) {
+ d, err := zstd.NewReader(bytes.NewReader(compressedData))
+ if err != nil {
+ return nil, err
+ }
+
+ uncompressed := make([]byte, originalSize)
+ size, err := d.Read(uncompressed)
+ if err != nil {
+ return nil, err
+ } else if size != originalSize {
+ return nil, errors.New("Invalid uncompressed size")
+ } else {
+ return uncompressed, nil
+ }
}
diff --git a/pkg/compression/zstd_native.go b/pkg/compression/zstd_native.go
deleted file mode 100644
index fa3649e..0000000
--- a/pkg/compression/zstd_native.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// +build !cgo
-
-// Pure GO ZStd library only supports decompression
-
-package compression
-
-import (
- "bytes"
- "github.com/klauspost/compress/zstd"
- "github.com/pkg/errors"
- log "github.com/sirupsen/logrus"
-)
-
-// Pure GO ZStd library only supports decompression
-
-type zstdProvider struct {
-}
-
-func NewZStdProvider() Provider {
- return &zstdProvider{}
-}
-
-func (zstdProvider) CanCompress() bool {
- return false
-}
-
-func (zstdProvider) Compress(data []byte) []byte {
- log.Panic("ZSTD compression is not supported if CGo is disabled")
- return nil
-}
-
-func (zstdProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) {
- d, err := zstd.NewReader(bytes.NewReader(compressedData))
- if err != nil {
- return nil, err
- }
-
- uncompressed := make([]byte, originalSize)
- size, err := d.Read(uncompressed)
- if err != nil {
- return nil, err
- } else if size != originalSize {
- return nil, errors.New("Invalid uncompressed size")
- } else {
- return uncompressed, nil
- }
-}