You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by rx...@apache.org on 2020/06/09 01:14:13 UTC
[pulsar-client-go] branch master updated: When CGO is enabled,
use C version of ZStd (#270)
This is an automated email from the ASF dual-hosted git repository.
rxl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git
The following commit(s) were added to refs/heads/master by this push:
new 1ad856e When CGO is enabled, use C version of ZStd (#270)
1ad856e is described below
commit 1ad856e67e156061d4389a9eb07f0c2bf53e9417
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Mon Jun 8 18:14:07 2020 -0700
When CGO is enabled, use C version of ZStd (#270)
* When CGO is enabled, use C version of ZStd
* Fixed formatting
---
go.mod | 3 +-
go.sum | 3 ++
pulsar/internal/compression/zstd.go | 2 ++
.../internal/compression/{zstd.go => zstd_cgo.go} | 33 +++++++++-------------
4 files changed, 21 insertions(+), 20 deletions(-)
diff --git a/go.mod b/go.mod
index 911facf..706e382 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ require (
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b
github.com/golang/protobuf v1.3.1
github.com/inconshreveable/mousetrap v1.0.0 // indirect
- github.com/klauspost/compress v1.10.5
+ github.com/klauspost/compress v1.10.8
github.com/pierrec/lz4 v2.0.5+incompatible
github.com/pkg/errors v0.8.1
github.com/sirupsen/logrus v1.4.1
@@ -16,4 +16,5 @@ require (
github.com/spf13/pflag v1.0.3 // indirect
github.com/stretchr/testify v1.4.0
github.com/yahoo/athenz v1.8.55
+ github.com/valyala/gozstd v1.7.0
)
diff --git a/go.sum b/go.sum
index 305907f..1008e02 100644
--- a/go.sum
+++ b/go.sum
@@ -22,6 +22,7 @@ github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLV
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc=
github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
@@ -47,6 +48,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ=
+github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
github.com/yahoo/athenz v1.8.55 h1:xGhxN3yLq334APyn0Zvcc+aqu78Q7BBhYJevM3EtTW0=
github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
diff --git a/pulsar/internal/compression/zstd.go b/pulsar/internal/compression/zstd.go
index 13b681f..ccf84ba 100644
--- a/pulsar/internal/compression/zstd.go
+++ b/pulsar/internal/compression/zstd.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !cgo
+
package compression
import (
diff --git a/pulsar/internal/compression/zstd.go b/pulsar/internal/compression/zstd_cgo.go
similarity index 57%
copy from pulsar/internal/compression/zstd.go
copy to pulsar/internal/compression/zstd_cgo.go
index 13b681f..3f74b93 100644
--- a/pulsar/internal/compression/zstd.go
+++ b/pulsar/internal/compression/zstd_cgo.go
@@ -15,37 +15,32 @@
// specific language governing permissions and limitations
// under the License.
+// +build cgo
+
+// If CGO is enabled, use ZSTD library that links with official
+// C based zstd which provides better performance compared with
+// respect to the native Go implementation of ZStd.
+
package compression
import (
- "github.com/klauspost/compress/zstd"
- "github.com/pkg/errors"
+ zstd "github.com/valyala/gozstd"
)
-type zstdProvider struct {
- encoder *zstd.Encoder
- decoder *zstd.Decoder
-}
+type zstdCGoProvider struct{}
func NewZStdProvider() Provider {
- p := &zstdProvider{}
- p.encoder, _ = zstd.NewWriter(nil)
- p.decoder, _ = zstd.NewReader(nil)
- return p
+ return &zstdCGoProvider{}
}
-func (p *zstdProvider) CanCompress() bool {
+func (*zstdCGoProvider) CanCompress() bool {
return true
}
-func (p *zstdProvider) Compress(data []byte) []byte {
- return p.encoder.EncodeAll(data, []byte{})
+func (*zstdCGoProvider) Compress(data []byte) []byte {
+ return zstd.Compress(nil, data)
}
-func (p *zstdProvider) Decompress(compressedData []byte, originalSize int) (dst []byte, err error) {
- dst, err = p.decoder.DecodeAll(compressedData, nil)
- if err == nil && len(dst) != originalSize {
- return nil, errors.New("Invalid uncompressed size")
- }
- return
+func (*zstdCGoProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) {
+ return zstd.Decompress(nil, compressedData)
}