You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@thrift.apache.org by ns...@apache.org on 2015/12/10 18:16:13 UTC
[1/4] thrift git commit: THRIFT-3433 Doubles aren't interpreted
correctly Client: Haskell Patch: Nobuaki Sukegawa
Repository: thrift
Updated Branches:
refs/heads/master 0156aaed1 -> dfb689643
THRIFT-3433 Doubles aren't interpreted correctly
Client: Haskell
Patch: Nobuaki Sukegawa
This closes #736
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/7c7d679a
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/7c7d679a
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/7c7d679a
Branch: refs/heads/master
Commit: 7c7d679a127ed5157464b061a7f9bfd40ad2f1fa
Parents: 4f6138b
Author: Nobuaki Sukegawa <ns...@apache.org>
Authored: Wed Dec 9 03:22:35 2015 +0900
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:18:01 2015 +0900
----------------------------------------------------------------------
lib/hs/CMakeLists.txt | 28 ++++++++---
lib/hs/LICENSE | 0
lib/hs/Makefile.am | 4 ++
lib/hs/README.md | 0
lib/hs/TODO | 0
lib/hs/Thrift.cabal | 8 ++++
lib/hs/src/Thrift/Protocol.hs | 22 +++++----
lib/hs/src/Thrift/Protocol/Compact.hs | 10 ++--
lib/hs/src/Thrift/Transport/Memory.hs | 77 ++++++++++++++++++++++++++++++
lib/hs/test/BinarySpec.hs | 68 ++++++++++++++++++++++++++
lib/hs/test/CompactSpec.hs | 58 ++++++++++++++++++++++
lib/hs/test/Spec.hs | 36 ++++++++++++++
12 files changed, 291 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/lib/hs/CMakeLists.txt b/lib/hs/CMakeLists.txt
index 37ea288..b5d1d20 100644
--- a/lib/hs/CMakeLists.txt
+++ b/lib/hs/CMakeLists.txt
@@ -36,6 +36,15 @@ set(haskell_sources
Thrift.cabal
)
+if(BUILD_TESTING)
+ list(APPEND haskell_soruces
+ test/Spec.hs
+ test/BinarySpec.hs
+ test/CompactSpec.hs
+ )
+ set(hs_enable_test "--enable-tests")
+endif()
+
set(haskell_artifacts thrift_cabal.stamp)
# Adding *.hi files so that any missing file triggers the build
foreach(SRC ${haskell_sources})
@@ -48,18 +57,19 @@ foreach(SRC ${haskell_sources})
endif()
endforeach()
-if (CMAKE_BUILD_TYPE STREQUAL "Debug")
- set(hs_optimize -O0)
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+ set(hs_optimize -O0)
elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
- set(hs_optimize -O1)
+ set(hs_optimize -O1)
endif()
add_custom_command(
OUTPUT ${haskell_artifacts}
COMMAND ${CABAL} update
# Build dependencies first without --builddir, otherwise it fails.
- COMMAND ${CABAL} install --only-dependencies
- COMMAND ${CABAL} configure ${hs_optimize}
+ COMMAND ${CABAL} install --only-dependencies ${hs_enable_test}
+ COMMAND ${CABAL} configure ${hs_optimize} ${hs_enable_test} --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
+ COMMAND ${CABAL} build --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
COMMAND ${CABAL} install --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/thrift_cabal.stamp
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -70,7 +80,13 @@ add_custom_target(haskell_library ALL
DEPENDS ${haskell_artifacts})
if(BUILD_TESTING)
- add_test(NAME CabalCheck
+ add_test(NAME HaskellCabalCheck
COMMAND ${CABAL} check
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+ add_test(NAME HaskellCabalTest
+ # Cabal fails to find built executable when --builddir is specified.
+ # So we invoke the executable directly.
+ # COMMAND ${CABAL} test --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
+ # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+ COMMAND dist/build/spec/spec)
endif()
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/LICENSE
----------------------------------------------------------------------
diff --git a/lib/hs/LICENSE b/lib/hs/LICENSE
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/Makefile.am
----------------------------------------------------------------------
diff --git a/lib/hs/Makefile.am b/lib/hs/Makefile.am
old mode 100755
new mode 100644
index 45529c7..543381f
--- a/lib/hs/Makefile.am
+++ b/lib/hs/Makefile.am
@@ -42,3 +42,7 @@ maintainer-clean-local:
check-local:
$(CABAL) check
+ $(CABAL) install --only-dependencies --enable-tests
+ $(CABAL) configure --enable-tests
+ $(CABAL) build
+ $(CABAL) test
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/README.md
----------------------------------------------------------------------
diff --git a/lib/hs/README.md b/lib/hs/README.md
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/TODO
----------------------------------------------------------------------
diff --git a/lib/hs/TODO b/lib/hs/TODO
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/Thrift.cabal
----------------------------------------------------------------------
diff --git a/lib/hs/Thrift.cabal b/lib/hs/Thrift.cabal
old mode 100755
new mode 100644
index 5610a5c..f0a09aa
--- a/lib/hs/Thrift.cabal
+++ b/lib/hs/Thrift.cabal
@@ -59,6 +59,7 @@ Library
Thrift.Transport.Handle,
Thrift.Transport.HttpClient,
Thrift.Transport.IOBuffer,
+ Thrift.Transport.Memory,
Thrift.Types
Extensions:
DeriveDataTypeable,
@@ -70,3 +71,10 @@ Library
RecordWildCards,
ScopedTypeVariables,
TypeSynonymInstances
+
+Test-Suite spec
+ Type: exitcode-stdio-1.0
+ Hs-Source-Dirs: test
+ Ghc-Options: -Wall
+ main-is: Spec.hs
+ Build-Depends: base, thrift, hspec, QuickCheck, bytestring >= 0.10, unordered-containers
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/src/Thrift/Protocol.hs
----------------------------------------------------------------------
diff --git a/lib/hs/src/Thrift/Protocol.hs b/lib/hs/src/Thrift/Protocol.hs
index ea58642..31e48b5 100644
--- a/lib/hs/src/Thrift/Protocol.hs
+++ b/lib/hs/src/Thrift/Protocol.hs
@@ -29,6 +29,7 @@ module Thrift.Protocol
, versionMask
, version1
, bsToDouble
+ , bsToDoubleLE
) where
import Control.Exception
@@ -119,18 +120,22 @@ handleEOF = const $ return mempty
-- therefore the behavior of this function varies based on whether the local
-- machine is big endian or little endian.
bsToDouble :: BS.ByteString -> Double
-bsToDouble bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBs
- where
+bsToDoubleLE :: BS.ByteString -> Double
#if __BYTE_ORDER == __LITTLE_ENDIAN
- castBs chrPtr = do
- w <- peek (castPtr chrPtr)
- poke (castPtr chrPtr) (byteSwap w)
- peek (castPtr chrPtr)
+bsToDouble bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBsSwapped
+bsToDoubleLE bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBs
#else
- castBs = peek . castPtr
+bsToDouble bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBs
+bsToDoubleLE bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBsSwapped
#endif
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+
+castBsSwapped chrPtr = do
+ w <- peek (castPtr chrPtr)
+ poke (castPtr chrPtr) (byteSwap w)
+ peek (castPtr chrPtr)
+castBs = peek . castPtr
+
-- | Swap endianness of a 64-bit word
byteSwap :: Word64 -> Word64
byteSwap w = (w `shiftL` 56 .&. 0xFF00000000000000) .|.
@@ -141,4 +146,3 @@ byteSwap w = (w `shiftL` 56 .&. 0xFF00000000000000) .|.
(w `shiftR` 24 .&. 0x0000000000FF0000) .|.
(w `shiftR` 40 .&. 0x000000000000FF00) .|.
(w `shiftR` 56 .&. 0x00000000000000FF)
-#endif
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/src/Thrift/Protocol/Compact.hs
----------------------------------------------------------------------
diff --git a/lib/hs/src/Thrift/Protocol/Compact.hs b/lib/hs/src/Thrift/Protocol/Compact.hs
index a329f4e..759466b 100644
--- a/lib/hs/src/Thrift/Protocol/Compact.hs
+++ b/lib/hs/src/Thrift/Protocol/Compact.hs
@@ -55,7 +55,7 @@ import qualified Data.Text.Lazy as LT
data CompactProtocol a = CompactProtocol a
-- ^ Constuct a 'CompactProtocol' with a 'Transport'
-protocolID, version, typeMask :: Int8
+protocolID, version, versionMask, typeMask, typeBits :: Word8
protocolID = 0x82 -- 1000 0010
version = 0x01
versionMask = 0x1f -- 0001 1111
@@ -69,8 +69,8 @@ instance Protocol CompactProtocol where
getTransport (CompactProtocol t) = t
writeMessageBegin p (n, t, s) = tWrite (getTransport p) $ toLazyByteString $
- B.int8 protocolID <>
- B.int8 ((version .&. versionMask) .|.
+ B.word8 protocolID <>
+ B.word8 ((version .&. versionMask) .|.
(((fromIntegral $ fromEnum t) `shiftL`
typeShiftAmount) .&. typeMask)) <>
buildVarint (i32ToZigZag s) <>
@@ -120,7 +120,7 @@ buildCompactValue (TByte b) = int8 b
buildCompactValue (TI16 i) = buildVarint $ i16ToZigZag i
buildCompactValue (TI32 i) = buildVarint $ i32ToZigZag i
buildCompactValue (TI64 i) = buildVarint $ i64ToZigZag i
-buildCompactValue (TDouble d) = doubleBE d
+buildCompactValue (TDouble d) = doubleLE d
buildCompactValue (TString s) = buildVarint len <> lazyByteString s
where
len = fromIntegral (LBS.length s) :: Word32
@@ -163,7 +163,7 @@ parseCompactValue T_BYTE = TByte . fromIntegral <$> P.anyWord8
parseCompactValue T_I16 = TI16 <$> parseVarint zigZagToI16
parseCompactValue T_I32 = TI32 <$> parseVarint zigZagToI32
parseCompactValue T_I64 = TI64 <$> parseVarint zigZagToI64
-parseCompactValue T_DOUBLE = TDouble . bsToDouble <$> P.take 8
+parseCompactValue T_DOUBLE = TDouble . bsToDoubleLE <$> P.take 8
parseCompactValue T_STRING = do
len :: Word32 <- parseVarint id
TString . LBS.fromStrict <$> P.take (fromIntegral len)
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/src/Thrift/Transport/Memory.hs
----------------------------------------------------------------------
diff --git a/lib/hs/src/Thrift/Transport/Memory.hs b/lib/hs/src/Thrift/Transport/Memory.hs
new file mode 100644
index 0000000..1c93af6
--- /dev/null
+++ b/lib/hs/src/Thrift/Transport/Memory.hs
@@ -0,0 +1,77 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+module Thrift.Transport.Memory
+ ( openMemoryBuffer
+ , MemoryBuffer(..)
+ ) where
+
+import Data.ByteString.Lazy.Builder
+import Data.Functor
+import Data.IORef
+import Data.Monoid
+import qualified Data.ByteString.Lazy as LBS
+
+import Thrift.Transport
+
+
+data MemoryBuffer = MemoryBuffer {
+ writeBuffer :: IORef Builder,
+ readBuffer :: IORef LBS.ByteString
+}
+
+openMemoryBuffer :: IO MemoryBuffer
+openMemoryBuffer = do
+ wbuf <- newIORef mempty
+ rbuf <- newIORef mempty
+ return MemoryBuffer {
+ writeBuffer = wbuf,
+ readBuffer = rbuf
+ }
+
+instance Transport MemoryBuffer where
+ tIsOpen = const $ return False
+ tClose = const $ return ()
+ tFlush trans = do
+ let wBuf = writeBuffer trans
+ wb <- readIORef wBuf
+ modifyIORef (readBuffer trans) $ \rb -> mappend rb $ toLazyByteString wb
+ writeIORef wBuf mempty
+
+ tRead _ 0 = return mempty
+ tRead trans n = do
+ let rbuf = readBuffer trans
+ rb <- readIORef rbuf
+ let len = fromIntegral $ LBS.length rb
+ if len == 0
+ then do
+ tFlush trans
+ rb2 <- readIORef (readBuffer trans)
+ if (fromIntegral $ LBS.length rb2) == 0
+ then return mempty
+ else tRead trans n
+ else do
+ let (ret, remain) = LBS.splitAt (fromIntegral n) rb
+ writeIORef rbuf remain
+ return ret
+
+ tPeek trans = (fmap fst . LBS.uncons) <$> readIORef (readBuffer trans)
+
+ tWrite trans v = do
+ modifyIORef (writeBuffer trans) (<> lazyByteString v)
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/test/BinarySpec.hs
----------------------------------------------------------------------
diff --git a/lib/hs/test/BinarySpec.hs b/lib/hs/test/BinarySpec.hs
new file mode 100644
index 0000000..5039610
--- /dev/null
+++ b/lib/hs/test/BinarySpec.hs
@@ -0,0 +1,68 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+module BinarySpec where
+
+import Test.Hspec
+import Test.Hspec.QuickCheck (prop)
+
+import qualified Data.ByteString.Lazy as LBS
+import qualified Data.ByteString.Lazy.Char8 as C
+
+import Thrift.Types
+import Thrift.Transport
+import Thrift.Transport.Memory
+import Thrift.Protocol
+import Thrift.Protocol.Binary
+
+spec :: Spec
+spec = do
+ describe "BinaryProtocol" $ do
+ describe "double" $ do
+ it "writes in big endian order" $ do
+ let val = 2 ** 53
+ trans <- openMemoryBuffer
+ let proto = BinaryProtocol trans
+ writeVal proto (TDouble val)
+ bin <- tRead trans 8
+ (LBS.unpack bin) `shouldBe`[67, 64, 0, 0, 0, 0, 0, 0]
+
+ it "reads in big endian order" $ do
+ let bin = LBS.pack [67, 64, 0, 0, 0, 0, 0, 0]
+ trans <- openMemoryBuffer
+ let proto = BinaryProtocol trans
+ tWrite trans bin
+ val <- readVal proto T_DOUBLE
+ val `shouldBe` (TDouble $ 2 ** 53)
+
+ prop "round trip" $ \val -> do
+ trans <- openMemoryBuffer
+ let proto = BinaryProtocol trans
+ writeVal proto $ TDouble val
+ val2 <- readVal proto T_DOUBLE
+ val2 `shouldBe` (TDouble val)
+
+ describe "string" $ do
+ it "writes" $ do
+ let val = C.pack "aaa"
+ trans <- openMemoryBuffer
+ let proto = BinaryProtocol trans
+ writeVal proto (TString val)
+ bin <- tRead trans 7
+ (LBS.unpack bin) `shouldBe` [0, 0, 0, 3, 97, 97, 97]
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/test/CompactSpec.hs
----------------------------------------------------------------------
diff --git a/lib/hs/test/CompactSpec.hs b/lib/hs/test/CompactSpec.hs
new file mode 100644
index 0000000..22708b4
--- /dev/null
+++ b/lib/hs/test/CompactSpec.hs
@@ -0,0 +1,58 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+module CompactSpec where
+
+import Test.Hspec
+import Test.Hspec.QuickCheck (prop)
+
+import qualified Data.ByteString.Lazy as LBS
+
+import Thrift.Types
+import Thrift.Transport
+import Thrift.Transport.Memory
+import Thrift.Protocol
+import Thrift.Protocol.Compact
+
+spec :: Spec
+spec = do
+ describe "CompactProtocol" $ do
+ describe "double" $ do
+ it "writes in little endian order" $ do
+ let val = 2 ** 53
+ trans <- openMemoryBuffer
+ let proto = CompactProtocol trans
+ writeVal proto (TDouble val)
+ bin <- tReadAll trans 8
+ (LBS.unpack bin) `shouldBe`[0, 0, 0, 0, 0, 0, 64, 67]
+
+ it "reads in little endian order" $ do
+ let bin = LBS.pack [0, 0, 0, 0, 0, 0, 64, 67]
+ trans <- openMemoryBuffer
+ let proto = CompactProtocol trans
+ tWrite trans bin
+ val <- readVal proto T_DOUBLE
+ val `shouldBe` (TDouble $ 2 ** 53)
+
+ prop "round trip" $ \val -> do
+ trans <- openMemoryBuffer
+ let proto = CompactProtocol trans
+ writeVal proto $ TDouble val
+ val2 <- readVal proto T_DOUBLE
+ val2 `shouldBe` (TDouble val)
http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/test/Spec.hs
----------------------------------------------------------------------
diff --git a/lib/hs/test/Spec.hs b/lib/hs/test/Spec.hs
new file mode 100644
index 0000000..0f5a816
--- /dev/null
+++ b/lib/hs/test/Spec.hs
@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+-- Our CI does not work well with auto discover.
+-- Need to add build-time PATH variable to hspec-discover dir from CMake
+-- or install hspec system-wide for the following to work.
+-- {-# OPTIONS_GHC -F -pgmF hspec-discover #-}
+
+import Test.Hspec
+
+import qualified BinarySpec
+import qualified CompactSpec
+
+main :: IO ()
+main = hspec spec
+
+spec :: Spec
+spec = do
+ describe "Binary" BinarySpec.spec
+ describe "Compact" CompactSpec.spec
[3/4] thrift git commit: THRIFT-3432 Add the TByteBuffer transport
for Java Client: Java Patch: Tom Lee
Posted by ns...@apache.org.
THRIFT-3432 Add the TByteBuffer transport for Java
Client: Java
Patch: Tom Lee
This closes #705
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/9d0f752a
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/9d0f752a
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/9d0f752a
Branch: refs/heads/master
Commit: 9d0f752abcd7b9435c245371d74a2086006a6417
Parents: 0156aae
Author: Tom Lee <gi...@tomlee.co>
Authored: Thu Nov 19 03:03:30 2015 -0800
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:18:01 2015 +0900
----------------------------------------------------------------------
.../apache/thrift/transport/TByteBuffer.java | 87 ++++++++++++++++++++
.../thrift/transport/TestTByteBuffer.java | 36 ++++++++
2 files changed, 123 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/9d0f752a/lib/java/src/org/apache/thrift/transport/TByteBuffer.java
----------------------------------------------------------------------
diff --git a/lib/java/src/org/apache/thrift/transport/TByteBuffer.java b/lib/java/src/org/apache/thrift/transport/TByteBuffer.java
new file mode 100644
index 0000000..a09f33d
--- /dev/null
+++ b/lib/java/src/org/apache/thrift/transport/TByteBuffer.java
@@ -0,0 +1,87 @@
+package org.apache.thrift.transport;
+
+import java.nio.BufferOverflowException;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+
+/**
+ * ByteBuffer-backed implementation of TTransport.
+ */
+public final class TByteBuffer extends TTransport {
+ private final ByteBuffer byteBuffer;
+
+ /**
+ * Creates a new TByteBuffer wrapping a given NIO ByteBuffer.
+ */
+ public TByteBuffer(ByteBuffer byteBuffer) {
+ this.byteBuffer = byteBuffer;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return true;
+ }
+
+ @Override
+ public void open() {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public int read(byte[] buf, int off, int len) throws TTransportException {
+ final int n = Math.min(byteBuffer.remaining(), len);
+ if (n > 0) {
+ try {
+ byteBuffer.get(buf, off, len);
+ } catch (BufferUnderflowException e) {
+ throw new TTransportException("Unexpected end of input buffer", e);
+ }
+ }
+ return n;
+ }
+
+ @Override
+ public void write(byte[] buf, int off, int len) throws TTransportException {
+ try {
+ byteBuffer.put(buf, off, len);
+ } catch (BufferOverflowException e) {
+ throw new TTransportException("Not enough room in output buffer", e);
+ }
+ }
+
+ /**
+ * Get the underlying NIO ByteBuffer.
+ */
+ public ByteBuffer getByteBuffer() {
+ return byteBuffer;
+ }
+
+ /**
+ * Convenience method to call clear() on the underlying NIO ByteBuffer.
+ */
+ public TByteBuffer clear() {
+ byteBuffer.clear();
+ return this;
+ }
+
+ /**
+ * Convenience method to call flip() on the underlying NIO ByteBuffer.
+ */
+ public TByteBuffer flip() {
+ byteBuffer.flip();
+ return this;
+ }
+
+ /**
+ * Convenience method to convert the underlying NIO ByteBuffer to a
+ * plain old byte array.
+ */
+ public byte[] toByteArray() {
+ final byte[] data = new byte[byteBuffer.remaining()];
+ byteBuffer.slice().get(data);
+ return data;
+ }
+}
http://git-wip-us.apache.org/repos/asf/thrift/blob/9d0f752a/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java
----------------------------------------------------------------------
diff --git a/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java b/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java
new file mode 100644
index 0000000..a73075b
--- /dev/null
+++ b/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java
@@ -0,0 +1,36 @@
+package org.apache.thrift.transport;
+
+import junit.framework.TestCase;
+import org.apache.commons.codec.Charsets;
+import org.apache.thrift.TException;
+
+import java.nio.ByteBuffer;
+
+public class TestTByteBuffer extends TestCase {
+ public void testReadWrite() throws Exception {
+ final TByteBuffer byteBuffer = new TByteBuffer(ByteBuffer.allocate(16));
+ byteBuffer.write("Hello World".getBytes(Charsets.UTF_8));
+ assertEquals("Hello World", new String(byteBuffer.flip().toByteArray(), Charsets.UTF_8));
+ }
+
+ public void testReuseReadWrite() throws Exception {
+ final TByteBuffer byteBuffer = new TByteBuffer(ByteBuffer.allocate(16));
+ byteBuffer.write("Hello World".getBytes(Charsets.UTF_8));
+ assertEquals("Hello World", new String(byteBuffer.flip().toByteArray(), Charsets.UTF_8));
+
+ byteBuffer.clear();
+
+ byteBuffer.write("Goodbye Horses".getBytes(Charsets.UTF_8));
+ assertEquals("Goodbye Horses", new String(byteBuffer.flip().toByteArray(), Charsets.UTF_8));
+ }
+
+ public void testOverflow() throws Exception {
+ final TByteBuffer byteBuffer = new TByteBuffer(ByteBuffer.allocate(4));
+ try {
+ byteBuffer.write("Hello World".getBytes(Charsets.UTF_8));
+ fail("Expected write operation to fail with TTransportException");
+ } catch (TTransportException e) {
+ assertEquals("Not enough room in output buffer", e.getMessage());
+ }
+ }
+}
[2/4] thrift git commit: THRIFT-3484 Consolidate temporary buffers in
Java's TCompactProtocol Client: Java Patch: Tom Lee
Posted by ns...@apache.org.
THRIFT-3484 Consolidate temporary buffers in Java's TCompactProtocol
Client: Java
Patch: Tom Lee
This closes #738
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/4f6138b7
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/4f6138b7
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/4f6138b7
Branch: refs/heads/master
Commit: 4f6138b7a2762f1937b49389f72b348736973e22
Parents: 9d0f752
Author: Tom Lee <gi...@tomlee.co>
Authored: Thu Dec 10 00:10:30 2015 -0800
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:18:01 2015 +0900
----------------------------------------------------------------------
.../thrift/protocol/TCompactProtocol.java | 46 ++++++++++----------
1 file changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/4f6138b7/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
----------------------------------------------------------------------
diff --git a/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java b/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
index 5973fcd..2a38810 100644
--- a/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
+++ b/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
@@ -37,6 +37,8 @@ import org.apache.thrift.transport.TTransport;
* and i64 fields you have, the more benefit you'll see.
*/
public class TCompactProtocol extends TProtocol {
+ private final static byte[] EMPTY_BYTES = new byte[0];
+ private final static ByteBuffer EMPTY_BUFFER = ByteBuffer.wrap(EMPTY_BYTES);
private final static long NO_LENGTH_LIMIT = -1;
@@ -144,6 +146,12 @@ public class TCompactProtocol extends TProtocol {
private final long containerLengthLimit_;
/**
+ * Temporary buffer used for various operations that would otherwise require a
+ * small allocation.
+ */
+ private final byte[] temp = new byte[10];
+
+ /**
* Create a TCompactProtocol.
*
* @param transport the TTransport object to read from or write to.
@@ -343,9 +351,8 @@ public class TCompactProtocol extends TProtocol {
* Write a double to the wire as 8 bytes.
*/
public void writeDouble(double dub) throws TException {
- byte[] data = new byte[]{0, 0, 0, 0, 0, 0, 0, 0};
- fixedLongToBytes(Double.doubleToLongBits(dub), data, 0);
- trans_.write(data);
+ fixedLongToBytes(Double.doubleToLongBits(dub), temp, 0);
+ trans_.write(temp, 0, 8);
}
/**
@@ -405,40 +412,38 @@ public class TCompactProtocol extends TProtocol {
* Write an i32 as a varint. Results in 1-5 bytes on the wire.
* TODO: make a permanent buffer like writeVarint64?
*/
- byte[] i32buf = new byte[5];
private void writeVarint32(int n) throws TException {
int idx = 0;
while (true) {
if ((n & ~0x7F) == 0) {
- i32buf[idx++] = (byte)n;
+ temp[idx++] = (byte)n;
// writeByteDirect((byte)n);
break;
// return;
} else {
- i32buf[idx++] = (byte)((n & 0x7F) | 0x80);
+ temp[idx++] = (byte)((n & 0x7F) | 0x80);
// writeByteDirect((byte)((n & 0x7F) | 0x80));
n >>>= 7;
}
}
- trans_.write(i32buf, 0, idx);
+ trans_.write(temp, 0, idx);
}
/**
* Write an i64 as a varint. Results in 1-10 bytes on the wire.
*/
- byte[] varint64out = new byte[10];
private void writeVarint64(long n) throws TException {
int idx = 0;
while (true) {
if ((n & ~0x7FL) == 0) {
- varint64out[idx++] = (byte)n;
+ temp[idx++] = (byte)n;
break;
} else {
- varint64out[idx++] = ((byte)((n & 0x7F) | 0x80));
+ temp[idx++] = ((byte)((n & 0x7F) | 0x80));
n >>>= 7;
}
}
- trans_.write(varint64out, 0, idx);
+ trans_.write(temp, 0, idx);
}
/**
@@ -476,10 +481,9 @@ public class TCompactProtocol extends TProtocol {
* Writes a byte without any possibility of all that field header nonsense.
* Used internally by other writing methods that know they need to write a byte.
*/
- private byte[] byteDirectBuffer = new byte[1];
private void writeByteDirect(byte b) throws TException {
- byteDirectBuffer[0] = b;
- trans_.write(byteDirectBuffer);
+ temp[0] = b;
+ trans_.write(temp, 0, 1);
}
/**
@@ -621,7 +625,6 @@ public class TCompactProtocol extends TProtocol {
return readByte() == Types.BOOLEAN_TRUE;
}
- byte[] byteRawBuf = new byte[1];
/**
* Read a single byte off the wire. Nothing interesting here.
*/
@@ -631,8 +634,8 @@ public class TCompactProtocol extends TProtocol {
b = trans_.getBuffer()[trans_.getBufferPosition()];
trans_.consumeBuffer(1);
} else {
- trans_.readAll(byteRawBuf, 0, 1);
- b = byteRawBuf[0];
+ trans_.readAll(temp, 0, 1);
+ b = temp[0];
}
return b;
}
@@ -662,9 +665,8 @@ public class TCompactProtocol extends TProtocol {
* No magic here - just read a double off the wire.
*/
public double readDouble() throws TException {
- byte[] longBits = new byte[8];
- trans_.readAll(longBits, 0, 8);
- return Double.longBitsToDouble(bytesToLong(longBits));
+ trans_.readAll(temp, 0, 8);
+ return Double.longBitsToDouble(bytesToLong(temp));
}
/**
@@ -697,7 +699,7 @@ public class TCompactProtocol extends TProtocol {
public ByteBuffer readBinary() throws TException {
int length = readVarint32();
checkStringReadLength(length);
- if (length == 0) return ByteBuffer.wrap(new byte[0]);
+ if (length == 0) return EMPTY_BUFFER;
if (trans_.getBytesRemainingInBuffer() >= length) {
ByteBuffer bb = ByteBuffer.wrap(trans_.getBuffer(), trans_.getBufferPosition(), length);
@@ -714,7 +716,7 @@ public class TCompactProtocol extends TProtocol {
* Read a byte[] of a known length from the wire.
*/
private byte[] readBinary(int length) throws TException {
- if (length == 0) return new byte[0];
+ if (length == 0) return EMPTY_BYTES;
byte[] buf = new byte[length];
trans_.readAll(buf, 0, length);
[4/4] thrift git commit: THRIFT-3483 Incorrect empty binary handling
introduced by THRIFT-3359 Client: C++, Node.js Patch: Nobuaki Sukegawa
Posted by ns...@apache.org.
THRIFT-3483 Incorrect empty binary handling introduced by THRIFT-3359
Client: C++, Node.js
Patch: Nobuaki Sukegawa
This closes #737
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/dfb68964
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/dfb68964
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/dfb68964
Branch: refs/heads/master
Commit: dfb6896436904d0fbe1f762de898a3c68c989aa9
Parents: 7c7d679
Author: Nobuaki Sukegawa <ns...@apache.org>
Authored: Wed Dec 9 22:09:26 2015 +0900
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:19:09 2015 +0900
----------------------------------------------------------------------
lib/cpp/src/thrift/protocol/TJSONProtocol.cpp | 8 +++++---
lib/nodejs/lib/thrift/binary_protocol.js | 2 +-
lib/nodejs/lib/thrift/compact_protocol.js | 2 +-
test/cpp/src/TestClient.cpp | 14 ++++++++++++++
4 files changed, 21 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
----------------------------------------------------------------------
diff --git a/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp b/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
index f1370bb..9569188 100644
--- a/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
+++ b/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
@@ -801,9 +801,11 @@ uint32_t TJSONProtocol::readJSONBase64(std::string& str) {
uint32_t len = static_cast<uint32_t>(tmp.length());
str.clear();
// Ignore padding
- uint32_t bound = len >= 2 ? len - 2 : 0;
- for (uint32_t i = len - 1; i >= bound && b[i] == '='; --i) {
- --len;
+ if (len >= 2) {
+ uint32_t bound = len - 2;
+ for (uint32_t i = len - 1; i >= bound && b[i] == '='; --i) {
+ --len;
+ }
}
while (len >= 4) {
base64_decode(b, 4);
http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/lib/nodejs/lib/thrift/binary_protocol.js
----------------------------------------------------------------------
diff --git a/lib/nodejs/lib/thrift/binary_protocol.js b/lib/nodejs/lib/thrift/binary_protocol.js
index 25e5634..6d3918e 100644
--- a/lib/nodejs/lib/thrift/binary_protocol.js
+++ b/lib/nodejs/lib/thrift/binary_protocol.js
@@ -276,7 +276,7 @@ TBinaryProtocol.prototype.readDouble = function() {
TBinaryProtocol.prototype.readBinary = function() {
var len = this.readI32();
if (len === 0) {
- return new Buffer();
+ return new Buffer(0);
}
if (len < 0) {
http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/lib/nodejs/lib/thrift/compact_protocol.js
----------------------------------------------------------------------
diff --git a/lib/nodejs/lib/thrift/compact_protocol.js b/lib/nodejs/lib/thrift/compact_protocol.js
index deecf48..b0e9148 100644
--- a/lib/nodejs/lib/thrift/compact_protocol.js
+++ b/lib/nodejs/lib/thrift/compact_protocol.js
@@ -761,7 +761,7 @@ TCompactProtocol.prototype.readDouble = function() {
TCompactProtocol.prototype.readBinary = function() {
var size = this.readVarint32();
if (size === 0) {
- return new Buffer();
+ return new Buffer(0);
}
if (size < 0) {
http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/test/cpp/src/TestClient.cpp
----------------------------------------------------------------------
diff --git a/test/cpp/src/TestClient.cpp b/test/cpp/src/TestClient.cpp
index a6773d4..0ad789d 100644
--- a/test/cpp/src/TestClient.cpp
+++ b/test/cpp/src/TestClient.cpp
@@ -463,6 +463,20 @@ int main(int argc, char** argv) {
/**
* BINARY TEST
*/
+ printf("testBinary(empty)\n");
+ try {
+ string bin_result;
+ testClient.testBinary(bin_result, string());
+ if (!bin_result.empty()) {
+ printf("*** FAILED ***\n");
+ printf("invalid length: %lu\n", static_cast<long unsigned int>(bin_result.size()));
+ return_code |= ERR_BASETYPES;
+ }
+ } catch (exception& ex) {
+ printf("}\n*** FAILED ***\n");
+ printf("%s\n", ex.what());
+ return_code |= ERR_BASETYPES;
+ }
printf("testBinary([-128..127]) = {");
const char bin_data[256]
= {-128, -127, -126, -125, -124, -123, -122, -121, -120, -119, -118, -117, -116, -115, -114,