You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@thrift.apache.org by ns...@apache.org on 2015/12/10 18:16:13 UTC

[1/4] thrift git commit: THRIFT-3433 Doubles aren't interpreted correctly Client: Haskell Patch: Nobuaki Sukegawa

Repository: thrift
Updated Branches:
  refs/heads/master 0156aaed1 -> dfb689643


THRIFT-3433 Doubles aren't interpreted correctly
Client: Haskell
Patch: Nobuaki Sukegawa

This closes #736


Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/7c7d679a
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/7c7d679a
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/7c7d679a

Branch: refs/heads/master
Commit: 7c7d679a127ed5157464b061a7f9bfd40ad2f1fa
Parents: 4f6138b
Author: Nobuaki Sukegawa <ns...@apache.org>
Authored: Wed Dec 9 03:22:35 2015 +0900
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:18:01 2015 +0900

----------------------------------------------------------------------
 lib/hs/CMakeLists.txt                 | 28 ++++++++---
 lib/hs/LICENSE                        |  0
 lib/hs/Makefile.am                    |  4 ++
 lib/hs/README.md                      |  0
 lib/hs/TODO                           |  0
 lib/hs/Thrift.cabal                   |  8 ++++
 lib/hs/src/Thrift/Protocol.hs         | 22 +++++----
 lib/hs/src/Thrift/Protocol/Compact.hs | 10 ++--
 lib/hs/src/Thrift/Transport/Memory.hs | 77 ++++++++++++++++++++++++++++++
 lib/hs/test/BinarySpec.hs             | 68 ++++++++++++++++++++++++++
 lib/hs/test/CompactSpec.hs            | 58 ++++++++++++++++++++++
 lib/hs/test/Spec.hs                   | 36 ++++++++++++++
 12 files changed, 291 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/lib/hs/CMakeLists.txt b/lib/hs/CMakeLists.txt
index 37ea288..b5d1d20 100644
--- a/lib/hs/CMakeLists.txt
+++ b/lib/hs/CMakeLists.txt
@@ -36,6 +36,15 @@ set(haskell_sources
     Thrift.cabal
 )
 
+if(BUILD_TESTING)
+    list(APPEND haskell_soruces
+        test/Spec.hs
+        test/BinarySpec.hs
+        test/CompactSpec.hs
+    )
+    set(hs_enable_test "--enable-tests")
+endif()
+
 set(haskell_artifacts thrift_cabal.stamp)
 # Adding *.hi files so that any missing file triggers the build
 foreach(SRC ${haskell_sources})
@@ -48,18 +57,19 @@ foreach(SRC ${haskell_sources})
     endif()
 endforeach()
 
-if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-  set(hs_optimize -O0)
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(hs_optimize -O0)
 elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
-  set(hs_optimize -O1)
+    set(hs_optimize -O1)
 endif()
 
 add_custom_command(
     OUTPUT ${haskell_artifacts}
     COMMAND ${CABAL} update
     # Build dependencies first without --builddir, otherwise it fails.
-    COMMAND ${CABAL} install --only-dependencies
-    COMMAND ${CABAL} configure ${hs_optimize}
+    COMMAND ${CABAL} install --only-dependencies ${hs_enable_test}
+    COMMAND ${CABAL} configure ${hs_optimize} ${hs_enable_test} --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
+    COMMAND ${CABAL} build --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
     COMMAND ${CABAL} install --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
     COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/thrift_cabal.stamp
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -70,7 +80,13 @@ add_custom_target(haskell_library ALL
     DEPENDS ${haskell_artifacts})
 
 if(BUILD_TESTING)
-    add_test(NAME CabalCheck
+    add_test(NAME HaskellCabalCheck
             COMMAND ${CABAL} check
             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    add_test(NAME HaskellCabalTest
+            # Cabal fails to find built executable when --builddir is specified.
+            # So we invoke the executable directly.
+            # COMMAND ${CABAL} test --builddir=${CMAKE_CURRENT_BINARY_DIR}/dist
+            # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+            COMMAND dist/build/spec/spec)
 endif()

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/LICENSE
----------------------------------------------------------------------
diff --git a/lib/hs/LICENSE b/lib/hs/LICENSE
old mode 100755
new mode 100644

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/Makefile.am
----------------------------------------------------------------------
diff --git a/lib/hs/Makefile.am b/lib/hs/Makefile.am
old mode 100755
new mode 100644
index 45529c7..543381f
--- a/lib/hs/Makefile.am
+++ b/lib/hs/Makefile.am
@@ -42,3 +42,7 @@ maintainer-clean-local:
 
 check-local:
 	$(CABAL) check
+	$(CABAL) install --only-dependencies --enable-tests
+	$(CABAL) configure --enable-tests
+	$(CABAL) build
+	$(CABAL) test

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/README.md
----------------------------------------------------------------------
diff --git a/lib/hs/README.md b/lib/hs/README.md
old mode 100755
new mode 100644

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/TODO
----------------------------------------------------------------------
diff --git a/lib/hs/TODO b/lib/hs/TODO
old mode 100755
new mode 100644

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/Thrift.cabal
----------------------------------------------------------------------
diff --git a/lib/hs/Thrift.cabal b/lib/hs/Thrift.cabal
old mode 100755
new mode 100644
index 5610a5c..f0a09aa
--- a/lib/hs/Thrift.cabal
+++ b/lib/hs/Thrift.cabal
@@ -59,6 +59,7 @@ Library
     Thrift.Transport.Handle,
     Thrift.Transport.HttpClient,
     Thrift.Transport.IOBuffer,
+    Thrift.Transport.Memory,
     Thrift.Types
   Extensions:
     DeriveDataTypeable,
@@ -70,3 +71,10 @@ Library
     RecordWildCards,
     ScopedTypeVariables,
     TypeSynonymInstances
+
+Test-Suite spec
+  Type: exitcode-stdio-1.0
+  Hs-Source-Dirs: test
+  Ghc-Options: -Wall
+  main-is: Spec.hs
+  Build-Depends: base, thrift, hspec, QuickCheck, bytestring >= 0.10, unordered-containers

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/src/Thrift/Protocol.hs
----------------------------------------------------------------------
diff --git a/lib/hs/src/Thrift/Protocol.hs b/lib/hs/src/Thrift/Protocol.hs
index ea58642..31e48b5 100644
--- a/lib/hs/src/Thrift/Protocol.hs
+++ b/lib/hs/src/Thrift/Protocol.hs
@@ -29,6 +29,7 @@ module Thrift.Protocol
     , versionMask
     , version1
     , bsToDouble
+    , bsToDoubleLE
     ) where
 
 import Control.Exception
@@ -119,18 +120,22 @@ handleEOF = const $ return mempty
 -- therefore the behavior of this function varies based on whether the local
 -- machine is big endian or little endian.
 bsToDouble :: BS.ByteString -> Double
-bsToDouble bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBs
-  where
+bsToDoubleLE :: BS.ByteString -> Double
 #if __BYTE_ORDER == __LITTLE_ENDIAN
-    castBs chrPtr = do
-      w <- peek (castPtr chrPtr)
-      poke (castPtr chrPtr) (byteSwap w)
-      peek (castPtr chrPtr)
+bsToDouble bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBsSwapped
+bsToDoubleLE bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBs
 #else
-    castBs = peek . castPtr
+bsToDouble bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBs
+bsToDoubleLE bs = unsafeDupablePerformIO $ unsafeUseAsCString bs castBsSwapped
 #endif
 
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+
+castBsSwapped chrPtr = do
+  w <- peek (castPtr chrPtr)
+  poke (castPtr chrPtr) (byteSwap w)
+  peek (castPtr chrPtr)
+castBs = peek . castPtr
+
 -- | Swap endianness of a 64-bit word
 byteSwap :: Word64 -> Word64
 byteSwap w = (w `shiftL` 56 .&. 0xFF00000000000000) .|.
@@ -141,4 +146,3 @@ byteSwap w = (w `shiftL` 56 .&. 0xFF00000000000000) .|.
              (w `shiftR` 24 .&. 0x0000000000FF0000) .|.
              (w `shiftR` 40 .&. 0x000000000000FF00) .|.
              (w `shiftR` 56 .&. 0x00000000000000FF)
-#endif

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/src/Thrift/Protocol/Compact.hs
----------------------------------------------------------------------
diff --git a/lib/hs/src/Thrift/Protocol/Compact.hs b/lib/hs/src/Thrift/Protocol/Compact.hs
index a329f4e..759466b 100644
--- a/lib/hs/src/Thrift/Protocol/Compact.hs
+++ b/lib/hs/src/Thrift/Protocol/Compact.hs
@@ -55,7 +55,7 @@ import qualified Data.Text.Lazy as LT
 data CompactProtocol a = CompactProtocol a
                          -- ^ Constuct a 'CompactProtocol' with a 'Transport'
 
-protocolID, version, typeMask :: Int8
+protocolID, version, versionMask, typeMask, typeBits :: Word8
 protocolID  = 0x82 -- 1000 0010
 version     = 0x01
 versionMask = 0x1f -- 0001 1111
@@ -69,8 +69,8 @@ instance Protocol CompactProtocol where
     getTransport (CompactProtocol t) = t
 
     writeMessageBegin p (n, t, s) = tWrite (getTransport p) $ toLazyByteString $
-      B.int8 protocolID <>
-      B.int8 ((version .&. versionMask) .|.
+      B.word8 protocolID <>
+      B.word8 ((version .&. versionMask) .|.
               (((fromIntegral $ fromEnum t) `shiftL`
                 typeShiftAmount) .&. typeMask)) <>
       buildVarint (i32ToZigZag s) <>
@@ -120,7 +120,7 @@ buildCompactValue (TByte b) = int8 b
 buildCompactValue (TI16 i) = buildVarint $ i16ToZigZag i
 buildCompactValue (TI32 i) = buildVarint $ i32ToZigZag i
 buildCompactValue (TI64 i) = buildVarint $ i64ToZigZag i
-buildCompactValue (TDouble d) = doubleBE d
+buildCompactValue (TDouble d) = doubleLE d
 buildCompactValue (TString s) = buildVarint len <> lazyByteString s
   where
     len = fromIntegral (LBS.length s) :: Word32
@@ -163,7 +163,7 @@ parseCompactValue T_BYTE = TByte . fromIntegral <$> P.anyWord8
 parseCompactValue T_I16 = TI16 <$> parseVarint zigZagToI16
 parseCompactValue T_I32 = TI32 <$> parseVarint zigZagToI32
 parseCompactValue T_I64 = TI64 <$> parseVarint zigZagToI64
-parseCompactValue T_DOUBLE = TDouble . bsToDouble <$> P.take 8
+parseCompactValue T_DOUBLE = TDouble . bsToDoubleLE <$> P.take 8
 parseCompactValue T_STRING = do
   len :: Word32 <- parseVarint id
   TString . LBS.fromStrict <$> P.take (fromIntegral len)

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/src/Thrift/Transport/Memory.hs
----------------------------------------------------------------------
diff --git a/lib/hs/src/Thrift/Transport/Memory.hs b/lib/hs/src/Thrift/Transport/Memory.hs
new file mode 100644
index 0000000..1c93af6
--- /dev/null
+++ b/lib/hs/src/Thrift/Transport/Memory.hs
@@ -0,0 +1,77 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+module Thrift.Transport.Memory
+       ( openMemoryBuffer
+       , MemoryBuffer(..)
+       ) where
+
+import Data.ByteString.Lazy.Builder
+import Data.Functor
+import Data.IORef
+import Data.Monoid
+import qualified Data.ByteString.Lazy as LBS
+
+import Thrift.Transport
+
+
+data MemoryBuffer = MemoryBuffer {
+  writeBuffer :: IORef Builder,
+  readBuffer :: IORef LBS.ByteString
+}
+
+openMemoryBuffer :: IO MemoryBuffer
+openMemoryBuffer = do
+  wbuf <- newIORef mempty
+  rbuf <- newIORef mempty
+  return MemoryBuffer {
+    writeBuffer = wbuf,
+    readBuffer = rbuf
+  }
+
+instance Transport MemoryBuffer where
+  tIsOpen = const $ return False
+  tClose  = const $ return ()
+  tFlush trans = do
+    let wBuf = writeBuffer trans
+    wb <- readIORef wBuf
+    modifyIORef (readBuffer trans) $ \rb -> mappend rb $ toLazyByteString wb
+    writeIORef wBuf mempty
+
+  tRead _ 0 = return mempty
+  tRead trans n = do
+    let rbuf = readBuffer trans
+    rb <- readIORef rbuf
+    let len = fromIntegral $ LBS.length rb
+    if len == 0
+      then do
+        tFlush trans
+        rb2 <- readIORef (readBuffer trans)
+        if (fromIntegral $ LBS.length rb2) == 0
+          then return mempty
+          else tRead trans n
+      else do
+        let (ret, remain) = LBS.splitAt (fromIntegral n) rb
+        writeIORef rbuf remain
+        return ret
+
+  tPeek trans = (fmap fst . LBS.uncons) <$> readIORef (readBuffer trans)
+
+  tWrite trans v = do
+    modifyIORef (writeBuffer trans) (<> lazyByteString v)

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/test/BinarySpec.hs
----------------------------------------------------------------------
diff --git a/lib/hs/test/BinarySpec.hs b/lib/hs/test/BinarySpec.hs
new file mode 100644
index 0000000..5039610
--- /dev/null
+++ b/lib/hs/test/BinarySpec.hs
@@ -0,0 +1,68 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+module BinarySpec where
+
+import Test.Hspec
+import Test.Hspec.QuickCheck (prop)
+
+import qualified Data.ByteString.Lazy as LBS
+import qualified Data.ByteString.Lazy.Char8 as C
+
+import Thrift.Types
+import Thrift.Transport
+import Thrift.Transport.Memory
+import Thrift.Protocol
+import Thrift.Protocol.Binary
+
+spec :: Spec
+spec = do
+  describe "BinaryProtocol" $ do
+    describe "double" $ do
+      it "writes in big endian order" $ do
+        let val = 2 ** 53
+        trans <- openMemoryBuffer
+        let proto = BinaryProtocol trans
+        writeVal proto (TDouble val)
+        bin <- tRead trans 8
+        (LBS.unpack bin) `shouldBe`[67, 64, 0, 0, 0, 0, 0, 0]
+
+      it "reads in big endian order" $ do
+        let bin = LBS.pack [67, 64, 0, 0, 0, 0, 0, 0]
+        trans <- openMemoryBuffer
+        let proto = BinaryProtocol trans
+        tWrite trans bin
+        val <- readVal proto T_DOUBLE
+        val `shouldBe` (TDouble $ 2 ** 53)
+
+      prop "round trip" $ \val -> do
+        trans <- openMemoryBuffer
+        let proto = BinaryProtocol trans
+        writeVal proto $ TDouble val
+        val2 <- readVal proto T_DOUBLE
+        val2 `shouldBe` (TDouble val)
+
+    describe "string" $ do
+      it "writes" $ do
+        let val = C.pack "aaa"
+        trans <- openMemoryBuffer
+        let proto = BinaryProtocol trans
+        writeVal proto (TString val)
+        bin <- tRead trans 7
+        (LBS.unpack bin) `shouldBe` [0, 0, 0, 3, 97, 97, 97]

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/test/CompactSpec.hs
----------------------------------------------------------------------
diff --git a/lib/hs/test/CompactSpec.hs b/lib/hs/test/CompactSpec.hs
new file mode 100644
index 0000000..22708b4
--- /dev/null
+++ b/lib/hs/test/CompactSpec.hs
@@ -0,0 +1,58 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+module CompactSpec where
+
+import Test.Hspec
+import Test.Hspec.QuickCheck (prop)
+
+import qualified Data.ByteString.Lazy as LBS
+
+import Thrift.Types
+import Thrift.Transport
+import Thrift.Transport.Memory
+import Thrift.Protocol
+import Thrift.Protocol.Compact
+
+spec :: Spec
+spec = do
+  describe "CompactProtocol" $ do
+    describe "double" $ do
+      it "writes in little endian order" $ do
+        let val = 2 ** 53
+        trans <- openMemoryBuffer
+        let proto = CompactProtocol trans
+        writeVal proto (TDouble val)
+        bin <- tReadAll trans 8
+        (LBS.unpack bin) `shouldBe`[0, 0, 0, 0, 0, 0, 64, 67]
+
+      it "reads in little endian order" $ do
+        let bin = LBS.pack [0, 0, 0, 0, 0, 0, 64, 67]
+        trans <- openMemoryBuffer
+        let proto = CompactProtocol trans
+        tWrite trans bin
+        val <- readVal proto T_DOUBLE
+        val `shouldBe` (TDouble $ 2 ** 53)
+
+      prop "round trip" $ \val -> do
+        trans <- openMemoryBuffer
+        let proto = CompactProtocol trans
+        writeVal proto $ TDouble val
+        val2 <- readVal proto T_DOUBLE
+        val2 `shouldBe` (TDouble val)

http://git-wip-us.apache.org/repos/asf/thrift/blob/7c7d679a/lib/hs/test/Spec.hs
----------------------------------------------------------------------
diff --git a/lib/hs/test/Spec.hs b/lib/hs/test/Spec.hs
new file mode 100644
index 0000000..0f5a816
--- /dev/null
+++ b/lib/hs/test/Spec.hs
@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+-- Our CI does not work well with auto discover.
+-- Need to add build-time PATH variable to hspec-discover dir from CMake
+-- or install hspec system-wide for the following to work.
+-- {-# OPTIONS_GHC -F -pgmF hspec-discover #-}
+
+import Test.Hspec
+
+import qualified BinarySpec
+import qualified CompactSpec
+
+main :: IO ()
+main = hspec spec
+
+spec :: Spec
+spec = do
+  describe "Binary" BinarySpec.spec
+  describe "Compact" CompactSpec.spec


[3/4] thrift git commit: THRIFT-3432 Add the TByteBuffer transport for Java Client: Java Patch: Tom Lee

Posted by ns...@apache.org.
THRIFT-3432 Add the TByteBuffer transport for Java
Client: Java
Patch: Tom Lee

This closes #705


Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/9d0f752a
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/9d0f752a
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/9d0f752a

Branch: refs/heads/master
Commit: 9d0f752abcd7b9435c245371d74a2086006a6417
Parents: 0156aae
Author: Tom Lee <gi...@tomlee.co>
Authored: Thu Nov 19 03:03:30 2015 -0800
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:18:01 2015 +0900

----------------------------------------------------------------------
 .../apache/thrift/transport/TByteBuffer.java    | 87 ++++++++++++++++++++
 .../thrift/transport/TestTByteBuffer.java       | 36 ++++++++
 2 files changed, 123 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/thrift/blob/9d0f752a/lib/java/src/org/apache/thrift/transport/TByteBuffer.java
----------------------------------------------------------------------
diff --git a/lib/java/src/org/apache/thrift/transport/TByteBuffer.java b/lib/java/src/org/apache/thrift/transport/TByteBuffer.java
new file mode 100644
index 0000000..a09f33d
--- /dev/null
+++ b/lib/java/src/org/apache/thrift/transport/TByteBuffer.java
@@ -0,0 +1,87 @@
+package org.apache.thrift.transport;
+
+import java.nio.BufferOverflowException;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+
+/**
+ * ByteBuffer-backed implementation of TTransport.
+ */
+public final class TByteBuffer extends TTransport {
+  private final ByteBuffer byteBuffer;
+
+  /**
+   * Creates a new TByteBuffer wrapping a given NIO ByteBuffer.
+   */
+  public TByteBuffer(ByteBuffer byteBuffer) {
+    this.byteBuffer = byteBuffer;
+  }
+
+  @Override
+  public boolean isOpen() {
+    return true;
+  }
+
+  @Override
+  public void open() {
+  }
+
+  @Override
+  public void close() {
+  }
+
+  @Override
+  public int read(byte[] buf, int off, int len) throws TTransportException {
+    final int n = Math.min(byteBuffer.remaining(), len);
+    if (n > 0) {
+      try {
+        byteBuffer.get(buf, off, len);
+      } catch (BufferUnderflowException e) {
+        throw new TTransportException("Unexpected end of input buffer", e);
+      }
+    }
+    return n;
+  }
+
+  @Override
+  public void write(byte[] buf, int off, int len) throws TTransportException {
+    try {
+      byteBuffer.put(buf, off, len);
+    } catch (BufferOverflowException e) {
+      throw new TTransportException("Not enough room in output buffer", e);
+    }
+  }
+
+  /**
+   * Get the underlying NIO ByteBuffer.
+   */
+  public ByteBuffer getByteBuffer() {
+    return byteBuffer;
+  }
+
+  /**
+   * Convenience method to call clear() on the underlying NIO ByteBuffer.
+   */
+  public TByteBuffer clear() {
+    byteBuffer.clear();
+    return this;
+  }
+
+  /**
+   * Convenience method to call flip() on the underlying NIO ByteBuffer.
+     */
+  public TByteBuffer flip() {
+    byteBuffer.flip();
+    return this;
+  }
+
+  /**
+   * Convenience method to convert the underlying NIO ByteBuffer to a
+   * plain old byte array.
+   */
+  public byte[] toByteArray() {
+    final byte[] data = new byte[byteBuffer.remaining()];
+    byteBuffer.slice().get(data);
+    return data;
+  }
+}

http://git-wip-us.apache.org/repos/asf/thrift/blob/9d0f752a/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java
----------------------------------------------------------------------
diff --git a/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java b/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java
new file mode 100644
index 0000000..a73075b
--- /dev/null
+++ b/lib/java/test/org/apache/thrift/transport/TestTByteBuffer.java
@@ -0,0 +1,36 @@
+package org.apache.thrift.transport;
+
+import junit.framework.TestCase;
+import org.apache.commons.codec.Charsets;
+import org.apache.thrift.TException;
+
+import java.nio.ByteBuffer;
+
+public class TestTByteBuffer extends TestCase {
+  public void testReadWrite() throws Exception {
+    final TByteBuffer byteBuffer = new TByteBuffer(ByteBuffer.allocate(16));
+    byteBuffer.write("Hello World".getBytes(Charsets.UTF_8));
+    assertEquals("Hello World", new String(byteBuffer.flip().toByteArray(), Charsets.UTF_8));
+  }
+
+  public void testReuseReadWrite() throws Exception {
+    final TByteBuffer byteBuffer = new TByteBuffer(ByteBuffer.allocate(16));
+    byteBuffer.write("Hello World".getBytes(Charsets.UTF_8));
+    assertEquals("Hello World", new String(byteBuffer.flip().toByteArray(), Charsets.UTF_8));
+
+    byteBuffer.clear();
+
+    byteBuffer.write("Goodbye Horses".getBytes(Charsets.UTF_8));
+    assertEquals("Goodbye Horses", new String(byteBuffer.flip().toByteArray(), Charsets.UTF_8));
+  }
+
+  public void testOverflow() throws Exception {
+    final TByteBuffer byteBuffer = new TByteBuffer(ByteBuffer.allocate(4));
+    try {
+      byteBuffer.write("Hello World".getBytes(Charsets.UTF_8));
+      fail("Expected write operation to fail with TTransportException");
+    } catch (TTransportException e) {
+      assertEquals("Not enough room in output buffer", e.getMessage());
+    }
+  }
+}


[2/4] thrift git commit: THRIFT-3484 Consolidate temporary buffers in Java's TCompactProtocol Client: Java Patch: Tom Lee

Posted by ns...@apache.org.
THRIFT-3484 Consolidate temporary buffers in Java's TCompactProtocol
Client: Java
Patch: Tom Lee

This closes #738


Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/4f6138b7
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/4f6138b7
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/4f6138b7

Branch: refs/heads/master
Commit: 4f6138b7a2762f1937b49389f72b348736973e22
Parents: 9d0f752
Author: Tom Lee <gi...@tomlee.co>
Authored: Thu Dec 10 00:10:30 2015 -0800
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:18:01 2015 +0900

----------------------------------------------------------------------
 .../thrift/protocol/TCompactProtocol.java       | 46 ++++++++++----------
 1 file changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/thrift/blob/4f6138b7/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
----------------------------------------------------------------------
diff --git a/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java b/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
index 5973fcd..2a38810 100644
--- a/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
+++ b/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java
@@ -37,6 +37,8 @@ import org.apache.thrift.transport.TTransport;
  * and i64 fields you have, the more benefit you'll see.
  */
 public class TCompactProtocol extends TProtocol {
+  private final static byte[] EMPTY_BYTES = new byte[0];
+  private final static ByteBuffer EMPTY_BUFFER = ByteBuffer.wrap(EMPTY_BYTES);
 
   private final static long NO_LENGTH_LIMIT = -1;
 
@@ -144,6 +146,12 @@ public class TCompactProtocol extends TProtocol {
   private final long containerLengthLimit_;
 
   /**
+   * Temporary buffer used for various operations that would otherwise require a
+   * small allocation.
+   */
+  private final byte[] temp = new byte[10];
+
+  /**
    * Create a TCompactProtocol.
    *
    * @param transport the TTransport object to read from or write to.
@@ -343,9 +351,8 @@ public class TCompactProtocol extends TProtocol {
    * Write a double to the wire as 8 bytes.
    */
   public void writeDouble(double dub) throws TException {
-    byte[] data = new byte[]{0, 0, 0, 0, 0, 0, 0, 0};
-    fixedLongToBytes(Double.doubleToLongBits(dub), data, 0);
-    trans_.write(data);
+    fixedLongToBytes(Double.doubleToLongBits(dub), temp, 0);
+    trans_.write(temp, 0, 8);
   }
 
   /**
@@ -405,40 +412,38 @@ public class TCompactProtocol extends TProtocol {
    * Write an i32 as a varint. Results in 1-5 bytes on the wire.
    * TODO: make a permanent buffer like writeVarint64?
    */
-  byte[] i32buf = new byte[5];
   private void writeVarint32(int n) throws TException {
     int idx = 0;
     while (true) {
       if ((n & ~0x7F) == 0) {
-        i32buf[idx++] = (byte)n;
+        temp[idx++] = (byte)n;
         // writeByteDirect((byte)n);
         break;
         // return;
       } else {
-        i32buf[idx++] = (byte)((n & 0x7F) | 0x80);
+        temp[idx++] = (byte)((n & 0x7F) | 0x80);
         // writeByteDirect((byte)((n & 0x7F) | 0x80));
         n >>>= 7;
       }
     }
-    trans_.write(i32buf, 0, idx);
+    trans_.write(temp, 0, idx);
   }
 
   /**
    * Write an i64 as a varint. Results in 1-10 bytes on the wire.
    */
-  byte[] varint64out = new byte[10];
   private void writeVarint64(long n) throws TException {
     int idx = 0;
     while (true) {
       if ((n & ~0x7FL) == 0) {
-        varint64out[idx++] = (byte)n;
+        temp[idx++] = (byte)n;
         break;
       } else {
-        varint64out[idx++] = ((byte)((n & 0x7F) | 0x80));
+        temp[idx++] = ((byte)((n & 0x7F) | 0x80));
         n >>>= 7;
       }
     }
-    trans_.write(varint64out, 0, idx);
+    trans_.write(temp, 0, idx);
   }
 
   /**
@@ -476,10 +481,9 @@ public class TCompactProtocol extends TProtocol {
    * Writes a byte without any possibility of all that field header nonsense.
    * Used internally by other writing methods that know they need to write a byte.
    */
-  private byte[] byteDirectBuffer = new byte[1];
   private void writeByteDirect(byte b) throws TException {
-    byteDirectBuffer[0] = b;
-    trans_.write(byteDirectBuffer);
+    temp[0] = b;
+    trans_.write(temp, 0, 1);
   }
 
   /**
@@ -621,7 +625,6 @@ public class TCompactProtocol extends TProtocol {
     return readByte() == Types.BOOLEAN_TRUE;
   }
 
-  byte[] byteRawBuf = new byte[1];
   /**
    * Read a single byte off the wire. Nothing interesting here.
    */
@@ -631,8 +634,8 @@ public class TCompactProtocol extends TProtocol {
       b = trans_.getBuffer()[trans_.getBufferPosition()];
       trans_.consumeBuffer(1);
     } else {
-      trans_.readAll(byteRawBuf, 0, 1);
-      b = byteRawBuf[0];
+      trans_.readAll(temp, 0, 1);
+      b = temp[0];
     }
     return b;
   }
@@ -662,9 +665,8 @@ public class TCompactProtocol extends TProtocol {
    * No magic here - just read a double off the wire.
    */
   public double readDouble() throws TException {
-    byte[] longBits = new byte[8];
-    trans_.readAll(longBits, 0, 8);
-    return Double.longBitsToDouble(bytesToLong(longBits));
+    trans_.readAll(temp, 0, 8);
+    return Double.longBitsToDouble(bytesToLong(temp));
   }
 
   /**
@@ -697,7 +699,7 @@ public class TCompactProtocol extends TProtocol {
   public ByteBuffer readBinary() throws TException {
     int length = readVarint32();
     checkStringReadLength(length);
-    if (length == 0) return ByteBuffer.wrap(new byte[0]);
+    if (length == 0) return EMPTY_BUFFER;
 
     if (trans_.getBytesRemainingInBuffer() >= length) {
       ByteBuffer bb = ByteBuffer.wrap(trans_.getBuffer(), trans_.getBufferPosition(), length);
@@ -714,7 +716,7 @@ public class TCompactProtocol extends TProtocol {
    * Read a byte[] of a known length from the wire.
    */
   private byte[] readBinary(int length) throws TException {
-    if (length == 0) return new byte[0];
+    if (length == 0) return EMPTY_BYTES;
 
     byte[] buf = new byte[length];
     trans_.readAll(buf, 0, length);


[4/4] thrift git commit: THRIFT-3483 Incorrect empty binary handling introduced by THRIFT-3359 Client: C++, Node.js Patch: Nobuaki Sukegawa

Posted by ns...@apache.org.
THRIFT-3483 Incorrect empty binary handling introduced by THRIFT-3359
Client: C++, Node.js
Patch: Nobuaki Sukegawa

This closes #737


Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/dfb68964
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/dfb68964
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/dfb68964

Branch: refs/heads/master
Commit: dfb6896436904d0fbe1f762de898a3c68c989aa9
Parents: 7c7d679
Author: Nobuaki Sukegawa <ns...@apache.org>
Authored: Wed Dec 9 22:09:26 2015 +0900
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Fri Dec 11 00:19:09 2015 +0900

----------------------------------------------------------------------
 lib/cpp/src/thrift/protocol/TJSONProtocol.cpp |  8 +++++---
 lib/nodejs/lib/thrift/binary_protocol.js      |  2 +-
 lib/nodejs/lib/thrift/compact_protocol.js     |  2 +-
 test/cpp/src/TestClient.cpp                   | 14 ++++++++++++++
 4 files changed, 21 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
----------------------------------------------------------------------
diff --git a/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp b/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
index f1370bb..9569188 100644
--- a/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
+++ b/lib/cpp/src/thrift/protocol/TJSONProtocol.cpp
@@ -801,9 +801,11 @@ uint32_t TJSONProtocol::readJSONBase64(std::string& str) {
   uint32_t len = static_cast<uint32_t>(tmp.length());
   str.clear();
   // Ignore padding
-  uint32_t bound = len >= 2 ? len - 2 : 0;
-  for (uint32_t i = len - 1; i >= bound && b[i] == '='; --i) {
-    --len;
+  if (len >= 2)  {
+    uint32_t bound = len - 2;
+    for (uint32_t i = len - 1; i >= bound && b[i] == '='; --i) {
+      --len;
+    }
   }
   while (len >= 4) {
     base64_decode(b, 4);

http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/lib/nodejs/lib/thrift/binary_protocol.js
----------------------------------------------------------------------
diff --git a/lib/nodejs/lib/thrift/binary_protocol.js b/lib/nodejs/lib/thrift/binary_protocol.js
index 25e5634..6d3918e 100644
--- a/lib/nodejs/lib/thrift/binary_protocol.js
+++ b/lib/nodejs/lib/thrift/binary_protocol.js
@@ -276,7 +276,7 @@ TBinaryProtocol.prototype.readDouble = function() {
 TBinaryProtocol.prototype.readBinary = function() {
   var len = this.readI32();
   if (len === 0) {
-    return new Buffer();
+    return new Buffer(0);
   }
 
   if (len < 0) {

http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/lib/nodejs/lib/thrift/compact_protocol.js
----------------------------------------------------------------------
diff --git a/lib/nodejs/lib/thrift/compact_protocol.js b/lib/nodejs/lib/thrift/compact_protocol.js
index deecf48..b0e9148 100644
--- a/lib/nodejs/lib/thrift/compact_protocol.js
+++ b/lib/nodejs/lib/thrift/compact_protocol.js
@@ -761,7 +761,7 @@ TCompactProtocol.prototype.readDouble = function() {
 TCompactProtocol.prototype.readBinary = function() {
   var size = this.readVarint32();
   if (size === 0) {
-    return new Buffer();
+    return new Buffer(0);
   }
 
   if (size < 0) {

http://git-wip-us.apache.org/repos/asf/thrift/blob/dfb68964/test/cpp/src/TestClient.cpp
----------------------------------------------------------------------
diff --git a/test/cpp/src/TestClient.cpp b/test/cpp/src/TestClient.cpp
index a6773d4..0ad789d 100644
--- a/test/cpp/src/TestClient.cpp
+++ b/test/cpp/src/TestClient.cpp
@@ -463,6 +463,20 @@ int main(int argc, char** argv) {
     /**
      * BINARY TEST
      */
+    printf("testBinary(empty)\n");
+    try {
+      string bin_result;
+      testClient.testBinary(bin_result, string());
+      if (!bin_result.empty()) {
+        printf("*** FAILED ***\n");
+        printf("invalid length: %lu\n", static_cast<long unsigned int>(bin_result.size()));
+        return_code |= ERR_BASETYPES;
+      }
+    } catch (exception& ex) {
+      printf("}\n*** FAILED ***\n");
+      printf("%s\n", ex.what());
+      return_code |= ERR_BASETYPES;
+    }
     printf("testBinary([-128..127]) = {");
     const char bin_data[256]
         = {-128, -127, -126, -125, -124, -123, -122, -121, -120, -119, -118, -117, -116, -115, -114,