You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@thrift.apache.org by je...@apache.org on 2014/02/07 22:25:21 UTC
[1/2] git commit: THRIFT-2346 C#: UTF-8 sent by PHP as JSON is not
understood by TJsonProtocol
Updated Branches:
refs/heads/master a2de41053 -> 7bb44a33b
THRIFT-2346 C#: UTF-8 sent by PHP as JSON is not understood by TJsonProtocol
Patch: Jens Geyer
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/73938622
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/73938622
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/73938622
Branch: refs/heads/master
Commit: 73938622ef9b3a53d45063aee70ad5155202605a
Parents: a2de410
Author: Jens Geyer <je...@apache.org>
Authored: Fri Feb 7 22:22:36 2014 +0100
Committer: Jens Geyer <je...@apache.org>
Committed: Fri Feb 7 22:22:36 2014 +0100
----------------------------------------------------------------------
lib/csharp/src/Protocol/TJSONProtocol.cs | 49 ++++++++------
lib/csharp/test/JSON/JSONTest.csproj | 67 ++++++++++++++++++++
lib/csharp/test/JSON/Program.cs | 55 ++++++++++++++++
lib/csharp/test/JSON/Properties/AssemblyInfo.cs | 36 +++++++++++
lib/csharp/test/JSON/app.config | 3 +
5 files changed, 190 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/73938622/lib/csharp/src/Protocol/TJSONProtocol.cs
----------------------------------------------------------------------
diff --git a/lib/csharp/src/Protocol/TJSONProtocol.cs b/lib/csharp/src/Protocol/TJSONProtocol.cs
index 14db9cc..9c62bec 100644
--- a/lib/csharp/src/Protocol/TJSONProtocol.cs
+++ b/lib/csharp/src/Protocol/TJSONProtocol.cs
@@ -58,7 +58,6 @@ namespace Thrift.Protocol
private static byte[] RBRACKET = new byte[] { (byte)']' };
private static byte[] QUOTE = new byte[] { (byte)'"' };
private static byte[] BACKSLASH = new byte[] { (byte)'\\' };
- private static byte[] ZERO = new byte[] { (byte)'0' };
private byte[] ESCSEQ = new byte[] { (byte)'\\', (byte)'u', (byte)'0', (byte)'0' };
@@ -735,28 +734,38 @@ namespace Thrift.Protocol
{
break;
}
- if (ch == ESCSEQ[0])
+
+ // escaped?
+ if (ch != ESCSEQ[0])
+ {
+ buffer.Write(new byte[] { (byte)ch }, 0, 1);
+ continue;
+ }
+
+ // distinguish between \uXXXX and \?
+ ch = reader.Read();
+ if (ch != ESCSEQ[1]) // control chars like \n
{
- ch = reader.Read();
- if (ch == ESCSEQ[1])
+ int off = Array.IndexOf(ESCAPE_CHARS, (char)ch);
+ if (off == -1)
{
- ReadJSONSyntaxChar(ZERO);
- ReadJSONSyntaxChar(ZERO);
- trans.ReadAll(tempBuffer, 0, 2);
- ch = (byte)((HexVal((byte)tempBuffer[0]) << 4) + HexVal(tempBuffer[1]));
+ throw new TProtocolException(TProtocolException.INVALID_DATA,
+ "Expected control char");
}
- else
- {
- int off = Array.IndexOf(ESCAPE_CHARS, (char)ch);
- if (off == -1)
- {
- throw new TProtocolException(TProtocolException.INVALID_DATA,
- "Expected control char");
- }
- ch = ESCAPE_CHAR_VALS[off];
- }
- }
- buffer.Write(new byte[] { (byte)ch }, 0, 1);
+ ch = ESCAPE_CHAR_VALS[off];
+ buffer.Write(new byte[] { (byte)ch }, 0, 1);
+ continue;
+ }
+
+
+ // it's \uXXXX
+ trans.ReadAll(tempBuffer, 0, 4);
+ var wch = (short)((HexVal((byte)tempBuffer[0]) << 12) +
+ (HexVal((byte)tempBuffer[1]) << 8) +
+ (HexVal((byte)tempBuffer[2]) << 4) +
+ HexVal(tempBuffer[3]));
+ var tmp = utf8Encoding.GetBytes(new char[] { (char)wch });
+ buffer.Write(tmp, 0, tmp.Length);
}
return buffer.ToArray();
}
http://git-wip-us.apache.org/repos/asf/thrift/blob/73938622/lib/csharp/test/JSON/JSONTest.csproj
----------------------------------------------------------------------
diff --git a/lib/csharp/test/JSON/JSONTest.csproj b/lib/csharp/test/JSON/JSONTest.csproj
new file mode 100644
index 0000000..73303b8
--- /dev/null
+++ b/lib/csharp/test/JSON/JSONTest.csproj
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">x86</Platform>
+ <ProductVersion>8.0.30703</ProductVersion>
+ <SchemaVersion>2.0</SchemaVersion>
+ <ProjectGuid>{E37A0034-DCBF-4886-A0DA-25A03D12D975}</ProjectGuid>
+ <OutputType>Exe</OutputType>
+ <AppDesignerFolder>Properties</AppDesignerFolder>
+ <RootNamespace>JSONTest</RootNamespace>
+ <AssemblyName>JSONTest</AssemblyName>
+ <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+ <TargetFrameworkProfile>
+ </TargetFrameworkProfile>
+ <FileAlignment>512</FileAlignment>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
+ <PlatformTarget>x86</PlatformTarget>
+ <DebugSymbols>true</DebugSymbols>
+ <DebugType>full</DebugType>
+ <Optimize>false</Optimize>
+ <OutputPath>bin\Debug\</OutputPath>
+ <DefineConstants>DEBUG;TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
+ <PlatformTarget>x86</PlatformTarget>
+ <DebugType>pdbonly</DebugType>
+ <Optimize>true</Optimize>
+ <OutputPath>bin\Release\</OutputPath>
+ <DefineConstants>TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <ItemGroup>
+ <Reference Include="System" />
+ <Reference Include="System.Core" />
+ <Reference Include="System.Xml.Linq" />
+ <Reference Include="System.Data.DataSetExtensions" />
+ <Reference Include="Microsoft.CSharp" />
+ <Reference Include="System.Data" />
+ <Reference Include="System.Xml" />
+ </ItemGroup>
+ <ItemGroup>
+ <Compile Include="Program.cs" />
+ <Compile Include="Properties\AssemblyInfo.cs" />
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="app.config" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\..\src\Thrift.csproj">
+ <Project>{499EB63C-D74C-47E8-AE48-A2FC94538E9D}</Project>
+ <Name>Thrift</Name>
+ </ProjectReference>
+ </ItemGroup>
+ <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+ <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Microsoft.Common.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/thrift/blob/73938622/lib/csharp/test/JSON/Program.cs
----------------------------------------------------------------------
diff --git a/lib/csharp/test/JSON/Program.cs b/lib/csharp/test/JSON/Program.cs
new file mode 100644
index 0000000..7bdb7f5
--- /dev/null
+++ b/lib/csharp/test/JSON/Program.cs
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Thrift.Protocol;
+using Thrift.Transport;
+
+namespace JSONTest
+{
+ class Program
+ {
+ static void Main(string[] args)
+ {
+ TestThrift2336();
+ }
+
+ public static void TestThrift2336()
+ {
+ const string RUSSIAN_TEXT = "\u0420\u0443\u0441\u0441\u043a\u043e\u0435 \u041d\u0430\u0437\u0432\u0430\u043d\u0438\u0435";
+ const string RUSSIAN_JSON = "\"\\u0420\\u0443\\u0441\\u0441\\u043a\\u043e\\u0435 \\u041d\\u0430\\u0437\\u0432\\u0430\\u043d\\u0438\\u0435\"";
+
+ // prepare buffer with JOSN data
+ byte[] rawBytes = new byte[RUSSIAN_JSON.Length];
+ for (var i = 0; i < RUSSIAN_JSON.Length; ++i)
+ rawBytes[i] = (byte)(RUSSIAN_JSON[i] & (char)0xFF); // only low bytes
+
+ // parse and check
+ var stm = new MemoryStream(rawBytes);
+ var trans = new TStreamTransport(stm, null);
+ var prot = new TJSONProtocol(trans);
+ Debug.Assert(prot.ReadString() == RUSSIAN_TEXT, "reading JSON with hex-encoded chars > 8 bit");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/thrift/blob/73938622/lib/csharp/test/JSON/Properties/AssemblyInfo.cs
----------------------------------------------------------------------
diff --git a/lib/csharp/test/JSON/Properties/AssemblyInfo.cs b/lib/csharp/test/JSON/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000..a60ebc1
--- /dev/null
+++ b/lib/csharp/test/JSON/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// Allgemeine Informationen über eine Assembly werden über die folgenden
+// Attribute gesteuert. Ändern Sie diese Attributwerte, um die Informationen zu ändern,
+// die mit einer Assembly verknüpft sind.
+[assembly: AssemblyTitle("JSONTest")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("JSONTest")]
+[assembly: AssemblyCopyright("Copyright © 2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Durch Festlegen von ComVisible auf "false" werden die Typen in dieser Assembly unsichtbar
+// für COM-Komponenten. Wenn Sie auf einen Typ in dieser Assembly von
+// COM zugreifen müssen, legen Sie das ComVisible-Attribut für diesen Typ auf "true" fest.
+[assembly: ComVisible(false)]
+
+// Die folgende GUID bestimmt die ID der Typbibliothek, wenn dieses Projekt für COM verfügbar gemacht wird
+[assembly: Guid("2b2e7d56-3e65-4368-92d7-e34d56b7105e")]
+
+// Versionsinformationen für eine Assembly bestehen aus den folgenden vier Werten:
+//
+// Hauptversion
+// Nebenversion
+// Buildnummer
+// Revision
+//
+// Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern
+// übernehmen, indem Sie "*" eingeben:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
http://git-wip-us.apache.org/repos/asf/thrift/blob/73938622/lib/csharp/test/JSON/app.config
----------------------------------------------------------------------
diff --git a/lib/csharp/test/JSON/app.config b/lib/csharp/test/JSON/app.config
new file mode 100644
index 0000000..cb2586b
--- /dev/null
+++ b/lib/csharp/test/JSON/app.config
@@ -0,0 +1,3 @@
+<?xml version="1.0"?>
+<configuration>
+<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.0"/></startup></configuration>
[2/2] git commit: THRIFT-2345 Delphi: UTF-8 sent by PHP as JSON is
not understood by TJsonProtocol
Posted by je...@apache.org.
THRIFT-2345 Delphi: UTF-8 sent by PHP as JSON is not understood by TJsonProtocol
Patch: Jens Geyer
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/7bb44a33
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/7bb44a33
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/7bb44a33
Branch: refs/heads/master
Commit: 7bb44a33bf2225d6c2220e4c5166669f7975f561
Parents: 7393862
Author: Jens Geyer <je...@apache.org>
Authored: Fri Feb 7 22:24:37 2014 +0100
Committer: Jens Geyer <je...@apache.org>
Committed: Fri Feb 7 22:24:37 2014 +0100
----------------------------------------------------------------------
lib/delphi/src/Thrift.Protocol.JSON.pas | 48 ++++++++++++++++------------
lib/delphi/test/TestClient.pas | 34 ++++++++++++++++++--
2 files changed, 60 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/7bb44a33/lib/delphi/src/Thrift.Protocol.JSON.pas
----------------------------------------------------------------------
diff --git a/lib/delphi/src/Thrift.Protocol.JSON.pas b/lib/delphi/src/Thrift.Protocol.JSON.pas
index cce6c3c..6d305d8 100644
--- a/lib/delphi/src/Thrift.Protocol.JSON.pas
+++ b/lib/delphi/src/Thrift.Protocol.JSON.pas
@@ -254,7 +254,6 @@ var
RBRACKET : TBytes;
QUOTE : TBytes;
BACKSLASH : TBytes;
- ZERO : TBytes;
ESCSEQ : TBytes;
const
@@ -815,7 +814,8 @@ end;
function TJSONProtocolImpl.ReadJSONString( skipContext : Boolean) : TBytes;
var buffer : TMemoryStream;
- ch : Byte;
+ ch : Byte;
+ wch : Word;
off : Integer;
tmp : TBytes;
begin
@@ -832,25 +832,34 @@ begin
if (ch = QUOTE[0])
then Break;
- if (ch = ESCSEQ[0])
+ // check for escapes
+ if (ch <> ESCSEQ[0]) then begin
+ buffer.Write( ch, 1);
+ Continue;
+ end;
+
+ // distuinguish between \uNNNN and \?
+ ch := FReader.Read;
+ if (ch <> ESCSEQ[1])
then begin
- ch := FReader.Read;
- if (ch = ESCSEQ[1])
- then begin
- ReadJSONSyntaxChar( ZERO[0]);
- ReadJSONSyntaxChar( ZERO[0]);
- SetLength( tmp, 2);
- Transport.ReadAll( tmp, 0, 2);
- ch := (HexVal(tmp[0]) shl 4) + HexVal(tmp[1]);
- end
- else begin
- off := Pos( Char(ch), ESCAPE_CHARS);
- if off < 1
- then raise TProtocolException.Create( TProtocolException.INVALID_DATA, 'Expected control char');
- ch := Byte( ESCAPE_CHAR_VALS[off]);
- end;
+ off := Pos( Char(ch), ESCAPE_CHARS);
+ if off < 1
+ then raise TProtocolException.Create( TProtocolException.INVALID_DATA, 'Expected control char');
+ ch := Byte( ESCAPE_CHAR_VALS[off]);
+ buffer.Write( ch, 1);
+ Continue;
end;
- buffer.Write( ch, 1);
+
+ // it is \uXXXX
+ SetLength( tmp, 4);
+ Transport.ReadAll( tmp, 0, 4);
+ wch := (HexVal(tmp[0]) shl 12)
+ + (HexVal(tmp[1]) shl 8)
+ + (HexVal(tmp[2]) shl 4)
+ + HexVal(tmp[3]);
+ // we need to make UTF8 bytes from it, to be decoded later
+ tmp := SysUtils.TEncoding.UTF8.GetBytes(Char(wch));
+ buffer.Write( tmp[0], length(tmp));
end;
SetLength( result, buffer.Size);
@@ -1174,6 +1183,5 @@ initialization
InitBytes( RBRACKET, [Byte(']')]);
InitBytes( QUOTE, [Byte('"')]);
InitBytes( BACKSLASH, [Byte('\')]);
- InitBytes( ZERO, [Byte('0')]);
InitBytes( ESCSEQ, [Byte('\'),Byte('u'),Byte('0'),Byte('0')]);
end.
http://git-wip-us.apache.org/repos/asf/thrift/blob/7bb44a33/lib/delphi/test/TestClient.pas
----------------------------------------------------------------------
diff --git a/lib/delphi/test/TestClient.pas b/lib/delphi/test/TestClient.pas
index 0f09489..9fb0b7a 100644
--- a/lib/delphi/test/TestClient.pas
+++ b/lib/delphi/test/TestClient.pas
@@ -1,4 +1,4 @@
-(*
+(*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -898,6 +898,9 @@ const
TEST_DOUBLE = -1.234e-56;
DELTA_DOUBLE = TEST_DOUBLE * 1e-14;
TEST_STRING = 'abc-'#$00E4#$00f6#$00fc; // german umlauts (en-us: "funny chars")
+ // Test THRIFT-2336 with 'Русское Название';
+ RUSSIAN_TEXT = #$0420#$0443#$0441#$0441#$043a#$043e#$0435' '#$041d#$0430#$0437#$0432#$0430#$043d#$0438#$0435;
+ RUSSIAN_JSON = '"\u0420\u0443\u0441\u0441\u043a\u043e\u0435 \u041d\u0430\u0437\u0432\u0430\u043d\u0438\u0435"';
// test both possible solidus encodings
SOLIDUS_JSON_DATA = '"one/two\/three"';
SOLIDUS_EXCPECTED = 'one/two/three';
@@ -965,6 +968,7 @@ begin
Expect( stm.Position = stm.Size, 'Stream position after read');
+
// Solidus can be encoded in two ways. Make sure we can read both
stm.Position := 0;
stm.Size := 0;
@@ -976,6 +980,32 @@ begin
Expect( prot.ReadString = SOLIDUS_EXCPECTED, 'Solidus encoding');
+ // Widechars should work too. Do they?
+ // After writing, we ensure that we are able to read it back
+ // We can't assume hex-encoding, since (nearly) any Unicode char is valid JSON
+ stm.Position := 0;
+ stm.Size := 0;
+ prot := TJSONProtocolImpl.Create(
+ TStreamTransportImpl.Create(
+ nil, TThriftStreamAdapterDelphi.Create( stm, FALSE)));
+ prot.WriteString( RUSSIAN_TEXT);
+ stm.Position := 0;
+ prot := TJSONProtocolImpl.Create(
+ TStreamTransportImpl.Create(
+ TThriftStreamAdapterDelphi.Create( stm, FALSE), nil));
+ Expect( prot.ReadString = RUSSIAN_TEXT, 'Writing JSON with chars > 8 bit');
+
+ // Widechars should work with hex-encoding too. Do they?
+ stm.Position := 0;
+ stm.Size := 0;
+ stm.WriteString( RUSSIAN_JSON);
+ stm.Position := 0;
+ prot := TJSONProtocolImpl.Create(
+ TStreamTransportImpl.Create(
+ TThriftStreamAdapterDelphi.Create( stm, FALSE), nil));
+ Expect( prot.ReadString = RUSSIAN_TEXT, 'Reading JSON with chars > 8 bit');
+
+
finally
stm.Free;
prot := nil; //-> Release
@@ -1068,10 +1098,10 @@ var
begin
// perform all tests
try
+ JSONProtocolReadWriteTest;
for i := 0 to FNumIteration - 1 do
begin
ClientTest;
- JSONProtocolReadWriteTest;
end;
except
on e:Exception do Expect( FALSE, 'unexpected exception: "'+e.message+'"');