You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by si...@apache.org on 2018/12/08 22:04:45 UTC
[pulsar] branch master updated: optimizing json deserialization in
sql (#3144)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 3f0e541 optimizing json deserialization in sql (#3144)
3f0e541 is described below
commit 3f0e5417905ed3de0afd572e8ae3a1286a117c16
Author: Boyang Jerry Peng <je...@gmail.com>
AuthorDate: Sat Dec 8 14:04:39 2018 -0800
optimizing json deserialization in sql (#3144)
### Motivation
Use Dsl json for json deserialization is much faster
---
pulsar-sql/presto-distribution/LICENSE | 2 +
pulsar-sql/presto-pulsar/pom.xml | 8 +++
.../pulsar/sql/presto/JSONSchemaHandler.java | 57 +++++++++-------------
.../pulsar/sql/presto/PulsarRecordCursor.java | 2 +-
4 files changed, 33 insertions(+), 36 deletions(-)
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 6424768..363d1ae 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -400,6 +400,8 @@ Protocol Buffers License
BSD 3-clause "New" or "Revised" License
* RE2J TD -- re2j-td-1.4.jar
+ * DSL Platform JSON
+ - dsl-json-1.8.4.jar
BSD License
* ANTLR 4 Runtime -- antlr4-runtime-4.6.jar
diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml
index e3b2021..6416567 100644
--- a/pulsar-sql/presto-pulsar/pom.xml
+++ b/pulsar-sql/presto-pulsar/pom.xml
@@ -40,6 +40,7 @@
<dep.javax-inject.version>1</dep.javax-inject.version>
<dep.guava.version>24.1-jre</dep.guava.version>
<jctools.version>2.1.2</jctools.version>
+ <dslJson.verson>1.8.4</dslJson.verson>
</properties>
<dependencies>
@@ -96,6 +97,13 @@
<artifactId>jctools-core</artifactId>
<version>${jctools.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>com.dslplatform</groupId>
+ <artifactId>dsl-json</artifactId>
+ <version>${dslJson.verson}</version>
+ </dependency>
+
<!-- Presto SPI -->
<dependency>
<groupId>com.facebook.presto</groupId>
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java
index aed7e0a..c941e58 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java
@@ -18,19 +18,14 @@
*/
package org.apache.pulsar.sql.presto;
+import com.dslplatform.json.DslJson;
import com.facebook.presto.spi.type.Type;
import io.airlift.log.Logger;
-import io.airlift.slice.Slice;
-import org.apache.pulsar.shade.com.google.gson.JsonElement;
-import org.apache.pulsar.shade.com.google.gson.JsonObject;
-import org.apache.pulsar.shade.com.google.gson.JsonParser;
-import java.util.Arrays;
+import java.io.IOException;
+import java.math.BigDecimal;
import java.util.List;
-
-import static com.facebook.presto.spi.type.IntegerType.INTEGER;
-import static com.facebook.presto.spi.type.RealType.REAL;
-import static com.facebook.presto.spi.type.SmallintType.SMALLINT;
+import java.util.Map;
public class JSONSchemaHandler implements SchemaHandler {
@@ -38,7 +33,7 @@ public class JSONSchemaHandler implements SchemaHandler {
private List<PulsarColumnHandle> columnHandles;
- private final JsonParser jsonParser = new JsonParser();
+ private final DslJson<Object> dslJson = new DslJson<>();
public JSONSchemaHandler(List<PulsarColumnHandle> columnHandles) {
this.columnHandles = columnHandles;
@@ -46,50 +41,42 @@ public class JSONSchemaHandler implements SchemaHandler {
@Override
public Object deserialize(byte[] bytes) {
- JsonElement jsonElement = this.jsonParser.parse(new String(bytes));
- return jsonElement.getAsJsonObject();
+ try {
+ return dslJson.deserialize(Map.class, bytes, bytes.length);
+ } catch (IOException e) {
+ log.error(e);
+ }
+ return null;
}
@Override
public Object extractField(int index, Object currentRecord) {
try {
- JsonObject jsonObject = (JsonObject) currentRecord;
+
+ Map jsonObject = (Map) currentRecord;
PulsarColumnHandle pulsarColumnHandle = columnHandles.get(index);
String[] fieldNames = pulsarColumnHandle.getFieldNames();
- JsonElement field = jsonObject.get(fieldNames[0]);
- if (field.isJsonNull()) {
+ Object field = jsonObject.get(fieldNames[0]);
+ if (field == null) {
return null;
}
for (int i = 1; i < fieldNames.length ; i++) {
- field = field.getAsJsonObject().get(fieldNames[i]);
- if (field.isJsonNull()) {
+ field = ((Map) field).get(fieldNames[i]);
+ if (field == null) {
return null;
}
}
Type type = pulsarColumnHandle.getType();
+
Class<?> javaType = type.getJavaType();
- if (javaType == long.class) {
- if (type.equals(INTEGER)) {
- return field.getAsInt();
- } else if (type.equals(REAL)) {
- return field.getAsFloat();
- } else if (type.equals(SMALLINT)) {
- return field.getAsShort();
- } else {
- return field.getAsLong();
- }
- } else if (javaType == boolean.class) {
- return field.getAsBoolean();
- } else if (javaType == double.class) {
- return field.getAsDouble();
- } else if (javaType == Slice.class) {
- return field.getAsString();
- } else {
- return null;
+ if (javaType == double.class) {
+ return ((BigDecimal) field).doubleValue();
}
+
+ return field;
} catch (Exception ex) {
log.debug(ex,"%s", ex);
}
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java
index e1791b4..e2198a8 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java
@@ -442,7 +442,7 @@ public class PulsarRecordCursor implements RecordCursor {
} else if (type.equals(DATE)) {
return ((Number) record).longValue();
} else if (type.equals(INTEGER)) {
- return (int) record;
+ return ((Number) record).intValue();
} else if (type.equals(REAL)) {
return Float.floatToIntBits(((Number) record).floatValue());
} else if (type.equals(SMALLINT)) {