You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/12/08 22:04:41 UTC
[GitHub] sijie closed pull request #3144: optimizing json deserialization in
sql
sijie closed pull request #3144: optimizing json deserialization in sql
URL: https://github.com/apache/pulsar/pull/3144
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 64247687c2..363d1aeaba 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -400,6 +400,8 @@ Protocol Buffers License
BSD 3-clause "New" or "Revised" License
* RE2J TD -- re2j-td-1.4.jar
+ * DSL Platform JSON
+ - dsl-json-1.8.4.jar
BSD License
* ANTLR 4 Runtime -- antlr4-runtime-4.6.jar
diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml
index e3b202137c..64165675e7 100644
--- a/pulsar-sql/presto-pulsar/pom.xml
+++ b/pulsar-sql/presto-pulsar/pom.xml
@@ -40,6 +40,7 @@
<dep.javax-inject.version>1</dep.javax-inject.version>
<dep.guava.version>24.1-jre</dep.guava.version>
<jctools.version>2.1.2</jctools.version>
+ <dslJson.verson>1.8.4</dslJson.verson>
</properties>
<dependencies>
@@ -96,6 +97,13 @@
<artifactId>jctools-core</artifactId>
<version>${jctools.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>com.dslplatform</groupId>
+ <artifactId>dsl-json</artifactId>
+ <version>${dslJson.verson}</version>
+ </dependency>
+
<!-- Presto SPI -->
<dependency>
<groupId>com.facebook.presto</groupId>
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java
index aed7e0af58..c941e58af6 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java
@@ -18,19 +18,14 @@
*/
package org.apache.pulsar.sql.presto;
+import com.dslplatform.json.DslJson;
import com.facebook.presto.spi.type.Type;
import io.airlift.log.Logger;
-import io.airlift.slice.Slice;
-import org.apache.pulsar.shade.com.google.gson.JsonElement;
-import org.apache.pulsar.shade.com.google.gson.JsonObject;
-import org.apache.pulsar.shade.com.google.gson.JsonParser;
-import java.util.Arrays;
+import java.io.IOException;
+import java.math.BigDecimal;
import java.util.List;
-
-import static com.facebook.presto.spi.type.IntegerType.INTEGER;
-import static com.facebook.presto.spi.type.RealType.REAL;
-import static com.facebook.presto.spi.type.SmallintType.SMALLINT;
+import java.util.Map;
public class JSONSchemaHandler implements SchemaHandler {
@@ -38,7 +33,7 @@
private List<PulsarColumnHandle> columnHandles;
- private final JsonParser jsonParser = new JsonParser();
+ private final DslJson<Object> dslJson = new DslJson<>();
public JSONSchemaHandler(List<PulsarColumnHandle> columnHandles) {
this.columnHandles = columnHandles;
@@ -46,50 +41,42 @@ public JSONSchemaHandler(List<PulsarColumnHandle> columnHandles) {
@Override
public Object deserialize(byte[] bytes) {
- JsonElement jsonElement = this.jsonParser.parse(new String(bytes));
- return jsonElement.getAsJsonObject();
+ try {
+ return dslJson.deserialize(Map.class, bytes, bytes.length);
+ } catch (IOException e) {
+ log.error(e);
+ }
+ return null;
}
@Override
public Object extractField(int index, Object currentRecord) {
try {
- JsonObject jsonObject = (JsonObject) currentRecord;
+
+ Map jsonObject = (Map) currentRecord;
PulsarColumnHandle pulsarColumnHandle = columnHandles.get(index);
String[] fieldNames = pulsarColumnHandle.getFieldNames();
- JsonElement field = jsonObject.get(fieldNames[0]);
- if (field.isJsonNull()) {
+ Object field = jsonObject.get(fieldNames[0]);
+ if (field == null) {
return null;
}
for (int i = 1; i < fieldNames.length ; i++) {
- field = field.getAsJsonObject().get(fieldNames[i]);
- if (field.isJsonNull()) {
+ field = ((Map) field).get(fieldNames[i]);
+ if (field == null) {
return null;
}
}
Type type = pulsarColumnHandle.getType();
+
Class<?> javaType = type.getJavaType();
- if (javaType == long.class) {
- if (type.equals(INTEGER)) {
- return field.getAsInt();
- } else if (type.equals(REAL)) {
- return field.getAsFloat();
- } else if (type.equals(SMALLINT)) {
- return field.getAsShort();
- } else {
- return field.getAsLong();
- }
- } else if (javaType == boolean.class) {
- return field.getAsBoolean();
- } else if (javaType == double.class) {
- return field.getAsDouble();
- } else if (javaType == Slice.class) {
- return field.getAsString();
- } else {
- return null;
+ if (javaType == double.class) {
+ return ((BigDecimal) field).doubleValue();
}
+
+ return field;
} catch (Exception ex) {
log.debug(ex,"%s", ex);
}
diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java
index e1791b432e..e2198a8600 100644
--- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java
+++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java
@@ -442,7 +442,7 @@ public long getLong(int field) {
} else if (type.equals(DATE)) {
return ((Number) record).longValue();
} else if (type.equals(INTEGER)) {
- return (int) record;
+ return ((Number) record).intValue();
} else if (type.equals(REAL)) {
return Float.floatToIntBits(((Number) record).floatValue());
} else if (type.equals(SMALLINT)) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services