You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/08/11 14:21:04 UTC
[41/50] [abbrv] incubator-lens git commit: LENS-717 : Fix result set
data types returned from elastic search driver
LENS-717 : Fix result set data types returned from elastic search driver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c224bca3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c224bca3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c224bca3
Branch: refs/heads/current-release-line
Commit: c224bca3128a367bf1bcbe461eb30553884f06f1
Parents: 96cbccd
Author: Amruth S <am...@gmail.com>
Authored: Mon Aug 10 10:42:10 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Aug 10 10:42:10 2015 +0530
----------------------------------------------------------------------
.../client/jest/JestResultSetTransformer.java | 83 ++++++++++++++++----
.../driver/es/ResultSetTransformationTest.java | 42 +++++-----
2 files changed, 87 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c224bca3/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java
index af313da..38d91f9 100644
--- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java
@@ -34,6 +34,7 @@ import com.google.common.collect.Lists;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
import lombok.NonNull;
/**
@@ -78,16 +79,23 @@ public abstract class JestResultSetTransformer {
final String key = entry.getKey();
if (key.equals(ResultSetConstants.KEY_STRING)) {
Validate.isTrue(keyCol != null, "Key not available");
- currentPath.set(schema.indexOf(keyCol), entry.getValue().getAsString());
+ final int index = columnAliases.indexOf(keyCol);
+ currentPath.set(
+ index,
+ getTypedValue(index, entry.getValue())
+ );
length++;
- if (length == schema.size()) {
+ if (length == columnAliases.size()) {
rows.add(new ResultRow(Lists.newArrayList(currentPath)));
}
} else if (element instanceof JsonObject && ((JsonObject) element).get(ResultSetConstants.VALUE_KEY) != null) {
- currentPath.set(schema.indexOf(key),
- ((JsonObject) element).get(ResultSetConstants.VALUE_KEY).getAsString());
+ final int index = columnAliases.indexOf(key);
+ currentPath.set(
+ index,
+ getTypedValue(index, ((JsonObject) element).get(ResultSetConstants.VALUE_KEY))
+ );
length++;
- if (length == schema.size()) {
+ if (length == columnAliases.size()) {
rows.add(new ResultRow(Lists.newArrayList(currentPath)));
}
} else if (element instanceof JsonObject) {
@@ -113,7 +121,7 @@ public abstract class JestResultSetTransformer {
return new ESResultSet(
rows.size(),
rows,
- getMetaData(schema)
+ getMetaData(columnAliases)
);
}
}
@@ -142,30 +150,34 @@ public abstract class JestResultSetTransformer {
.getAsJsonObject()
.getAsJsonObject(ResultSetConstants.FIELDS_KEY)
.entrySet()) {
+ int index = columnNames.indexOf(entry.getKey());
objects.set(
- selectedColumns.indexOf(entry.getKey())
- , entry.getValue().getAsString()
+ index
+ , getTypedValue(index, entry.getValue().getAsJsonArray().get(0))
);
}
rows.add(new ResultRow(objects));
}
- return new ESResultSet(rows.size(), rows, getMetaData(schema));
+ return new ESResultSet(rows.size(), rows, getMetaData(columnAliases));
}
-
}
@NonNull
protected final JsonObject result;
@NonNull
- protected final List<String> schema;
+ protected final List<String> columnAliases;
@NonNull
- protected final List<String> selectedColumns;
+ protected final List<String> columnNames;
+ protected final List<Type> columnDataTypes = Lists.newArrayList();
- public JestResultSetTransformer(JsonObject result, List<String> schema, List<String> selectedColumns) {
- this.schema = schema;
+ public JestResultSetTransformer(JsonObject result, List<String> columnAliases, List<String> columnNames) {
+ this.columnAliases = columnAliases;
this.result = result;
- this.selectedColumns = selectedColumns;
+ this.columnNames = columnNames;
+ for(int i=0; i< columnAliases.size(); i++) {
+ columnDataTypes.add(Type.NULL_TYPE);
+ }
}
public static ESResultSet transformFrom(JsonObject jsonResult, List<String> schema, List<String> selectedColumns) {
@@ -179,12 +191,48 @@ public abstract class JestResultSetTransformer {
protected List<Object> getEmptyRow() {
List<Object> objects = Lists.newArrayList();
int i = 0;
- while (i++ < schema.size()) {
+ while (i++ < columnAliases.size()) {
objects.add(null);
}
return objects;
}
+ protected Object getTypedValue(int colPosition, JsonElement jsonObjectValue) {
+ final Type type = getDataType(colPosition, jsonObjectValue);
+ switch (type) {
+ case NULL_TYPE:
+ return null;
+ case DOUBLE_TYPE:
+ return jsonObjectValue.getAsDouble();
+ case BOOLEAN_TYPE:
+ return jsonObjectValue.getAsBoolean();
+ default:
+ return jsonObjectValue.getAsString();
+ }
+ }
+
+ private Type getDataType(int colPosition, JsonElement jsonObjectValue) {
+ if (columnDataTypes.get(colPosition) != Type.NULL_TYPE) {
+ return columnDataTypes.get(colPosition);
+ }
+
+ final JsonPrimitive jsonPrimitive = jsonObjectValue.getAsJsonPrimitive();
+ if (jsonPrimitive.isJsonNull()) {
+ return Type.NULL_TYPE;
+ }
+
+ final Type type;
+ if (jsonPrimitive.isBoolean()) {
+ type = Type.BOOLEAN_TYPE;
+ } else if (jsonPrimitive.isNumber()) {
+ type = Type.DOUBLE_TYPE;
+ } else {
+ type = Type.STRING_TYPE;
+ }
+ columnDataTypes.set(colPosition, type);
+ return type;
+ }
+
public abstract ESResultSet transform();
protected LensResultSetMetadata getMetaData(final List<String> schema) {
@@ -195,8 +243,9 @@ public abstract class JestResultSetTransformer {
int i = 0;
for (final String col : schema) {
descriptors.add(
- new ColumnDescriptor(col, col, new TypeDescriptor(Type.STRING_TYPE), i++)
+ new ColumnDescriptor(col, col, new TypeDescriptor(columnDataTypes.get(i)), i)
);
+ i++;
}
return descriptors;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c224bca3/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java
index 82835b1..0b78639 100644
--- a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java
+++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java
@@ -161,14 +161,14 @@ public class ResultSetTransformationTest extends ESDriverTest {
+ " \"key\": \"g2v1\",\n"
+ " \"doc_count\": 10432335,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 1\n"
+ + " \"value\": 1.0\n"
+ " }\n"
+ " },\n"
+ " {\n"
+ " \"key\": \"g2v2\",\n"
+ " \"doc_count\": 2,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 2\n"
+ + " \"value\": 2.0\n"
+ " }\n"
+ " }\n"
+ " ]\n"
@@ -185,7 +185,7 @@ public class ResultSetTransformationTest extends ESDriverTest {
+ " \"key\": \"g2v3\",\n"
+ " \"doc_count\": 3,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 3\n"
+ + " \"value\": 3.0\n"
+ " }\n"
+ " }\n"
+ " ]\n"
@@ -201,9 +201,9 @@ public class ResultSetTransformationTest extends ESDriverTest {
new ESResultSet(
3,
Lists.newArrayList(
- new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v1", "1")),
- new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v2", "2")),
- new ResultRow(Lists.<Object>newArrayList("g1v2", "g2v3", "3"))
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v1", 1.0)),
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v2", 2.0)),
+ new ResultRow(Lists.<Object>newArrayList("g1v2", "g2v3", 3.0))
),
new LensResultSetMetadata() {
@Override
@@ -211,7 +211,7 @@ public class ResultSetTransformationTest extends ESDriverTest {
return Lists.newArrayList(
new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0),
new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 1),
- new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.STRING_TYPE), 2)
+ new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.DOUBLE_TYPE), 2)
);
}
})
@@ -252,14 +252,14 @@ public class ResultSetTransformationTest extends ESDriverTest {
+ " \"key\": \"g2v1\",\n"
+ " \"doc_count\": 10432335,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 1\n"
+ + " \"value\": 1.0\n"
+ " }\n"
+ " },\n"
+ " {\n"
+ " \"key\": \"g2v2\",\n"
+ " \"doc_count\": 2,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 2\n"
+ + " \"value\": 2.0\n"
+ " }\n"
+ " }\n"
+ " ]\n"
@@ -276,7 +276,7 @@ public class ResultSetTransformationTest extends ESDriverTest {
+ " \"key\": \"g2v3\",\n"
+ " \"doc_count\": 3,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 3\n"
+ + " \"value\": 3.0\n"
+ " }\n"
+ " }\n"
+ " ]\n"
@@ -292,16 +292,16 @@ public class ResultSetTransformationTest extends ESDriverTest {
new ESResultSet(
3,
Lists.newArrayList(
- new ResultRow(Lists.<Object>newArrayList("g1v1", "1", "g2v1")),
- new ResultRow(Lists.<Object>newArrayList("g1v1", "2", "g2v2")),
- new ResultRow(Lists.<Object>newArrayList("g1v2", "3", "g2v3"))
+ new ResultRow(Lists.<Object>newArrayList("g1v1", 1.0, "g2v1")),
+ new ResultRow(Lists.<Object>newArrayList("g1v1", 2.0, "g2v2")),
+ new ResultRow(Lists.<Object>newArrayList("g1v2", 3.0, "g2v3"))
),
new LensResultSetMetadata() {
@Override
public List<ColumnDescriptor> getColumns() {
return Lists.newArrayList(
new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0),
- new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.STRING_TYPE), 1),
+ new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.DOUBLE_TYPE), 1),
new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 2)
);
}
@@ -419,14 +419,14 @@ public class ResultSetTransformationTest extends ESDriverTest {
+ " \"key\": \"g2v1\",\n"
+ " \"doc_count\": 10432335,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 1\n"
+ + " \"value\": 1.0\n"
+ " }\n"
+ " },\n"
+ " {\n"
+ " \"key\": \"g2v2\",\n"
+ " \"doc_count\": 2,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 2\n"
+ + " \"value\": 2.0\n"
+ " }\n"
+ " }\n"
+ " ]\n"
@@ -443,7 +443,7 @@ public class ResultSetTransformationTest extends ESDriverTest {
+ " \"key\": \"g2v3\",\n"
+ " \"doc_count\": 3,\n"
+ " \"aggr_col\": {\n"
- + " \"value\": 3\n"
+ + " \"value\": 3.0\n"
+ " }\n"
+ " }\n"
+ " ]\n"
@@ -459,9 +459,9 @@ public class ResultSetTransformationTest extends ESDriverTest {
new ESResultSet(
3,
Lists.newArrayList(
- new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v1", "1")),
- new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v2", "2")),
- new ResultRow(Lists.<Object>newArrayList("g1v2", "g2v3", "3"))
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v1", 1.0)),
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v2", 2.0)),
+ new ResultRow(Lists.<Object>newArrayList("g1v2", "g2v3", 3.0))
),
new LensResultSetMetadata() {
@Override
@@ -469,7 +469,7 @@ public class ResultSetTransformationTest extends ESDriverTest {
return Lists.newArrayList(
new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0),
new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 1),
- new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.STRING_TYPE), 2)
+ new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.DOUBLE_TYPE), 2)
);
}
})