You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ad...@apache.org on 2016/09/13 01:32:14 UTC
[27/50] [abbrv] drill git commit: MD-703: Cannot select `_id` field
from MapR-DB JSON Table
MD-703: Cannot select `_id` field from MapR-DB JSON Table
+ Included Smidth's fix for column shuffling.
+ Fix the private-drill-mapr-plugin build.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/5d36f801
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/5d36f801
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/5d36f801
Branch: refs/heads/master
Commit: 5d36f8013829406f698610bc4a4f7185698365e6
Parents: 16d4ab8
Author: Aditya <ad...@mapr.com>
Authored: Thu Feb 4 16:43:27 2016 -0800
Committer: Aditya Kishore <ad...@apache.org>
Committed: Fri Sep 9 10:08:35 2016 -0700
----------------------------------------------------------------------
contrib/format-maprdb/pom.xml | 6 +++++
.../exec/store/maprdb/MapRDBGroupScan.java | 2 +-
.../store/maprdb/MapRDBPushFilterIntoScan.java | 4 ++--
.../maprdb/json/MaprDBJsonRecordReader.java | 24 ++++++++++++--------
.../drill/maprdb/tests/json/TestSimpleJson.java | 4 ++--
5 files changed, 26 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml
index bcb6c29..db0572f 100644
--- a/contrib/format-maprdb/pom.xml
+++ b/contrib/format-maprdb/pom.xml
@@ -35,6 +35,7 @@
<mapr.version>${project.parent.version}</mapr.version>
<drill.version>${project.version}</drill.version>
<hbase.version>0.98.12-mapr-1506</hbase.version>
+ <guava.version>18.0</guava.version>
<maprdb.TestSuite>**/MaprDBTestsSuite.class</maprdb.TestSuite>
</properties>
@@ -118,6 +119,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${drill.version}</version>
http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
index cbfb18c..4d30399 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
@@ -61,7 +61,7 @@ public abstract class MapRDBGroupScan extends AbstractGroupScan {
private boolean filterPushedDown = false;
- private Stopwatch watch = new Stopwatch();
+ private Stopwatch watch = Stopwatch.createUnstarted();
private static final Comparator<List<MapRDBSubScanSpec>> LIST_SIZE_COMPARATOR = new Comparator<List<MapRDBSubScanSpec>>() {
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
index 714221f..8a0902e 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
@@ -90,11 +90,11 @@ public abstract class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRul
if (scan.getGroupScan() instanceof BinaryTableGroupScan) {
BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan();
- doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition);
+ doPushFilterIntoBinaryGroupScan(call, filter, project, scan, groupScan, condition);
} else {
assert(scan.getGroupScan() instanceof JsonTableGroupScan);
JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan();
- doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition);
+ doPushFilterIntoJsonGroupScan(call, filter, project, scan, groupScan, condition);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
index 2e0e0c1..b69f4c4 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
@@ -17,11 +17,9 @@
*/
package org.apache.drill.exec.store.maprdb.json;
-import static org.ojai.DocumentConstants.ID_FIELD;
import static org.ojai.DocumentConstants.ID_KEY;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -51,7 +49,6 @@ import org.ojai.store.QueryCondition;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.mapr.db.MapRDB;
import com.mapr.db.Table;
@@ -98,18 +95,27 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns) {
Set<SchemaPath> transformed = Sets.newLinkedHashSet();
if (!isStarQuery()) {
- ArrayList<Object> projectedFieldsList = Lists.newArrayList();
+ Set<FieldPath> projectedFieldsSet = Sets.newTreeSet();
for (SchemaPath column : columns) {
if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
+ /*
+ * we do not include _id field in the set of projected fields
+ * because the DB currently can not return a document if only
+ * the _id field was projected. This should really be fixed in
+ * the DB client (Bug 21708) to avoid transferring the entire
+ * document when only _id is requested.
+ */
+ // projectedFieldsList.add(ID_FIELD);
transformed.add(ID_PATH);
- projectedFieldsList.add(ID_FIELD);
includeId = true;
} else {
transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath()));
- projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath()));
+ projectedFieldsSet.add(FieldPath.parseFrom(column.getAsUnescapedPath()));
}
}
- projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]);
+ if (projectedFieldsSet.size() > 0) {
+ projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]);
+ }
} else {
transformed.add(ID_PATH);
includeId = true;
@@ -135,7 +141,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
@Override
public int next() {
- Stopwatch watch = new Stopwatch();
+ Stopwatch watch = Stopwatch.createUnstarted();
watch.start();
writer.allocate();
@@ -230,7 +236,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
default:
throw unsupportedError("Unsupported type: %s encountered during the query.", event);
}
- } catch (IllegalArgumentException e) {
+ } catch (IllegalStateException | IllegalArgumentException e) {
logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'",
IdCodec.asString(reader.getId()), fieldName), e);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
index f05b87a..414b823 100644
--- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
@@ -48,9 +48,9 @@ public class TestSimpleJson extends BaseTestQuery {
@Test
public void testMe() throws Exception {
- setColumnWidths(new int[] {25, 40, 40, 40});
+ setColumnWidths(new int[] {23});
final String sql = "SELECT\n"
- + " _id, name, categories, full_address\n"
+ + " _id\n"
+ "FROM\n"
+ " hbase.`business` business";
runSQLAndVerifyCount(sql, 10);