You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ad...@apache.org on 2016/09/13 01:32:01 UTC
[14/50] [abbrv] drill git commit: Adding support for Json tables.
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
new file mode 100644
index 0000000..2f2e561
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
@@ -0,0 +1,386 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.maprdb.json;
+
+import static org.ojai.DocumentConstants.ID_FIELD;
+import static org.ojai.DocumentConstants.ID_KEY;
+import io.netty.buffer.DrillBuf;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.ops.OperatorStats;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.store.AbstractRecordReader;
+import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec;
+import org.apache.drill.exec.store.maprdb.util.CommonFns;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter;
+import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
+import org.ojai.DocumentReader;
+import org.ojai.DocumentReader.EventType;
+import org.ojai.DocumentStream;
+import org.ojai.FieldPath;
+import org.ojai.Value;
+import org.ojai.store.QueryCondition;
+import org.ojai.store.QueryCondition.Op;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.mapr.db.DBDocument;
+import com.mapr.db.MapRDB;
+import com.mapr.db.Table;
+import com.mapr.db.Table.TableOption;
+import com.mapr.db.exceptions.DBException;
+import com.mapr.db.impl.IdCodec;
+import com.mapr.db.ojai.DBDocumentReader;
+import com.mapr.db.util.ByteBufs;
+import com.mapr.org.apache.hadoop.hbase.util.Bytes;
+
+public class MaprDBJsonRecordReader extends AbstractRecordReader {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MaprDBJsonRecordReader.class);
+
+ public static final SchemaPath ID_PATH = SchemaPath.getSimplePath(ID_KEY);
+
+ private Table table;
+ private QueryCondition condition;
+ private FieldPath[] projectedFields;
+
+ private String tableName;
+ private OperatorContext operatorContext;
+ private VectorContainerWriter writer;
+
+ @SuppressWarnings("unused")
+ private boolean idOnly;
+
+ private DrillBuf buffer;
+
+ private DocumentStream<DBDocument> documentStream;
+
+ private Iterator<DocumentReader> documentReaderIterators;
+
+ public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec,
+ List<SchemaPath> projectedColumns, FragmentContext context) {
+ buffer = context.getManagedBuffer();
+ tableName = Preconditions.checkNotNull(subScanSpec, "MapRDB reader needs a sub-scan spec").getTableName();
+ condition = MapRDB.newCondition().and();
+ addKeyCondition(condition, Op.GREATER_OR_EQUAL, subScanSpec.getStartRow());
+ addKeyCondition(condition, Op.LESS, subScanSpec.getStopRow());
+ if (subScanSpec.getSerializedFilter() != null) {
+ condition.condition(com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter())));
+ }
+ condition.close().build();
+ setColumns(projectedColumns);
+ }
+
+ private void addKeyCondition(QueryCondition condition, Op op, byte[] key) {
+ if (!CommonFns.isNullOrEmpty(key)) {
+ Value value = IdCodec.decode(key);
+ switch (value.getType()) {
+ case STRING:
+ condition.is(ID_FIELD, op, value.getString());
+ return;
+ case BINARY:
+ condition.is(ID_FIELD, op, value.getBinary());
+ return;
+ default:
+ throw new UnsupportedOperationException("");
+ }
+ }
+ }
+
+ @Override
+ protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns) {
+ Set<SchemaPath> transformed = Sets.newLinkedHashSet();
+ idOnly = true; // TODO: handle the case when only ID is requested.
+ if (!isStarQuery()) {
+ ArrayList<Object> projectedFieldsList = Lists.newArrayList();
+ for (SchemaPath column : columns) {
+ if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
+ transformed.add(ID_PATH);
+ continue;
+ }
+ idOnly = false;
+ projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath()));
+ }
+ projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]);
+ } else {
+ idOnly = false;
+ transformed.add(ID_PATH);
+ }
+
+ return transformed;
+ }
+
+ @Override
+ public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException {
+ this.writer = new VectorContainerWriter(output);
+ this.operatorContext = context;
+
+ try {
+ table = MapRDB.getTable(tableName);
+ table.setOption(TableOption.EXCLUDEID, true);
+ documentStream = table.find(condition, projectedFields);
+ documentReaderIterators = documentStream.documentReaders().iterator();
+ } catch (DBException e) {
+ throw new ExecutionSetupException(e);
+ }
+ }
+
+ @Override
+ public int next() {
+ Stopwatch watch = new Stopwatch();
+ watch.start();
+
+ writer.allocate();
+ writer.reset();
+
+ int recordCount = 0;
+
+ while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION) {
+ DBDocumentReader reader = nextDocumentReader();
+ if (reader == null) break;
+ writer.setPosition(recordCount);
+ if (reader.next() != EventType.START_MAP) {
+ throw new IllegalStateException("The document did not start with START_MAP!");
+ }
+ try {
+ MapWriter map = writer.rootAsMap();
+ if (reader.getId() != null) {
+ switch (reader.getId().getType()) {
+ case BINARY:
+ writeBinary(map.varBinary(ID_KEY), reader.getId().getBinary());
+ break;
+ case STRING:
+ writeString(map.varChar(ID_KEY), reader.getId().getString());
+ break;
+ default:
+ throw new UnsupportedOperationException(reader.getId().getType() +
+ " is not a supported type for _id field.");
+ }
+ }
+ writeToMap(reader, map);
+ recordCount++;
+ } catch (IllegalStateException e) {
+ logger.warn(String.format("Possible schema change at _id: %s",
+ IdCodec.asString(reader.getId())), e);
+ }
+ }
+
+ writer.setValueCount(recordCount);
+ logger.debug("Took {} ms to get {} records", watch.elapsed(TimeUnit.MILLISECONDS), recordCount);
+ return recordCount;
+ }
+
+ private void writeToMap(DBDocumentReader reader, MapWriter map) {
+ String fieldName = null;
+ map.start();
+ outside: while (true) {
+ EventType event = reader.next();
+ if (event == null) break outside;
+ switch (event) {
+ case FIELD_NAME:
+ fieldName = reader.getFieldName();
+ break;
+ case NULL:
+ map.varChar(fieldName).write(null); // treat as VARCHAR for now
+ case BINARY:
+ writeBinary(map.varBinary(fieldName), reader.getBinary());
+ break;
+ case BOOLEAN:
+ map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0);
+ break;
+ case STRING:
+ writeString(map.varChar(fieldName), reader.getString());
+ break;
+ case BYTE:
+ map.tinyInt(fieldName).writeTinyInt(reader.getByte());
+ break;
+ case SHORT:
+ map.smallInt(fieldName).writeSmallInt(reader.getShort());
+ break;
+ case INT:
+ map.integer(fieldName).writeInt(reader.getInt());
+ break;
+ case LONG:
+ map.bigInt(fieldName).writeBigInt(reader.getLong());
+ break;
+ case FLOAT:
+ map.float4(fieldName).writeFloat4(reader.getFloat());
+ break;
+ case DOUBLE:
+ map.float8(fieldName).writeFloat8(reader.getDouble());
+ break;
+ case DECIMAL:
+ throw new UnsupportedOperationException("Decimals are currently not supported.");
+ case DATE:
+ map.date(fieldName).writeDate(reader.getDate().getTime());
+ break;
+ case TIME:
+ map.time(fieldName).writeTime(reader.getTimeInt());
+ break;
+ case TIMESTAMP:
+ map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong());
+ break;
+ case INTERVAL:
+ throw new UnsupportedOperationException("Interval is currently not supported.");
+ case START_MAP:
+ writeToMap(reader, map.map(fieldName));
+ break;
+ case END_MAP:
+ break outside;
+ case START_ARRAY:
+ writeToList(reader, map.list(fieldName));
+ break;
+ case END_ARRAY:
+ throw new IllegalStateException("Shouldn't get a END_ARRAY inside a map");
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + event);
+ }
+ }
+ map.end();
+ }
+
+ private void writeToList(DBDocumentReader reader, ListWriter list) {
+ list.start();
+ outside: while (true) {
+ EventType event = reader.next();
+ if (event == null) break outside;
+ switch (event) {
+ case FIELD_NAME:
+ throw new IllegalStateException("Shouldn't get a field name inside a list");
+ case NULL:
+ list.varChar().write(null); // treat as VARCHAR for now
+ case BINARY:
+ writeBinary(list.varBinary(), reader.getBinary());
+ break;
+ case BOOLEAN:
+ list.bit().writeBit(reader.getBoolean() ? 1 : 0);
+ break;
+ case STRING:
+ writeString(list.varChar(), reader.getString());
+ break;
+ case BYTE:
+ list.tinyInt().writeTinyInt(reader.getByte());
+ break;
+ case SHORT:
+ list.smallInt().writeSmallInt(reader.getShort());
+ break;
+ case INT:
+ list.integer().writeInt(reader.getInt());
+ break;
+ case LONG:
+ list.bigInt().writeBigInt(reader.getLong());
+ break;
+ case FLOAT:
+ list.float4().writeFloat4(reader.getFloat());
+ break;
+ case DOUBLE:
+ list.float8().writeFloat8(reader.getDouble());
+ break;
+ case DECIMAL:
+ throw new UnsupportedOperationException("Decimals are currently not supported.");
+ case DATE:
+ list.date().writeDate(reader.getDate().getTime());
+ break;
+ case TIME:
+ list.time().writeTime(reader.getTimeInt());
+ break;
+ case TIMESTAMP:
+ list.timeStamp().writeTimeStamp(reader.getTimestampLong());
+ break;
+ case INTERVAL:
+ throw new UnsupportedOperationException("Interval is currently not supported.");
+ case START_MAP:
+ writeToMap(reader, list.map());
+ break;
+ case END_MAP:
+ throw new IllegalStateException("Shouldn't get a END_MAP inside a list");
+ case START_ARRAY:
+ writeToList(reader, list.list());
+ break;
+ case END_ARRAY:
+ break outside;
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + event);
+ }
+ }
+ list.end();
+ }
+
+ private void writeBinary(VarBinaryWriter binaryWriter, ByteBuffer buf) {
+ buffer.reallocIfNeeded(buf.remaining());
+ buffer.setBytes(0, buf, buf.position(), buf.remaining());
+ binaryWriter.writeVarBinary(0, buf.remaining(), buffer);
+ }
+
+ private void writeString(VarCharWriter varCharWriter, String string) {
+ final byte[] strBytes = Bytes.toBytes(string);
+ buffer.reallocIfNeeded(strBytes.length);
+ buffer.setBytes(0, strBytes);
+ varCharWriter.writeVarChar(0, strBytes.length, buffer);
+ }
+
+ private DBDocumentReader nextDocumentReader() {
+ final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats();
+ try {
+ if (operatorStats != null) {
+ operatorStats.startWait();
+ }
+ try {
+ if (!documentReaderIterators.hasNext()) {
+ return null;
+ } else {
+ return (DBDocumentReader) documentReaderIterators.next();
+ }
+ } finally {
+ if (operatorStats != null) {
+ operatorStats.stopWait();
+ }
+ }
+ } catch (DBException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ if (documentStream != null) {
+ documentStream.close();
+ }
+ if (table != null) {
+ table.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java
new file mode 100644
index 0000000..894e5bd
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.maprdb.util;
+
+public class CommonFns {
+
+ public static boolean isNullOrEmpty(final byte[] key) {
+ return key == null || key.length == 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java
new file mode 100644
index 0000000..cd1333a
--- /dev/null
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.mapr.drill.maprdb.tests;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.dfs.FileSystemConfig;
+import org.apache.drill.hbase.HBaseTestsSuite;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+import org.ojai.Document;
+import org.ojai.DocumentStream;
+import org.ojai.json.Json;
+
+import com.mapr.db.Admin;
+import com.mapr.db.MapRDB;
+import com.mapr.db.Table;
+import com.mapr.drill.maprdb.tests.binary.TestMapRDBFilterPushDown;
+import com.mapr.drill.maprdb.tests.binary.TestMapRDBSimple;
+import com.mapr.drill.maprdb.tests.json.TestSimpleJson;
+
+@RunWith(Suite.class)
+@SuiteClasses({
+ TestMapRDBSimple.class,
+ TestMapRDBFilterPushDown.class,
+ TestSimpleJson.class
+})
+public class MaprDBTestsSuite {
+ private static final String TMP_BUSINESS_TABLE = "/tmp/business";
+
+ private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
+
+ private static volatile AtomicInteger initCount = new AtomicInteger(0);
+ private static volatile Configuration conf;
+
+ private static Admin admin;
+
+ @BeforeClass
+ public static void setupTests() throws Exception {
+ if (initCount.get() == 0) {
+ synchronized (MaprDBTestsSuite.class) {
+ if (initCount.get() == 0) {
+ HBaseTestsSuite.configure(false, true);
+ HBaseTestsSuite.initCluster();
+ createJsonTables();
+
+ // Sleep to allow table data to be flushed to tables.
+ // Without this, the row count stats to return 0,
+ // causing the planner to reject optimized plans.
+ System.out.println("Sleeping for 5 seconds to allow table flushes");
+ Thread.sleep(5000);
+
+ conf = HBaseTestsSuite.getConf();
+ initCount.incrementAndGet(); // must increment while inside the synchronized block
+ return;
+ }
+ }
+ }
+ initCount.incrementAndGet();
+ return;
+ }
+
+ @AfterClass
+ public static void cleanupTests() throws Exception {
+ synchronized (MaprDBTestsSuite.class) {
+ if (initCount.decrementAndGet() == 0) {
+ HBaseTestsSuite.tearDownCluster();
+ deleteJsonTables();
+ }
+ }
+ }
+
+ private static volatile boolean pluginCreated;
+
+ public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws Exception {
+ if (!pluginCreated) {
+ synchronized (MaprDBTestsSuite.class) {
+ if (!pluginCreated) {
+ String pluginConfStr = "{" +
+ " \"type\": \"file\"," +
+ " \"enabled\": true," +
+ " \"connection\": \"maprfs:///\"," +
+ " \"workspaces\": {" +
+ " \"default\": {" +
+ " \"location\": \"/tmp\"," +
+ " \"writable\": false," +
+ " \"defaultInputFormat\": \"maprdb\"" +
+ " }" +
+ " }," +
+ " \"formats\": {" +
+ " \"maprdb\": {" +
+ " \"type\": \"maprdb\"" +
+ " }" +
+ " }" +
+ "}";
+
+ FileSystemConfig pluginConfig = ctx.getConfig().getMapper().readValue(pluginConfStr, FileSystemConfig.class);
+ // create the plugin with "hbase" name so that we can run HBase unit tests against them
+ ctx.getStorage().createOrUpdate("hbase", pluginConfig, true);
+ }
+ }
+ }
+ return conf;
+ }
+
+ public static boolean isDebug() {
+ return IS_DEBUG;
+ }
+
+ public static InputStream getJsonStream(String resourceName) {
+ return MaprDBTestsSuite.class.getClassLoader().getResourceAsStream(resourceName);
+ }
+
+ public static void createJsonTables() throws IOException {
+ admin = MapRDB.newAdmin();
+ if (admin.tableExists(TMP_BUSINESS_TABLE)) {
+ admin.deleteTable(TMP_BUSINESS_TABLE);
+ }
+
+ try (Table table = admin.createTable(TMP_BUSINESS_TABLE);
+ InputStream in = getJsonStream("json/business.json");
+ DocumentStream<Document> stream = Json.newDocumentStream(in)) {
+ for (Document document : stream) {
+ table.insert(document, "business_id");
+ }
+ table.flush();
+ }
+ }
+
+ public static void deleteJsonTables() {
+ if (admin != null) {
+ if (admin.tableExists(TMP_BUSINESS_TABLE)) {
+ admin.deleteTable(TMP_BUSINESS_TABLE);
+ }
+ admin.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java
new file mode 100644
index 0000000..b049f37
--- /dev/null
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.mapr.drill.maprdb.tests.binary;
+
+import org.apache.drill.hbase.TestHBaseFilterPushDown;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+import com.mapr.drill.maprdb.tests.MaprDBTestsSuite;
+import com.mapr.tests.annotations.ClusterTest;
+
+/**
+ * This class does not define any test method but includes all test methods
+ * defined in the parent class, all of which are tested against MapRDB instead
+ * of HBase.
+ */
+@Category(ClusterTest.class)
+public class TestMapRDBFilterPushDown extends TestHBaseFilterPushDown {
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ MaprDBTestsSuite.setupTests();
+ conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ MaprDBTestsSuite.cleanupTests();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java
new file mode 100644
index 0000000..894e64d
--- /dev/null
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.mapr.drill.maprdb.tests.binary;
+
+import org.apache.drill.hbase.BaseHBaseTest;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.mapr.drill.maprdb.tests.MaprDBTestsSuite;
+import com.mapr.tests.annotations.ClusterTest;
+
+@Category(ClusterTest.class)
+public class TestMapRDBSimple extends BaseHBaseTest {
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ MaprDBTestsSuite.setupTests();
+ conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ MaprDBTestsSuite.cleanupTests();
+ }
+
+ @Test
+ public void testMe() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`[TABLE_NAME]` tableName";
+ runHBaseSQLVerifyCount(sql, 7);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
new file mode 100644
index 0000000..c92fc44
--- /dev/null
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.mapr.drill.maprdb.tests.json;
+
+import java.util.List;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.mapr.drill.maprdb.tests.MaprDBTestsSuite;
+import com.mapr.tests.annotations.ClusterTest;
+
+@Category(ClusterTest.class)
+public class TestSimpleJson extends BaseTestQuery {
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ MaprDBTestsSuite.setupTests();
+ MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ MaprDBTestsSuite.cleanupTests();
+ }
+
+ @Test
+ public void testMe() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business";
+ runSQLAndVerifyCount(sql, 10);
+ }
+
+ protected List<QueryDataBatch> runHBaseSQLlWithResults(String sql) throws Exception {
+ System.out.println("Running query:\n" + sql);
+ return testSqlWithResults(sql);
+ }
+
+ protected void runSQLAndVerifyCount(String sql, int expectedRowCount) throws Exception{
+ List<QueryDataBatch> results = runHBaseSQLlWithResults(sql);
+ printResultAndVerifyRowCount(results, expectedRowCount);
+ }
+
+ private void printResultAndVerifyRowCount(List<QueryDataBatch> results, int expectedRowCount) throws SchemaChangeException {
+ int rowCount = printResult(results);
+ if (expectedRowCount != -1) {
+ Assert.assertEquals(expectedRowCount, rowCount);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/resources/hbase-site.xml b/contrib/format-maprdb/src/test/resources/hbase-site.xml
new file mode 100644
index 0000000..92e8a86
--- /dev/null
+++ b/contrib/format-maprdb/src/test/resources/hbase-site.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <property>
+ <name>hbase.table.namespace.mappings</name>
+ <value>*:/tmp/</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/resources/json/business.json
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/resources/json/business.json b/contrib/format-maprdb/src/test/resources/json/business.json
new file mode 100644
index 0000000..e1d46ac
--- /dev/null
+++ b/contrib/format-maprdb/src/test/resources/json/business.json
@@ -0,0 +1,10 @@
+{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"b
runch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":tru
e,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"roma
ntic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"}
+{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/resources/logback.xml b/contrib/format-maprdb/src/test/resources/logback.xml
index 00cc85b..38c2fc8 100644
--- a/contrib/format-maprdb/src/test/resources/logback.xml
+++ b/contrib/format-maprdb/src/test/resources/logback.xml
@@ -28,7 +28,7 @@
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<!-- The property 'logback.log.dir' is defined in pom.xml -->
- <file>${logback.log.dir:-./target/surefire-reports}/hbase-tests-${bySecond}.log</file>
+ <file>${logback.log.dir:-./target/surefire-reports}/maprdb-tests-${bySecond}.log</file>
<append>false</append>
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
@@ -41,6 +41,16 @@
</encoder>
</appender>
+ <logger name="com.mapr" additivity="false">
+ <level value="info" />
+ <appender-ref ref="FILE" />
+ </logger>
+
+ <logger name="com.mapr" additivity="false">
+ <level value="debug" />
+ <appender-ref ref="SOCKET" />
+ </logger>
+
<logger name="org.apache.drill" additivity="false">
<level value="info" />
<appender-ref ref="FILE" />