You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/21 22:28:43 UTC
[5/5] git commit: DRILL-680: INFORMATION_SCHEMA.COLUMNS does not
display HBase column families
DRILL-680: INFORMATION_SCHEMA.COLUMNS does not display HBase column families
+ Enhanced result layout with option to set output width on per column basis.
+ Pretty print plan fragments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ae570aad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ae570aad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ae570aad
Branch: refs/heads/master
Commit: ae570aad8b572a5c1fe612f79fe2524cb5e4e879
Parents: ea1ab62
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue May 20 06:34:48 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed May 21 08:41:57 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/hbase/DrillHBaseTable.java | 62 ++++++++++++++++++++
.../exec/store/hbase/HBaseSchemaFactory.java | 6 +-
.../org/apache/drill/hbase/BaseHBaseTest.java | 12 ++--
.../drill/hbase/TestHBaseFilterPushDown.java | 6 ++
.../drill/hbase/TestHBaseProjectPushDown.java | 8 ++-
.../planner/fragment/SimpleParallelizer.java | 8 +--
.../org/apache/drill/exec/rpc/RpcException.java | 10 ++--
.../org/apache/drill/exec/util/VectorUtil.java | 52 ++++++++++------
8 files changed, 126 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
new file mode 100644
index 0000000..ff6991f
--- /dev/null
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.sql.type.SqlTypeName;
+
+public class DrillHBaseTable extends DrillTable implements DrillHBaseConstants {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHBaseTable.class);
+
+ private HTableDescriptor table;
+
+ public DrillHBaseTable(String storageEngineName, HBaseStoragePlugin plugin, HBaseScanSpec scanSpec) {
+ super(storageEngineName, plugin, scanSpec);
+ try(HBaseAdmin admin = new HBaseAdmin(plugin.getConfig().getHBaseConf())) {
+ table = admin.getTableDescriptor(HBaseUtils.getBytes(scanSpec.getTableName()));
+ } catch (IOException e) {
+ logger.warn("Failure while loading table names for database '{}'.", storageEngineName, e);
+ }
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ ArrayList<RelDataType> typeList = new ArrayList<>();
+ ArrayList<String> fieldNameList = new ArrayList<>();
+
+ fieldNameList.add(ROW_KEY);
+ typeList.add(typeFactory.createSqlType(SqlTypeName.ANY));
+
+ Set<byte[]> families = table.getFamiliesKeys();
+ for (byte[] family : families) {
+ fieldNameList.add(Bytes.toString(family));
+ typeList.add(typeFactory.createMapType(typeFactory.createSqlType(SqlTypeName.VARCHAR), typeFactory.createSqlType(SqlTypeName.ANY)));
+ }
+ return typeFactory.createStructType(typeList, fieldNameList);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
index 84f363b..7b76092 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
@@ -25,7 +25,6 @@ import net.hydromatic.optiq.Schema;
import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.Table;
-import org.apache.drill.exec.planner.logical.DynamicDrillTable;
import org.apache.drill.exec.rpc.user.UserSession;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.SchemaFactory;
@@ -74,8 +73,8 @@ public class HBaseSchemaFactory implements SchemaFactory {
@Override
public Table getTable(String name) {
- Object selection = new HBaseScanSpec(name);
- return new DynamicDrillTable(plugin, schemaName, selection);
+ HBaseScanSpec scanSpec = new HBaseScanSpec(name);
+ return new DrillHBaseTable(schemaName, plugin, scanSpec);
}
@Override
@@ -99,4 +98,5 @@ public class HBaseSchemaFactory implements SchemaFactory {
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
index 9e07d9f..96f0c4a 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
@@ -45,7 +45,7 @@ public class BaseHBaseTest extends BaseTestQuery {
@Rule public TestName TEST_NAME = new TestName();
- private int columnWidth = 8;
+ private int[] columnWidths = new int[] { 8 };
@Before
public void printID() throws Exception {
@@ -58,7 +58,7 @@ public class BaseHBaseTest extends BaseTestQuery {
* Change the following to HBaseTestsSuite.configure(false, true)
* if you want to test against an externally running HBase cluster.
*/
- HBaseTestsSuite.configure(true, true);
+ HBaseTestsSuite.configure(false, true);
HBaseTestsSuite.initCluster();
HBaseStoragePlugin plugin = (HBaseStoragePlugin) bit.getContext().getStorage().getPlugin("hbase");
@@ -71,9 +71,13 @@ public class BaseHBaseTest extends BaseTestQuery {
}
protected void setColumnWidth(int columnWidth) {
- this.columnWidth = columnWidth;
+ this.columnWidths = new int[] { columnWidth };
}
+ protected void setColumnWidths(int[] columnWidths) {
+ this.columnWidths = columnWidths;
+ }
+
protected String getPlanText(String planFile, String tableName) throws IOException {
return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8)
.replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort())
@@ -102,7 +106,7 @@ public class BaseHBaseTest extends BaseTestQuery {
if (loader.getRecordCount() <= 0) {
break;
}
- VectorUtil.showVectorAccessibleContent(loader, columnWidth);
+ VectorUtil.showVectorAccessibleContent(loader, columnWidths);
loader.clear();
result.release();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
index 90404b7..2071e90 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
@@ -23,6 +23,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyEqual() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -34,6 +35,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyGreaterThan() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -45,6 +47,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyBetween() throws Exception {
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -56,6 +59,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownMultiColumns() throws Exception {
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -67,6 +71,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownConvertExpression() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
@@ -78,6 +83,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
@Test
public void testFilterPushDownRowKeyLessThanOrEqualTo() throws Exception {
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
+ " *\n"
+ "FROM\n"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
index b66d2ed..7037b9a 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
@@ -23,6 +23,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testRowKeyPushDown() throws Exception{
+ setColumnWidth(8);
runSQLVerifyCount("SELECT\n"
+ "row_key\n"
+ "FROM\n"
@@ -32,6 +33,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testColumnWith1RowPushDown() throws Exception{
+ setColumnWidth(6);
runSQLVerifyCount("SELECT\n"
+ "f2['c7'] as `f[c7]`\n"
+ "FROM\n"
@@ -41,7 +43,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testRowKeyAndColumnPushDown() throws Exception{
- setColumnWidth(9);
+ setColumnWidths(new int[] {8, 9, 6, 2, 6});
runSQLVerifyCount("SELECT\n"
+ "row_key, f['c1']*31 as `f[c1]*31`, f['c2'] as `f[c2]`, 5 as `5`, 'abc' as `'abc'`\n"
+ "FROM\n"
@@ -51,9 +53,9 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
@Test
public void testColumnFamilyPushDown() throws Exception{
- setColumnWidth(74);
+ setColumnWidths(new int[] {8, 74, 38});
runSQLVerifyCount("SELECT\n"
- + "f, f2\n"
+ + "row_key, f, f2\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
, 6);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
index c34869d..313a81d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.planner.fragment;
import java.util.Collection;
import java.util.List;
+import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
import org.apache.drill.exec.exception.FragmentSetupException;
@@ -38,9 +39,6 @@ import org.apache.drill.exec.work.QueryWorkUnit;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import org.apache.drill.exec.expr.fn.impl.DateUtility;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
/**
* The simple parallelizer determines the level of parallelization of a plan based on the cost of the underlying
@@ -143,11 +141,11 @@ public class SimpleParallelizer {
.build();
if (isRootNode) {
- logger.debug("Root fragment:\n {}", fragment);
+ logger.debug("Root fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
rootFragment = fragment;
rootOperator = root;
} else {
- logger.debug("Remote fragment:\n {}", fragment);
+ logger.debug("Remote fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
fragments.add(fragment);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
index dba72a1..9b5eb1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
@@ -19,13 +19,14 @@ package org.apache.drill.exec.rpc;
import java.util.concurrent.ExecutionException;
-import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.DrillIOException;
/**
* Parent class for all rpc exceptions.
*/
public class RpcException extends DrillIOException{
+ private static final long serialVersionUID = -5964230316010502319L;
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcException.class);
public RpcException() {
@@ -47,17 +48,16 @@ public class RpcException extends DrillIOException{
public RpcException(Throwable cause) {
super(cause);
}
-
+
public static RpcException mapException(Throwable t){
while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause();
if(t instanceof RpcException) return ((RpcException) t);
return new RpcException(t);
}
-
+
public static RpcException mapException(String message, Throwable t){
while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause();
return new RpcException(message, t);
}
-
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index a43225e..bdba1d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -17,14 +17,11 @@
*/
package org.apache.drill.exec.util;
-import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
import com.beust.jcommander.internal.Lists;
@@ -37,7 +34,7 @@ public class VectorUtil {
int rows = va.getRecordCount();
List<String> columns = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath());
+ columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
}
int width = columns.size();
@@ -71,55 +68,74 @@ public class VectorUtil {
}
public static void showVectorAccessibleContent(VectorAccessible va) {
- showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH);
+ showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH);
}
+
public static void showVectorAccessibleContent(VectorAccessible va, int columnWidth) {
+ showVectorAccessibleContent(va, new int[]{ columnWidth });
+ }
- int rows = va.getRecordCount();
+ public static void showVectorAccessibleContent(VectorAccessible va, int[] columnWidths) {
+ int width = 0;
+ int columnIndex = 0;
List<String> columns = Lists.newArrayList();
+ List<String> formats = Lists.newArrayList();
for (VectorWrapper<?> vw : va) {
- columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath());
+ int columnWidth = getColumnWidth(columnWidths, columnIndex);
+ width += columnWidth + 2;
+ formats.add("| %-" + columnWidth + "s");
+ columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
+ columnIndex++;
}
- int width = columns.size() * (columnWidth + 2);
-
- String format = ("| %-" + columnWidth + "s");
-
+ int rows = va.getRecordCount();
for (int row = 0; row < rows; row++) {
+ // header, every 50 rows.
if (row%50 == 0) {
System.out.println(StringUtils.repeat("-", width + 1));
+ columnIndex = 0;
for (String column : columns) {
- System.out.printf(format, column.length() <= columnWidth ? column : column.substring(0, columnWidth - 1));
+ int columnWidth = getColumnWidth(columnWidths, columnIndex);
+ System.out.printf(formats.get(columnIndex), column.length() <= columnWidth ? column : column.substring(0, columnWidth - 1));
+ columnIndex++;
}
System.out.printf("|\n");
System.out.println(StringUtils.repeat("-", width + 1));
}
+ // column values
+ columnIndex = 0;
for (VectorWrapper<?> vw : va) {
+ int columnWidth = getColumnWidth(columnWidths, columnIndex);
Object o = vw.getValueVector().getAccessor().getObject(row);
if (o == null) {
//null value
- System.out.printf(format, "");
+ System.out.printf(formats.get(columnIndex), "");
}
else if (o instanceof byte[]) {
String value = new String((byte[]) o);
- System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0, columnWidth - 1));
+ System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0, columnWidth - 1));
} else if (o instanceof List) {
System.out.printf("| %s", o);
} else {
String value = o.toString();
- System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0,columnWidth - 1));
+ System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0,columnWidth - 1));
}
+ columnIndex++;
}
System.out.printf("|\n");
}
+ if (rows > 0) {
+ System.out.println(StringUtils.repeat("-", width + 1));
+ }
for (VectorWrapper<?> vw : va) {
vw.clear();
}
-
- if (rows > 0 )
- System.out.println(StringUtils.repeat("-", width + 1));
}
+ private static int getColumnWidth(int[] columnWidths, int columnIndex) {
+ return (columnWidths == null) ? DEFAULT_COLUMN_WIDTH
+ : (columnWidths.length > columnIndex) ? columnWidths[columnIndex] : columnWidths[0];
+ }
}