You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/02/21 19:39:53 UTC
[impala] 05/13: IMPALA-7140 (part 1). Support fetching schema info
in LocalCatalog
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch 2.x
in repository https://gitbox.apache.org/repos/asf/impala.git
commit db138c21a31116cc95ce93bd23d530bb090ab760
Author: Todd Lipcon <to...@cloudera.com>
AuthorDate: Wed Jun 6 18:17:19 2018 -0700
IMPALA-7140 (part 1). Support fetching schema info in LocalCatalog
This adds support for loading the Table object from HMS and parsing out
the Column and partitioning information from it.
With this change, I'm able to connect to an impalad running in "local
catalog" mode and run DESCRIBE, DESCRIBE EXTENDED, and SHOW CREATE TABLE
commands. Other commands like SHOW PARTITIONS don't work properly yet,
and type-specific table functionality (eg views, HBase tables, etc) are
not yet supported.
Again a simple unit test is included to check that column information is
loaded. More thorough testing is deferred until we've reached enough
coverage that we can start running e2e tests against a cluster running
in "local" mode.
Change-Id: I640f27e36198955e057da62a3ce25a858406e496
Reviewed-on: http://gerrit.cloudera.org:8080/10630
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Todd Lipcon <to...@apache.org>
---
.../org/apache/impala/catalog/FeCatalogUtils.java | 123 ++++++++++++++++
.../java/org/apache/impala/catalog/FeFsTable.java | 3 +
.../java/org/apache/impala/catalog/HdfsTable.java | 9 +-
.../main/java/org/apache/impala/catalog/Table.java | 54 +------
.../catalog/local/LocalCatalogException.java | 5 +-
.../org/apache/impala/catalog/local/LocalDb.java | 4 +
.../apache/impala/catalog/local/LocalTable.java | 155 ++++++++++++++++++---
.../org/apache/impala/catalog/CatalogTest.java | 5 +-
.../impala/catalog/local/LocalCatalogTest.java | 21 ++-
9 files changed, 300 insertions(+), 79 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java b/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
new file mode 100644
index 0000000..15f139b
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Static utility functions shared between FeCatalog implementations.
+ */
+public abstract class FeCatalogUtils {
+ /**
+ * Gets the ColumnType from the given FieldSchema by using Impala's SqlParser.
+ *
+ * The type can either be:
+ * - Supported by Impala, in which case the type is returned.
+ * - A type Impala understands but is not yet implemented (e.g. date), the type is
+ * returned but type.IsSupported() returns false.
+ * - A supported type that exceeds an Impala limit, e.g., on the nesting depth.
+ * - A type Impala can't understand at all, and a TableLoadingException is thrown.
+ *
+ * Throws a TableLoadingException if the FieldSchema could not be parsed. In this
+ * case, 'tableName' is included in the error message.
+ */
+ public static Type parseColumnType(FieldSchema fs, String tableName)
+ throws TableLoadingException {
+ Type type = Type.parseColumnType(fs.getType());
+ if (type == null) {
+ throw new TableLoadingException(String.format(
+ "Unsupported type '%s' in column '%s' of table '%s'",
+ fs.getType(), fs.getName(), tableName));
+ }
+ if (type.exceedsMaxNestingDepth()) {
+ throw new TableLoadingException(String.format(
+ "Type exceeds the maximum nesting depth of %s:\n%s",
+ Type.MAX_NESTING_DEPTH, type.toSql()));
+ }
+ return type;
+ }
+
+ /**
+ * Convert a list of HMS FieldSchemas to internal Column types.
+ * @throws TableLoadingException if any type is invalid
+ */
+ public static ImmutableList<Column> fieldSchemasToColumns(
+ Iterable<FieldSchema> fieldSchemas,
+ String tableName) throws TableLoadingException {
+ int pos = 0;
+ ImmutableList.Builder<Column> ret = ImmutableList.builder();
+ for (FieldSchema s : fieldSchemas) {
+ Type type = parseColumnType(s, tableName);
+ ret.add(new Column(s.getName(), type, s.getComment(), pos));
+ ++pos;
+ }
+ return ret.build();
+ }
+
+ /**
+ * Validate that the clustering columns are valid for a table
+ *
+ * TODO(todd): consider refactoring to combine with
+ * HdfsTable.addColumnsFromFieldSchema
+ *
+ * @throws TableLoadingException if the columns are invalid
+ */
+ public static void validateClusteringColumns(
+ Iterable<Column> cols, String tableName)
+ throws TableLoadingException {
+ // Check if we support partitioning on columns of such a type.
+ for (Column c : cols) {
+ Type type = c.getType();
+ if (!type.supportsTablePartitioning()) {
+ throw new TableLoadingException(
+ String.format("Failed to load metadata for table '%s' because of " +
+ "unsupported partition-column type '%s' in partition column '%s'",
+ tableName, type.toString(), c.getName()));
+ }
+ }
+ }
+
+ /**
+ * Returns the value of the ROW_COUNT constant, or -1 if not found.
+ */
+ public static long getRowCount(Map<String, String> parameters) {
+ return getLongParam(StatsSetupConst.ROW_COUNT, parameters);
+ }
+
+ public static long getTotalSize(Map<String, String> parameters) {
+ return getLongParam(StatsSetupConst.TOTAL_SIZE, parameters);
+ }
+
+ private static long getLongParam(String key, Map<String, String> parameters) {
+ if (parameters == null) return -1;
+ String value = parameters.get(key);
+ if (value == null) return -1;
+ try {
+ return Long.valueOf(value);
+ } catch (NumberFormatException exc) {
+ // ignore
+ }
+ return -1;
+ }
+
+}
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java b/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
index 91059a4..c64b922 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
@@ -34,6 +34,9 @@ import org.apache.impala.util.ListMap;
* Frontend interface for interacting with a filesystem-backed table.
*/
public interface FeFsTable extends FeTable {
+ /** hive's default value for table property 'serialization.null.format' */
+ public static final String DEFAULT_NULL_COLUMN_VALUE = "\\N";
+
/**
* @return true if the table and all its partitions reside at locations which
* support caching (e.g. HDFS).
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 7ad9fb8..ef4f2eb 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -114,9 +114,6 @@ import com.google.common.collect.Sets;
*
*/
public class HdfsTable extends Table implements FeFsTable {
- // hive's default value for table property 'serialization.null.format'
- private static final String DEFAULT_NULL_COLUMN_VALUE = "\\N";
-
// Name of default partition for unpartitioned tables
private static final String DEFAULT_PARTITION_NAME = "";
@@ -809,7 +806,7 @@ public class HdfsTable extends Table implements FeFsTable {
// to this table's partition list. Skip the partition.
if (partition == null) continue;
if (msPartition.getParameters() != null) {
- partition.setNumRows(getRowCount(msPartition.getParameters()));
+ partition.setNumRows(FeCatalogUtils.getRowCount(msPartition.getParameters()));
}
if (!TAccessLevelUtil.impliesWriteAccess(partition.getAccessLevel())) {
// TODO: READ_ONLY isn't exactly correct because the it's possible the
@@ -1566,7 +1563,7 @@ public class HdfsTable extends Table implements FeFsTable {
// set NULL indicator string from table properties
nullColumnValue_ =
msTbl.getParameters().get(serdeConstants.SERIALIZATION_NULL_FORMAT);
- if (nullColumnValue_ == null) nullColumnValue_ = DEFAULT_NULL_COLUMN_VALUE;
+ if (nullColumnValue_ == null) nullColumnValue_ = FeFsTable.DEFAULT_NULL_COLUMN_VALUE;
// Excludes partition columns.
nonPartFieldSchemas_.addAll(msTbl.getSd().getCols());
@@ -1622,7 +1619,7 @@ public class HdfsTable extends Table implements FeFsTable {
// this table's partition list. Skip the partition.
if (partition == null) continue;
if (msPartition.getParameters() != null) {
- partition.setNumRows(getRowCount(msPartition.getParameters()));
+ partition.setNumRows(FeCatalogUtils.getRowCount(msPartition.getParameters()));
}
if (!TAccessLevelUtil.impliesWriteAccess(partition.getAccessLevel())) {
// TODO: READ_ONLY isn't exactly correct because the it's possible the
diff --git a/fe/src/main/java/org/apache/impala/catalog/Table.java b/fe/src/main/java/org/apache/impala/catalog/Table.java
index 6e7d784..cdeffbf 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Table.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Table.java
@@ -25,7 +25,6 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
@@ -174,8 +173,8 @@ public abstract class Table extends CatalogObjectImpl implements FeTable {
* Sets 'tableStats_' by extracting the table statistics from the given HMS table.
*/
public void setTableStats(org.apache.hadoop.hive.metastore.api.Table msTbl) {
- tableStats_ = new TTableStats(getRowCount(msTbl.getParameters()));
- tableStats_.setTotal_file_bytes(getTotalSize(msTbl.getParameters()));
+ tableStats_ = new TTableStats(FeCatalogUtils.getRowCount(msTbl.getParameters()));
+ tableStats_.setTotal_file_bytes(FeCatalogUtils.getTotalSize(msTbl.getParameters()));
}
public void addColumn(Column col) {
@@ -243,29 +242,6 @@ public abstract class Table extends CatalogObjectImpl implements FeTable {
}
/**
- * Returns the value of the ROW_COUNT constant, or -1 if not found.
- */
- protected static long getRowCount(Map<String, String> parameters) {
- return getLongParam(StatsSetupConst.ROW_COUNT, parameters);
- }
-
- protected static long getTotalSize(Map<String, String> parameters) {
- return getLongParam(StatsSetupConst.TOTAL_SIZE, parameters);
- }
-
- private static long getLongParam(String key, Map<String, String> parameters) {
- if (parameters == null) return -1;
- String value = parameters.get(key);
- if (value == null) return -1;
- try {
- return Long.valueOf(value);
- } catch (NumberFormatException exc) {
- // ignore
- }
- return -1;
- }
-
- /**
* Creates a table of the appropriate type based on the given hive.metastore.api.Table
* object.
*/
@@ -411,29 +387,11 @@ public abstract class Table extends CatalogObjectImpl implements FeTable {
}
/**
- * Gets the ColumnType from the given FieldSchema by using Impala's SqlParser.
- * Throws a TableLoadingException if the FieldSchema could not be parsed.
- * The type can either be:
- * - Supported by Impala, in which case the type is returned.
- * - A type Impala understands but is not yet implemented (e.g. date), the type is
- * returned but type.IsSupported() returns false.
- * - A supported type that exceeds an Impala limit, e.g., on the nesting depth.
- * - A type Impala can't understand at all, and a TableLoadingException is thrown.
+ * @see FeCatalogUtils#parseColumnType(FieldSchema, String)
*/
- protected Type parseColumnType(FieldSchema fs) throws TableLoadingException {
- Type type = Type.parseColumnType(fs.getType());
- if (type == null) {
- throw new TableLoadingException(String.format(
- "Unsupported type '%s' in column '%s' of table '%s'",
- fs.getType(), fs.getName(), getName()));
- }
- if (type.exceedsMaxNestingDepth()) {
- throw new TableLoadingException(String.format(
- "Type exceeds the maximum nesting depth of %s:\n%s",
- Type.MAX_NESTING_DEPTH, type.toSql()));
- }
- return type;
- }
+ protected Type parseColumnType(FieldSchema fs) throws TableLoadingException {
+ return FeCatalogUtils.parseColumnType(fs, getName());
+ }
@Override // FeTable
public Db getDb() { return db_; }
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalogException.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalogException.java
index 59e0be4..7b4624c 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalogException.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalogException.java
@@ -16,7 +16,6 @@
package org.apache.impala.catalog.local;
-
/**
* Exception indicating an error loading catalog information into
* the impalad.
@@ -31,4 +30,8 @@ public class LocalCatalogException extends RuntimeException {
public LocalCatalogException(String msg, Throwable cause) {
super(msg, cause);
}
+
+ public LocalCatalogException(Throwable cause) {
+ super(cause);
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
index b3bf010..2d223a6 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
@@ -172,4 +172,8 @@ class LocalDb implements FeDb {
tdb.setMetastore_db(getMetaStoreDb());
return tdb;
}
+
+ LocalCatalog getCatalog() {
+ return catalog_;
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
index 58c0580..d6b0c88 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
@@ -21,17 +21,30 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
+import javax.annotation.concurrent.Immutable;
+
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.impala.analysis.TableName;
import org.apache.impala.catalog.ArrayType;
import org.apache.impala.catalog.Column;
+import org.apache.impala.catalog.FeCatalogUtils;
import org.apache.impala.catalog.FeDb;
+import org.apache.impala.catalog.FeFsTable;
import org.apache.impala.catalog.FeTable;
+import org.apache.impala.catalog.StructField;
+import org.apache.impala.catalog.StructType;
+import org.apache.impala.catalog.TableLoadingException;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TTableDescriptor;
import org.apache.impala.thrift.TTableStats;
+import org.apache.thrift.TException;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
/**
* Table instance loaded from {@link LocalCatalog}.
@@ -43,19 +56,23 @@ class LocalTable implements FeTable {
private final LocalDb db_;
/** The lower-case name of the table. */
private final String name_;
+ private final SchemaInfo schemaInfo_;
public static LocalTable load(LocalDb db, String tblName) {
+ // In order to know which kind of table subclass to instantiate, we need
+ // to eagerly grab and parse the top-level Table object from the HMS.
+ SchemaInfo schemaInfo = SchemaInfo.load(db, tblName);
+
// TODO: change this function to instantiate the appropriate
// subclass based on the table type (eg view, hbase table, etc)
- return new LocalTable(db, tblName);
+ return new LocalTable(db, tblName, schemaInfo);
}
- public LocalTable(LocalDb db, String tblName) {
- Preconditions.checkNotNull(db);
- Preconditions.checkNotNull(tblName);
+ public LocalTable(LocalDb db, String tblName, SchemaInfo schemaInfo) {
+ this.db_ = Preconditions.checkNotNull(db);
+ this.name_ = Preconditions.checkNotNull(tblName);
+ this.schemaInfo_ = Preconditions.checkNotNull(schemaInfo);
Preconditions.checkArgument(tblName.toLowerCase().equals(tblName));
- this.db_ = db;
- this.name_ = tblName;
}
@Override
@@ -65,17 +82,18 @@ class LocalTable implements FeTable {
@Override
public Table getMetaStoreTable() {
- throw new UnsupportedOperationException("TODO");
+ return schemaInfo_.msTable_;
}
@Override
public String getStorageHandlerClassName() {
- throw new UnsupportedOperationException("TODO");
+ // Subclasses should override as appropriate.
+ return null;
}
@Override
public TCatalogObjectType getCatalogObjectType() {
- throw new UnsupportedOperationException("TODO");
+ return TCatalogObjectType.TABLE;
}
@Override
@@ -95,47 +113,53 @@ class LocalTable implements FeTable {
@Override
public ArrayList<Column> getColumns() {
- throw new UnsupportedOperationException("TODO");
+ // TODO(todd) why does this return ArrayList instead of List?
+ return new ArrayList<>(schemaInfo_.colsByPos_);
}
@Override
public List<Column> getColumnsInHiveOrder() {
- throw new UnsupportedOperationException("TODO");
+ ArrayList<Column> columns = Lists.newArrayList(getNonClusteringColumns());
+ columns.addAll(getClusteringColumns());
+ return columns;
}
@Override
public List<String> getColumnNames() {
- throw new UnsupportedOperationException("TODO");
+ return Column.toColumnNames(schemaInfo_.colsByPos_);
}
@Override
public List<Column> getClusteringColumns() {
- throw new UnsupportedOperationException("TODO");
+ return ImmutableList.copyOf(
+ schemaInfo_.colsByPos_.subList(0, schemaInfo_.numClusteringCols_));
}
@Override
public List<Column> getNonClusteringColumns() {
- throw new UnsupportedOperationException("TODO");
+ return ImmutableList.copyOf(schemaInfo_.colsByPos_.subList(
+ schemaInfo_.numClusteringCols_,
+ schemaInfo_.colsByPos_.size()));
}
@Override
public int getNumClusteringCols() {
- throw new UnsupportedOperationException("TODO");
+ return schemaInfo_.numClusteringCols_;
}
@Override
public boolean isClusteringColumn(Column c) {
- throw new UnsupportedOperationException("TODO");
+ return schemaInfo_.isClusteringColumn(c);
}
@Override
public Column getColumn(String name) {
- throw new UnsupportedOperationException("TODO");
+ return schemaInfo_.colsByName_.get(name.toLowerCase());
}
@Override
public ArrayType getType() {
- throw new UnsupportedOperationException("TODO");
+ return schemaInfo_.type_;
}
@Override
@@ -145,16 +169,107 @@ class LocalTable implements FeTable {
@Override
public long getNumRows() {
- throw new UnsupportedOperationException("TODO");
+ return schemaInfo_.tableStats_.num_rows;
}
@Override
public TTableStats getTTableStats() {
- throw new UnsupportedOperationException("TODO");
+ return schemaInfo_.tableStats_;
}
@Override
public TTableDescriptor toThriftDescriptor(int tableId, Set<Long> referencedPartitions) {
throw new UnsupportedOperationException("TODO");
}
+
+ /**
+ * The table schema, loaded from the HMS Table object. This is common
+ * to all Table implementations and includes the column definitions and
+ * table-level stats.
+ *
+ * TODO(todd): some of this code is lifted from 'Table' and, with some
+ * effort, could be refactored to avoid duplication.
+ */
+ @Immutable
+ private static class SchemaInfo {
+ private final Table msTable_;
+
+ private final ArrayType type_;
+ private final ImmutableList<Column> colsByPos_;
+ private final ImmutableMap<String, Column> colsByName_;
+
+ private final int numClusteringCols_;
+ private final String nullColumnValue_;
+
+ private final TTableStats tableStats_;
+
+ /**
+ * Load the schema info from the metastore.
+ */
+ static SchemaInfo load(LocalDb db, String tblName) {
+ try {
+ Table msTbl = db.getCatalog().getMetaProvider().loadTable(
+ db.getName(), tblName);
+ return new SchemaInfo(msTbl);
+ } catch (TException e) {
+ throw new LocalCatalogException(String.format(
+ "Could not load table %s.%s from metastore",
+ db.getName(), tblName), e);
+ } catch (TableLoadingException e) {
+ // In this case, the exception message already has the table name
+ // in the exception message.
+ throw new LocalCatalogException(e);
+ }
+ }
+
+ SchemaInfo(Table msTbl) throws TableLoadingException {
+ msTable_ = msTbl;
+ // set NULL indicator string from table properties
+ String tableNullFormat =
+ msTbl.getParameters().get(serdeConstants.SERIALIZATION_NULL_FORMAT);
+ nullColumnValue_ = tableNullFormat != null ? tableNullFormat :
+ FeFsTable.DEFAULT_NULL_COLUMN_VALUE;
+
+ final String fullName = msTbl.getDbName() + "." + msTbl.getTableName();
+
+ // The number of clustering columns is the number of partition keys.
+ numClusteringCols_ = msTbl.getPartitionKeys().size();
+ // Add all columns to the table. Ordering is important: partition columns first,
+ // then all other columns.
+ colsByPos_ = FeCatalogUtils.fieldSchemasToColumns(
+ Iterables.concat(msTbl.getPartitionKeys(),
+ msTbl.getSd().getCols()),
+ fullName);
+ FeCatalogUtils.validateClusteringColumns(
+ colsByPos_.subList(0, numClusteringCols_), fullName);
+ colsByName_ = indexColumnNames(colsByPos_);
+ type_ = new ArrayType(columnsToStructType(colsByPos_));
+
+ tableStats_ = new TTableStats(
+ FeCatalogUtils.getRowCount(msTable_.getParameters()));
+ tableStats_.setTotal_file_bytes(
+ FeCatalogUtils.getTotalSize(msTable_.getParameters()));
+ }
+
+ private static StructType columnsToStructType(List<Column> cols) {
+ ArrayList<StructField> fields = Lists.newArrayListWithCapacity(cols.size());
+ for (Column col : cols) {
+ fields.add(new StructField(col.getName(), col.getType(), col.getComment()));
+ }
+ return new StructType(fields);
+ }
+
+ private static ImmutableMap<String, Column> indexColumnNames(List<Column> cols) {
+ ImmutableMap.Builder<String, Column> builder = ImmutableMap.builder();
+ for (Column col : cols) {
+ builder.put(col.getName().toLowerCase(), col);
+ }
+ return builder.build();
+ }
+
+ private boolean isClusteringColumn(Column c) {
+ Preconditions.checkArgument(colsByPos_.get(c.getPosition()) == c);
+ return c.getPosition() < numClusteringCols_;
+ }
+ }
}
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
index 04fe6cf..f8a2682 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
@@ -37,7 +37,6 @@ import org.apache.impala.analysis.FunctionName;
import org.apache.impala.analysis.LiteralExpr;
import org.apache.impala.analysis.NumericLiteral;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
-import org.apache.impala.service.FeSupport;
import org.apache.impala.testutil.CatalogServiceTestCatalog;
import org.apache.impala.thrift.TFunctionBinaryType;
import org.junit.Test;
@@ -51,9 +50,9 @@ public class CatalogTest {
private static CatalogServiceCatalog catalog_ =
CatalogServiceTestCatalog.create();
- private void checkTableCols(Db db, String tblName, int numClusteringCols,
+ public static void checkTableCols(FeDb db, String tblName, int numClusteringCols,
String[] colNames, Type[] colTypes) throws TableLoadingException {
- Table tbl = db.getTable(tblName);
+ FeTable tbl = db.getTable(tblName);
assertEquals(tbl.getName(), tblName);
assertEquals(tbl.getNumClusteringCols(), numClusteringCols);
List<Column> cols = tbl.getColumns();
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index 52ff255..c8315df 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -21,8 +21,10 @@ import static org.junit.Assert.*;
import java.util.Set;
+import org.apache.impala.catalog.CatalogTest;
import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.FeTable;
+import org.apache.impala.catalog.Type;
import org.apache.impala.util.PatternMatcher;
import org.junit.Before;
import org.junit.Test;
@@ -57,7 +59,7 @@ public class LocalCatalogTest {
}
@Test
- public void testTables() throws Exception {
+ public void testListTables() throws Exception {
Set<String> names = ImmutableSet.copyOf(catalog_.getTableNames(
"functional", PatternMatcher.MATCHER_MATCH_ALL));
assertTrue(names.contains("alltypes"));
@@ -73,4 +75,21 @@ public class LocalCatalogTest {
assertEquals("functional.alltypes", t.getFullName());
}
+ @Test
+ public void testLoadTableBasics() throws Exception {
+ FeDb functionalDb = catalog_.getDb("functional");
+ CatalogTest.checkTableCols(functionalDb, "alltypes", 2,
+ new String[]
+ {"year", "month", "id", "bool_col", "tinyint_col", "smallint_col",
+ "int_col", "bigint_col", "float_col", "double_col", "date_string_col",
+ "string_col", "timestamp_col"},
+ new Type[]
+ {Type.INT, Type.INT, Type.INT,
+ Type.BOOLEAN, Type.TINYINT, Type.SMALLINT,
+ Type.INT, Type.BIGINT, Type.FLOAT,
+ Type.DOUBLE, Type.STRING, Type.STRING,
+ Type.TIMESTAMP});
+ FeTable t = functionalDb.getTable("alltypes");
+ assertEquals(7300, t.getNumRows());
+ }
}