You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/01/27 23:15:57 UTC
[41/51] [partial] Initial commit of master branch from github
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
new file mode 100644
index 0000000..92d8995
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -0,0 +1,472 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.compile;
+
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.parse.BindTableNode;
+import org.apache.phoenix.parse.ColumnDef;
+import org.apache.phoenix.parse.CreateTableStatement;
+import org.apache.phoenix.parse.DerivedTableNode;
+import org.apache.phoenix.parse.JoinTableNode;
+import org.apache.phoenix.parse.NamedTableNode;
+import org.apache.phoenix.parse.SelectStatement;
+import org.apache.phoenix.parse.SingleTableSQLStatement;
+import org.apache.phoenix.parse.TableName;
+import org.apache.phoenix.parse.TableNode;
+import org.apache.phoenix.parse.TableNodeVisitor;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.AmbiguousTableException;
+import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
+import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PColumnFamilyImpl;
+import org.apache.phoenix.schema.PColumnImpl;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.util.SchemaUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.io.Closeables;
+
+/**
+ * Validates FROM clause and builds a ColumnResolver for resolving column references
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class FromCompiler {
+ private static final Logger logger = LoggerFactory.getLogger(FromCompiler.class);
+
+ public static final ColumnResolver EMPTY_TABLE_RESOLVER = new ColumnResolver() {
+
+ @Override
+ public List<TableRef> getTables() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public ColumnRef resolveColumn(String schemaName, String tableName, String colName) throws SQLException {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ public static ColumnResolver getResolver(final CreateTableStatement statement, final PhoenixConnection connection)
+ throws SQLException {
+ TableName baseTable = statement.getBaseTableName();
+ if (baseTable == null) {
+ return EMPTY_TABLE_RESOLVER;
+ }
+ NamedTableNode tableNode = NamedTableNode.create(null, baseTable, Collections.<ColumnDef>emptyList());
+ // Always use non-tenant-specific connection here
+ try {
+ SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, tableNode, false, true);
+ return visitor;
+ } catch (TableNotFoundException e) {
+ // Used for mapped VIEW, since we won't be able to resolve that.
+ // Instead, we create a table with just the dynamic columns.
+ // A tenant-specific connection may not create a mapped VIEW.
+ if (connection.getTenantId() == null && statement.getTableType() == PTableType.VIEW) {
+ ConnectionQueryServices services = connection.getQueryServices();
+ byte[] fullTableName = SchemaUtil.getTableNameAsBytes(baseTable.getSchemaName(), baseTable.getTableName());
+ HTableInterface htable = null;
+ try {
+ htable = services.getTable(fullTableName);
+ } catch (UnsupportedOperationException ignore) {
+ throw e; // For Connectionless
+ } finally {
+ if (htable != null) Closeables.closeQuietly(htable);
+ }
+ tableNode = NamedTableNode.create(null, baseTable, statement.getColumnDefs());
+ return new SingleTableColumnResolver(connection, tableNode, e.getTimeStamp());
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Iterate through the nodes in the FROM clause to build a column resolver used to lookup a column given the name
+ * and alias.
+ *
+ * @param statement
+ * the select statement
+ * @return the column resolver
+ * @throws SQLException
+ * @throws SQLFeatureNotSupportedException
+ * if unsupported constructs appear in the FROM clause. Currently only a single table name is supported.
+ * @throws TableNotFoundException
+ * if table name not found in schema
+ */
+ public static ColumnResolver getResolver(SelectStatement statement, PhoenixConnection connection)
+ throws SQLException {
+ List<TableNode> fromNodes = statement.getFrom();
+ if (fromNodes.size() > 1) { throw new SQLFeatureNotSupportedException("Joins not supported"); }
+ SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, (NamedTableNode)fromNodes.get(0), false, false);
+ return visitor;
+ }
+
+ public static ColumnResolver getMultiTableResolver(SelectStatement statement, PhoenixConnection connection)
+ throws SQLException {
+ List<TableNode> fromNodes = statement.getFrom();
+ MultiTableColumnResolver visitor = new MultiTableColumnResolver(connection);
+ for (TableNode node : fromNodes) {
+ node.accept(visitor);
+ }
+ return visitor;
+ }
+
+ public static ColumnResolver getResolver(NamedTableNode tableNode, PhoenixConnection connection) throws SQLException {
+ SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, tableNode, false, false);
+ return visitor;
+ }
+
+ public static ColumnResolver getResolver(SingleTableSQLStatement statement, PhoenixConnection connection,
+ List<ColumnDef> dyn_columns) throws SQLException {
+ SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, statement.getTable(), true, false);
+ return visitor;
+ }
+
+ public static ColumnResolver getResolver(SingleTableSQLStatement statement, PhoenixConnection connection)
+ throws SQLException {
+ return getResolver(statement, connection, Collections.<ColumnDef>emptyList());
+ }
+
+ private static class SingleTableColumnResolver extends BaseColumnResolver {
+ private final List<TableRef> tableRefs;
+ private final String alias;
+
+ public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode table, long timeStamp) throws SQLException {
+ super(connection);
+ List<PColumnFamily> families = Lists.newArrayListWithExpectedSize(table.getDynamicColumns().size());
+ for (ColumnDef def : table.getDynamicColumns()) {
+ if (def.getColumnDefName().getFamilyName() != null) {
+ families.add(new PColumnFamilyImpl(PNameFactory.newName(def.getColumnDefName().getFamilyName()),Collections.<PColumn>emptyList()));
+ }
+ }
+ Long scn = connection.getSCN();
+ PTable theTable = new PTableImpl(table.getName().getSchemaName(), table.getName().getTableName(), scn == null ? HConstants.LATEST_TIMESTAMP : scn, families);
+ theTable = this.addDynamicColumns(table.getDynamicColumns(), theTable);
+ alias = null;
+ tableRefs = ImmutableList.of(new TableRef(alias, theTable, timeStamp, !table.getDynamicColumns().isEmpty()));
+ }
+
+ public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode table, boolean updateCacheOnlyIfAutoCommit, boolean allowMultiTenant) throws SQLException {
+ super(connection);
+ alias = table.getAlias();
+ TableName tableNameNode = table.getName();
+ String schemaName = tableNameNode.getSchemaName();
+ String tableName = tableNameNode.getTableName();
+ SQLException sqlE = null;
+ long timeStamp = QueryConstants.UNSET_TIMESTAMP;
+ TableRef tableRef = null;
+ boolean retry = true;
+ boolean didRetry = false;
+ MetaDataMutationResult result = null;
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+ while (true) {
+ try {
+ if (!updateCacheOnlyIfAutoCommit || connection.getAutoCommit()) {
+ retry = false; // No reason to retry after this
+ result = client.updateCache(schemaName, tableName);
+ timeStamp = result.getMutationTime();
+ }
+ PTable theTable;
+ try {
+ theTable = connection.getPMetaData().getTable(fullTableName);
+ } catch (TableNotFoundException e) {
+ if (allowMultiTenant && result != null && result.getTable() != null) {
+ theTable = result.getTable();
+ } else {
+ throw e;
+ }
+ }
+ // If dynamic columns have been specified add them to the table declaration
+ if (!table.getDynamicColumns().isEmpty()) {
+ theTable = this.addDynamicColumns(table.getDynamicColumns(), theTable);
+ }
+ tableRef = new TableRef(alias, theTable, timeStamp, !table.getDynamicColumns().isEmpty());
+ if (didRetry && logger.isDebugEnabled()) {
+ logger.debug("Re-resolved stale table " + fullTableName + " with seqNum " + tableRef.getTable().getSequenceNumber() + " at timestamp " + tableRef.getTable().getTimeStamp() + " with " + tableRef.getTable().getColumns().size() + " columns: " + tableRef.getTable().getColumns());
+ }
+ break;
+ } catch (TableNotFoundException e) {
+ sqlE = new TableNotFoundException(e,timeStamp);
+ }
+ // If we haven't already tried, update our cache and retry
+ // Only loop back if the cache was updated or we got a result back and we're allowed to multi-tenant tables
+ if (retry && ((result = client.updateCache(schemaName, tableName)).wasUpdated() || (result.getTable() != null && allowMultiTenant))) {
+ timeStamp = result.getMutationTime();
+ retry = false;
+ didRetry = true;
+ continue;
+ }
+ throw sqlE;
+ }
+ tableRefs = ImmutableList.of(tableRef);
+ }
+
+ @Override
+ public List<TableRef> getTables() {
+ return tableRefs;
+ }
+
+ @Override
+ public ColumnRef resolveColumn(String schemaName, String tableName,
+ String colName) throws SQLException {
+ TableRef tableRef = tableRefs.get(0);
+ boolean resolveCF = false;
+ if (schemaName != null || tableName != null) {
+ String resolvedTableName = tableRef.getTable().getTableName().getString();
+ String resolvedSchemaName = tableRef.getTable().getSchemaName().getString();
+ if (schemaName != null && tableName != null) {
+ if ( ! ( schemaName.equals(resolvedSchemaName) &&
+ tableName.equals(resolvedTableName) )) {
+ if (!(resolveCF = schemaName.equals(alias))) {
+ throw new ColumnNotFoundException(schemaName, tableName, null, colName);
+ }
+ }
+ } else { // schemaName == null && tableName != null
+ if (tableName != null && !tableName.equals(alias) && (!tableName.equals(resolvedTableName) || !resolvedSchemaName.equals(""))) {
+ resolveCF = true;
+ }
+ }
+
+ }
+ PColumn column = resolveCF
+ ? tableRef.getTable().getColumnFamily(tableName).getColumn(colName)
+ : tableRef.getTable().getColumn(colName);
+ return new ColumnRef(tableRef, column.getPosition());
+ }
+
+ }
+
+ private static abstract class BaseColumnResolver implements ColumnResolver {
+ protected final PhoenixConnection connection;
+ protected final MetaDataClient client;
+
+ private BaseColumnResolver(PhoenixConnection connection) {
+ this.connection = connection;
+ this.client = new MetaDataClient(connection);
+ }
+
+ protected PTable addDynamicColumns(List<ColumnDef> dynColumns, PTable theTable)
+ throws SQLException {
+ if (!dynColumns.isEmpty()) {
+ List<PColumn> allcolumns = new ArrayList<PColumn>();
+ allcolumns.addAll(theTable.getColumns());
+ int position = allcolumns.size();
+ PName defaultFamilyName = PNameFactory.newName(SchemaUtil.getEmptyColumnFamily(theTable.getColumnFamilies()));
+ for (ColumnDef dynColumn : dynColumns) {
+ PName familyName = defaultFamilyName;
+ PName name = PNameFactory.newName(dynColumn.getColumnDefName().getColumnName());
+ String family = dynColumn.getColumnDefName().getFamilyName();
+ if (family != null) {
+ theTable.getColumnFamily(family); // Verifies that column family exists
+ familyName = PNameFactory.newName(family);
+ }
+ allcolumns.add(new PColumnImpl(name, familyName, dynColumn.getDataType(), dynColumn.getMaxLength(),
+ dynColumn.getScale(), dynColumn.isNull(), position, dynColumn.getColumnModifier(), dynColumn.getArraySize()));
+ position++;
+ }
+ theTable = PTableImpl.makePTable(theTable, allcolumns);
+ }
+ return theTable;
+ }
+ }
+
+ // TODO: unused, but should be used for joins - make private once used
+ public static class MultiTableColumnResolver extends BaseColumnResolver implements TableNodeVisitor {
+ private final ListMultimap<String, TableRef> tableMap;
+ private final List<TableRef> tables;
+
+ private MultiTableColumnResolver(PhoenixConnection connection) {
+ super(connection);
+ tableMap = ArrayListMultimap.<String, TableRef> create();
+ tables = Lists.newArrayList();
+ }
+
+ @Override
+ public List<TableRef> getTables() {
+ return tables;
+ }
+
+ @Override
+ public void visit(BindTableNode boundTableNode) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void visit(JoinTableNode joinNode) throws SQLException {
+ joinNode.getTable().accept(this);
+ }
+
+ private TableRef createTableRef(String alias, String schemaName, String tableName,
+ List<ColumnDef> dynamicColumnDefs) throws SQLException {
+ MetaDataMutationResult result = client.updateCache(schemaName, tableName);
+ long timeStamp = result.getMutationTime();
+ PTable theTable = connection.getPMetaData().getTable(SchemaUtil.getTableName(schemaName, tableName));
+
+ // If dynamic columns have been specified add them to the table declaration
+ if (!dynamicColumnDefs.isEmpty()) {
+ theTable = this.addDynamicColumns(dynamicColumnDefs, theTable);
+ }
+ TableRef tableRef = new TableRef(alias, theTable, timeStamp, !dynamicColumnDefs.isEmpty());
+ return tableRef;
+ }
+
+
+ @Override
+ public void visit(NamedTableNode namedTableNode) throws SQLException {
+ String tableName = namedTableNode.getName().getTableName();
+ String schemaName = namedTableNode.getName().getSchemaName();
+
+ String alias = namedTableNode.getAlias();
+ List<ColumnDef> dynamicColumnDefs = namedTableNode.getDynamicColumns();
+
+ TableRef tableRef = createTableRef(alias, schemaName, tableName, dynamicColumnDefs);
+ PTable theTable = tableRef.getTable();
+
+ if (alias != null) {
+ tableMap.put(alias, tableRef);
+ }
+
+ String name = theTable.getName().getString();
+ //avoid having one name mapped to two identical TableRef.
+ if (alias == null || !alias.equals(name)) {
+ tableMap.put(name, tableRef);
+ }
+ tables.add(tableRef);
+ }
+
+ @Override
+ public void visit(DerivedTableNode subselectNode) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ private static class ColumnFamilyRef {
+ private final TableRef tableRef;
+ private final PColumnFamily family;
+
+ ColumnFamilyRef(TableRef tableRef, PColumnFamily family) {
+ this.tableRef = tableRef;
+ this.family = family;
+ }
+
+ public TableRef getTableRef() {
+ return tableRef;
+ }
+
+ public PColumnFamily getFamily() {
+ return family;
+ }
+ }
+
+ private TableRef resolveTable(String schemaName, String tableName) throws SQLException {
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+ List<TableRef> tableRefs = tableMap.get(fullTableName);
+ if (tableRefs.size() == 0) {
+ throw new TableNotFoundException(fullTableName);
+ } else if (tableRefs.size() > 1) {
+ throw new AmbiguousTableException(tableName);
+ } else {
+ return tableRefs.get(0);
+ }
+ }
+
+ private ColumnFamilyRef resolveColumnFamily(String tableName, String cfName) throws SQLException {
+ if (tableName == null) {
+ ColumnFamilyRef theColumnFamilyRef = null;
+ Iterator<TableRef> iterator = tables.iterator();
+ while (iterator.hasNext()) {
+ TableRef tableRef = iterator.next();
+ try {
+ PColumnFamily columnFamily = tableRef.getTable().getColumnFamily(cfName);
+ if (theColumnFamilyRef != null) { throw new TableNotFoundException(cfName); }
+ theColumnFamilyRef = new ColumnFamilyRef(tableRef, columnFamily);
+ } catch (ColumnFamilyNotFoundException e) {}
+ }
+ if (theColumnFamilyRef != null) { return theColumnFamilyRef; }
+ throw new TableNotFoundException(cfName);
+ } else {
+ TableRef tableRef = resolveTable(null, tableName);
+ PColumnFamily columnFamily = tableRef.getTable().getColumnFamily(cfName);
+ return new ColumnFamilyRef(tableRef, columnFamily);
+ }
+ }
+
+ @Override
+ public ColumnRef resolveColumn(String schemaName, String tableName, String colName) throws SQLException {
+ if (tableName == null) {
+ int theColumnPosition = -1;
+ TableRef theTableRef = null;
+ Iterator<TableRef> iterator = tables.iterator();
+ while (iterator.hasNext()) {
+ TableRef tableRef = iterator.next();
+ try {
+ PColumn column = tableRef.getTable().getColumn(colName);
+ if (theTableRef != null) { throw new AmbiguousColumnException(colName); }
+ theTableRef = tableRef;
+ theColumnPosition = column.getPosition();
+ } catch (ColumnNotFoundException e) {
+
+ }
+ }
+ if (theTableRef != null) { return new ColumnRef(theTableRef, theColumnPosition); }
+ throw new ColumnNotFoundException(colName);
+ } else {
+ try {
+ TableRef tableRef = resolveTable(schemaName, tableName);
+ PColumn column = tableRef.getTable().getColumn(colName);
+ return new ColumnRef(tableRef, column.getPosition());
+ } catch (TableNotFoundException e) {
+ // Try using the tableName as a columnFamily reference instead
+ ColumnFamilyRef cfRef = resolveColumnFamily(schemaName, tableName);
+ PColumn column = cfRef.getFamily().getColumn(colName);
+ return new ColumnRef(cfRef.getTableRef(), column.getPosition());
+ }
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java
new file mode 100644
index 0000000..e9b08f8
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.compile;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.http.annotation.Immutable;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.phoenix.compile.TrackOrderPreservingExpressionCompiler.Entry;
+import org.apache.phoenix.compile.TrackOrderPreservingExpressionCompiler.Ordering;
+import org.apache.phoenix.coprocessor.GroupedAggregateRegionObserver;
+import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.expression.CoerceExpression;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.parse.AliasedNode;
+import org.apache.phoenix.parse.ParseNode;
+import org.apache.phoenix.parse.SelectStatement;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.PDataType;
+
+
+/**
+ *
+ * Validates GROUP BY clause and builds a {@link GroupBy} instance to encapsulate the
+ * group by expressions.
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+public class GroupByCompiler {
+ @Immutable
+ public static class GroupBy {
+ private final List<Expression> expressions;
+ private final List<Expression> keyExpressions;
+ private final String scanAttribName;
+ public static final GroupByCompiler.GroupBy EMPTY_GROUP_BY = new GroupBy(new GroupByBuilder());
+
+ private GroupBy(GroupByBuilder builder) {
+ this.expressions = ImmutableList.copyOf(builder.expressions);
+ this.keyExpressions = ImmutableList.copyOf(builder.keyExpressions);
+ this.scanAttribName = builder.scanAttribName;
+ assert(expressions.size() == keyExpressions.size());
+ }
+
+ public List<Expression> getExpressions() {
+ return expressions;
+ }
+
+ public List<Expression> getKeyExpressions() {
+ return keyExpressions;
+ }
+
+ public String getScanAttribName() {
+ return scanAttribName;
+ }
+
+ public boolean isEmpty() {
+ return expressions.isEmpty();
+ }
+
+ public static class GroupByBuilder {
+ private String scanAttribName;
+ private List<Expression> expressions = Collections.emptyList();
+ private List<Expression> keyExpressions = Collections.emptyList();
+
+ public GroupByBuilder() {
+ }
+
+ public GroupByBuilder setScanAttribName(String scanAttribName) {
+ this.scanAttribName = scanAttribName;
+ return this;
+ }
+
+ public GroupByBuilder setExpressions(List<Expression> expressions) {
+ this.expressions = expressions;
+ return this;
+ }
+
+ public GroupByBuilder setKeyExpressions(List<Expression> keyExpressions) {
+ this.keyExpressions = keyExpressions;
+ return this;
+ }
+
+ public GroupBy build() {
+ return new GroupBy(this);
+ }
+ }
+
+ public boolean isOrderPreserving() {
+ return !GroupedAggregateRegionObserver.UNORDERED_GROUP_BY_EXPRESSIONS.equals(scanAttribName);
+ }
+
+ public void explain(List<String> planSteps) {
+ if (scanAttribName != null) {
+ if (UngroupedAggregateRegionObserver.UNGROUPED_AGG.equals(scanAttribName)) {
+ planSteps.add(" SERVER AGGREGATE INTO SINGLE ROW");
+ } else if (GroupedAggregateRegionObserver.UNORDERED_GROUP_BY_EXPRESSIONS.equals(scanAttribName)) {
+ planSteps.add(" SERVER AGGREGATE INTO DISTINCT ROWS BY " + getExpressions());
+ } else {
+ planSteps.add(" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY " + getExpressions());
+ }
+ }
+ }
+ }
+
+ /**
+ * Get list of columns in the GROUP BY clause.
+ * @param context query context kept between compilation of different query clauses
+ * @param statement SQL statement being compiled
+ * @return the {@link GroupBy} instance encapsulating the group by clause
+ * @throws ColumnNotFoundException if column name could not be resolved
+ * @throws AmbiguousColumnException if an unaliased column name is ambiguous across multiple tables
+ */
+ public static GroupBy compile(StatementContext context, SelectStatement statement) throws SQLException {
+ List<ParseNode> groupByNodes = statement.getGroupBy();
+ /**
+ * Distinct can use an aggregate plan if there's no group by.
+ * Otherwise, we need to insert a step after the Merge that dedups.
+ * Order by only allowed on columns in the select distinct
+ */
+ if (groupByNodes.isEmpty()) {
+ if (statement.isAggregate()) {
+ return new GroupBy.GroupByBuilder().setScanAttribName(UngroupedAggregateRegionObserver.UNGROUPED_AGG).build();
+ }
+ if (!statement.isDistinct()) {
+ return GroupBy.EMPTY_GROUP_BY;
+ }
+
+ groupByNodes = Lists.newArrayListWithExpectedSize(statement.getSelect().size());
+ for (AliasedNode aliasedNode : statement.getSelect()) {
+ groupByNodes.add(aliasedNode.getNode());
+ }
+ }
+
+ // Accumulate expressions in GROUP BY
+ TrackOrderPreservingExpressionCompiler groupByVisitor =
+ new TrackOrderPreservingExpressionCompiler(context,
+ GroupBy.EMPTY_GROUP_BY, groupByNodes.size(),
+ Ordering.UNORDERED);
+ for (ParseNode node : groupByNodes) {
+ Expression expression = node.accept(groupByVisitor);
+ if (groupByVisitor.isAggregate()) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.AGGREGATE_IN_GROUP_BY)
+ .setMessage(expression.toString()).build().buildException();
+ }
+ if (!expression.isStateless()) {
+ groupByVisitor.addEntry(expression);
+ }
+ groupByVisitor.reset();
+ }
+
+ List<Entry> groupByEntries = groupByVisitor.getEntries();
+ if (groupByEntries.isEmpty()) {
+ return GroupBy.EMPTY_GROUP_BY;
+ }
+
+ boolean isRowKeyOrderedGrouping = groupByVisitor.isOrderPreserving();
+ List<Expression> expressions = Lists.newArrayListWithCapacity(groupByEntries.size());
+ List<Expression> keyExpressions = expressions;
+ String groupExprAttribName;
+ // This is true if the GROUP BY is composed of only PK columns. We further check here that
+ // there are no "gaps" in the PK columns positions used (i.e. we start with the first PK
+ // column and use each subsequent one in PK order).
+ if (isRowKeyOrderedGrouping) {
+ groupExprAttribName = GroupedAggregateRegionObserver.KEY_ORDERED_GROUP_BY_EXPRESSIONS;
+ for (Entry groupByEntry : groupByEntries) {
+ expressions.add(groupByEntry.getExpression());
+ }
+ } else {
+ /*
+ * Otherwise, our coprocessor needs to collect all distinct groups within a region, sort them, and
+ * hold on to them until the scan completes.
+ */
+ groupExprAttribName = GroupedAggregateRegionObserver.UNORDERED_GROUP_BY_EXPRESSIONS;
+ /*
+ * Put fixed length nullables at the end, so that we can represent null by the absence of the trailing
+ * value in the group by key. If there is more than one, we'll need to convert the ones not at the end
+ * into a Decimal so that we can use an empty byte array as our representation for null (which correctly
+ * maintains the sort order). We convert the Decimal back to the appropriate type (Integer or Long) when
+ * it's retrieved from the result set.
+ *
+ * More specifically, order into the following buckets:
+ * 1) non nullable fixed width
+ * 2) variable width
+ * 3) nullable fixed width
+ * Within each bucket, order based on the column position in the schema. Putting the fixed width values
+ * in the beginning optimizes access to subsequent values.
+ */
+ Collections.sort(groupByEntries, new Comparator<Entry>() {
+ @Override
+ public int compare(Entry o1, Entry o2) {
+ Expression e1 = o1.getExpression();
+ Expression e2 = o2.getExpression();
+ boolean isFixed1 = e1.getDataType().isFixedWidth();
+ boolean isFixed2 = e2.getDataType().isFixedWidth();
+ boolean isFixedNullable1 = e1.isNullable() &&isFixed1;
+ boolean isFixedNullable2 = e2.isNullable() && isFixed2;
+ if (isFixedNullable1 == isFixedNullable2) {
+ if (isFixed1 == isFixed2) {
+ // Not strictly necessary, but forces the order to match the schema
+ // column order (with PK columns before value columns).
+ return o1.getColumnPosition() - o2.getColumnPosition();
+ } else if (isFixed1) {
+ return -1;
+ } else {
+ return 1;
+ }
+ } else if (isFixedNullable1) {
+ return 1;
+ } else {
+ return -1;
+ }
+ }
+ });
+ for (Entry groupByEntry : groupByEntries) {
+ expressions.add(groupByEntry.getExpression());
+ }
+ for (int i = expressions.size()-2; i >= 0; i--) {
+ Expression expression = expressions.get(i);
+ PDataType keyType = getKeyType(expression);
+ if (keyType == expression.getDataType()) {
+ continue;
+ }
+ // Copy expressions only when keyExpressions will be different than expressions
+ if (keyExpressions == expressions) {
+ keyExpressions = new ArrayList<Expression>(expressions);
+ }
+ // Wrap expression in an expression that coerces the expression to the required type..
+ // This is done so that we have a way of expressing null as an empty key when more
+ // than one fixed and nullable types are used in a group by clause
+ keyExpressions.set(i, CoerceExpression.create(expression, keyType));
+ }
+ }
+
+ // Set attribute with serialized expressions for coprocessor
+ // FIXME: what if group by is empty (i.e. only literals)?
+ GroupedAggregateRegionObserver.serializeIntoScan(context.getScan(), groupExprAttribName, keyExpressions);
+ GroupBy groupBy = new GroupBy.GroupByBuilder().setScanAttribName(groupExprAttribName).setExpressions(expressions).setKeyExpressions(keyExpressions).build();
+ return groupBy;
+ }
+
+ private static PDataType getKeyType(Expression expression) {
+ PDataType type = expression.getDataType();
+ if (!expression.isNullable() || !type.isFixedWidth()) {
+ return type;
+ }
+ if (type.isCoercibleTo(PDataType.DECIMAL)) {
+ return PDataType.DECIMAL;
+ }
+ // Should never happen
+ throw new IllegalStateException("Multiple occurrences of type " + type + " may not occur in a GROUP BY clause");
+ }
+
+ private GroupByCompiler() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/compile/HavingCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/HavingCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/HavingCompiler.java
new file mode 100644
index 0000000..857d48a
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/HavingCompiler.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.compile;
+
+import java.sql.SQLException;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.phoenix.compile.GroupByCompiler.GroupBy;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.expression.LiteralExpression;
+import org.apache.phoenix.parse.AddParseNode;
+import org.apache.phoenix.parse.AndParseNode;
+import org.apache.phoenix.parse.BetweenParseNode;
+import org.apache.phoenix.parse.CaseParseNode;
+import org.apache.phoenix.parse.ColumnParseNode;
+import org.apache.phoenix.parse.ComparisonParseNode;
+import org.apache.phoenix.parse.DivideParseNode;
+import org.apache.phoenix.parse.FunctionParseNode;
+import org.apache.phoenix.parse.IsNullParseNode;
+import org.apache.phoenix.parse.MultiplyParseNode;
+import org.apache.phoenix.parse.OrParseNode;
+import org.apache.phoenix.parse.ParseNode;
+import org.apache.phoenix.parse.SelectStatement;
+import org.apache.phoenix.parse.SelectStatementRewriter;
+import org.apache.phoenix.parse.SubtractParseNode;
+import org.apache.phoenix.parse.TraverseNoParseNodeVisitor;
+import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.PDataType;
+import org.apache.phoenix.schema.TypeMismatchException;
+
+
+public class HavingCompiler {
+
+ private HavingCompiler() {
+ }
+
+ public static Expression compile(StatementContext context, SelectStatement statement, GroupBy groupBy) throws SQLException {
+ ParseNode having = statement.getHaving();
+ if (having == null) {
+ return null;
+ }
+ ExpressionCompiler expressionBuilder = new ExpressionCompiler(context, groupBy);
+ Expression expression = having.accept(expressionBuilder);
+ if (expression.getDataType() != PDataType.BOOLEAN) {
+ throw TypeMismatchException.newException(PDataType.BOOLEAN, expression.getDataType(), expression.toString());
+ }
+ if (LiteralExpression.isFalse(expression)) {
+ context.setScanRanges(ScanRanges.NOTHING);
+ return null;
+ } else if (LiteralExpression.isTrue(expression)) {
+ return null;
+ }
+ if (!expressionBuilder.isAggregate()) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_AGGREGATE_IN_HAVING_CLAUSE).build().buildException();
+ }
+ return expression;
+ }
+
+ public static SelectStatement rewrite(StatementContext context, SelectStatement statement, GroupBy groupBy) throws SQLException {
+ ParseNode having = statement.getHaving();
+ if (having == null) {
+ return statement;
+ }
+ HavingClauseVisitor visitor = new HavingClauseVisitor(context, groupBy);
+ having.accept(visitor);
+ statement = SelectStatementRewriter.moveFromHavingToWhereClause(statement, visitor.getMoveToWhereClauseExpressions());
+ return statement;
+ }
+
+ /**
+ *
+ * Visitor that figures out if an expression can be moved from the HAVING clause to
+ * the WHERE clause, since it's more optimal to pre-filter instead of post-filter.
+ *
+ * The visitor traverses through AND expressions only and into comparison expresssions.
+ * If a comparison expression uses a GROUP BY column and does not use any aggregate
+ * functions, then it's moved. For example, these HAVING expressions would be moved:
+ *
+ * select count(1) from atable group by a_string having a_string = 'foo'
+ * select count(1) from atable group by a_date having round(a_date,'hour') > ?
+ * select count(1) from atable group by a_date,a_string having a_date > ? or a_string = 'a'
+ * select count(1) from atable group by a_string,b_string having a_string = 'a' and b_string = 'b'
+ *
+ * while these would not be moved:
+ *
+ * select count(1) from atable having min(a_integer) < 5
+ * select count(1) from atable group by a_string having count(a_string) >= 1
+ * select count(1) from atable group by a_date,a_string having a_date > ? or min(a_string) = 'a'
+ * select count(1) from atable group by a_date having round(min(a_date),'hour') < ?
+ *
+ * @author jtaylor
+ * @since 0.1
+ */
+ private static class HavingClauseVisitor extends TraverseNoParseNodeVisitor<Void> {
+ private ParseNode topNode = null;
+ private boolean hasNoAggregateFunctions = true;
+ private Boolean hasOnlyAggregateColumns;
+ private final StatementContext context;
+ private final GroupBy groupBy;
+ private final Set<ParseNode> moveToWhereClause = new LinkedHashSet<ParseNode>();
+
+ HavingClauseVisitor(StatementContext context, GroupBy groupBy) {
+ this.context = context;
+ this.groupBy = groupBy;
+ }
+
+ public Set<ParseNode> getMoveToWhereClauseExpressions() {
+ return moveToWhereClause;
+ }
+
+ @Override
+ public boolean visitEnter(AndParseNode node) throws SQLException {
+ return true;
+ }
+
+ @Override
+ public boolean visitEnter(OrParseNode node) throws SQLException {
+ enterBooleanNode(node);
+ return true;
+ }
+
+ @Override
+ public boolean visitEnter(ComparisonParseNode node) throws SQLException {
+ enterBooleanNode(node);
+ return true;
+ }
+
+ @Override
+ public boolean visitEnter(IsNullParseNode node) throws SQLException {
+ enterBooleanNode(node);
+ return true;
+ }
+
+ private void enterBooleanNode(ParseNode node) {
+ if (topNode == null) {
+ topNode = node;
+ }
+ }
+
+ private void leaveBooleanNode(ParseNode node) {
+ if (topNode == node) {
+ if ( hasNoAggregateFunctions && !Boolean.FALSE.equals(hasOnlyAggregateColumns)) {
+ moveToWhereClause.add(node);
+ }
+ hasNoAggregateFunctions = true;
+ hasOnlyAggregateColumns = null;
+ topNode = null;
+ }
+ }
+
+ @Override
+ public Void visitLeave(OrParseNode node, List<Void> l) throws SQLException {
+ leaveBooleanNode(node);
+ return null;
+ }
+
+ @Override
+ public Void visitLeave(ComparisonParseNode node, List<Void> l) throws SQLException {
+ leaveBooleanNode(node);
+ return null;
+ }
+
+ @Override
+ public Void visitLeave(IsNullParseNode node, List<Void> l) throws SQLException {
+ leaveBooleanNode(node);
+ return null;
+ }
+
+ @Override
+ public boolean visitEnter(FunctionParseNode node) throws SQLException {
+ boolean isAggregate = node.isAggregate();
+ this.hasNoAggregateFunctions = this.hasNoAggregateFunctions && !isAggregate;
+ return !isAggregate;
+ }
+
+ @Override
+ public boolean visitEnter(CaseParseNode node) throws SQLException {
+ return true;
+ }
+
+ @Override
+ public Void visit(ColumnParseNode node) throws SQLException {
+ ColumnRef ref = context.getResolver().resolveColumn(node.getSchemaName(), node.getTableName(), node.getName());
+ boolean isAggregateColumn = groupBy.getExpressions().indexOf(ref.newColumnExpression()) >= 0;
+ if (hasOnlyAggregateColumns == null) {
+ hasOnlyAggregateColumns = isAggregateColumn;
+ } else {
+ hasOnlyAggregateColumns &= isAggregateColumn;
+ }
+
+ return null;
+ }
+
+ @Override
+ public boolean visitEnter(SubtractParseNode node) throws SQLException {
+ return true;
+ }
+
+ @Override
+ public boolean visitEnter(AddParseNode node) throws SQLException {
+ return true;
+ }
+
+ @Override
+ public boolean visitEnter(MultiplyParseNode node) throws SQLException {
+ return true;
+ }
+
+ @Override
+ public boolean visitEnter(DivideParseNode node) throws SQLException {
+ return true;
+ }
+
+ @Override
+ public boolean visitEnter(BetweenParseNode node) throws SQLException {
+ return true;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/50d523f6/phoenix-core/src/main/java/org/apache/phoenix/compile/IndexStatementRewriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/IndexStatementRewriter.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/IndexStatementRewriter.java
new file mode 100644
index 0000000..84ac0a1
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/IndexStatementRewriter.java
@@ -0,0 +1,103 @@
+package org.apache.phoenix.compile;
+
+import java.sql.SQLException;
+import java.util.Map;
+
+import org.apache.phoenix.parse.ColumnParseNode;
+import org.apache.phoenix.parse.FamilyWildcardParseNode;
+import org.apache.phoenix.parse.ParseNode;
+import org.apache.phoenix.parse.ParseNodeFactory;
+import org.apache.phoenix.parse.ParseNodeRewriter;
+import org.apache.phoenix.parse.SelectStatement;
+import org.apache.phoenix.parse.TableName;
+import org.apache.phoenix.parse.WildcardParseNode;
+import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.PDataType;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.util.IndexUtil;
+
+public class IndexStatementRewriter extends ParseNodeRewriter {
+ private static final ParseNodeFactory FACTORY = new ParseNodeFactory();
+
+ private Map<TableRef, TableRef> multiTableRewriteMap;
+
+ public IndexStatementRewriter(ColumnResolver dataResolver, Map<TableRef, TableRef> multiTableRewriteMap) {
+ super(dataResolver);
+ this.multiTableRewriteMap = multiTableRewriteMap;
+ }
+
+ /**
+ * Rewrite the select statement by translating all data table column references to
+ * references to the corresponding index column.
+ * @param statement the select statement
+ * @param dataResolver the column resolver
+ * @return new select statement or the same one if nothing was rewritten.
+ * @throws SQLException
+ */
+ public static SelectStatement translate(SelectStatement statement, ColumnResolver dataResolver) throws SQLException {
+ return translate(statement, dataResolver, null);
+ }
+
+ /**
+ * Rewrite the select statement containing multiple tables by translating all
+ * data table column references to references to the corresponding index column.
+ * @param statement the select statement
+ * @param dataResolver the column resolver
+ * @param multiTableRewriteMap the data table to index table map
+ * @return new select statement or the same one if nothing was rewritten.
+ * @throws SQLException
+ */
+ public static SelectStatement translate(SelectStatement statement, ColumnResolver dataResolver, Map<TableRef, TableRef> multiTableRewriteMap) throws SQLException {
+ return rewrite(statement, new IndexStatementRewriter(dataResolver, multiTableRewriteMap));
+ }
+
+ @Override
+ public ParseNode visit(ColumnParseNode node) throws SQLException {
+ ColumnRef dataColRef = getResolver().resolveColumn(node.getSchemaName(), node.getTableName(), node.getName());
+ TableName tName = null;
+ if (multiTableRewriteMap != null) {
+ TableRef origRef = dataColRef.getTableRef();
+ TableRef tableRef = multiTableRewriteMap.get(origRef);
+ if (tableRef == null)
+ return node;
+
+ if (origRef.getTableAlias() != null) {
+ tName = FACTORY.table(null, origRef.getTableAlias());
+ } else {
+ String schemaName = tableRef.getTable().getSchemaName().getString();
+ schemaName = schemaName.length() == 0 ? null : '"' + schemaName + '"';
+ String tableName = '"' + tableRef.getTable().getTableName().getString() + '"';
+ tName = FACTORY.table(schemaName, tableName);
+ }
+ }
+ String indexColName = IndexUtil.getIndexColumnName(dataColRef.getColumn());
+ // Same alias as before, but use the index column name instead of the data column name
+ ParseNode indexColNode = new ColumnParseNode(tName, indexColName, node.getAlias());
+ PDataType indexColType = IndexUtil.getIndexColumnDataType(dataColRef.getColumn());
+ PDataType dataColType = dataColRef.getColumn().getDataType();
+
+ // Coerce index column reference back to same type as data column so that
+ // expression behave exactly the same. No need to invert, as this will be done
+ // automatically as needed. If node is used at the top level, do not convert, as
+ // otherwise the wrapper gets in the way in the group by clause. For example,
+ // an INTEGER column in a GROUP BY gets doubly wrapped like this:
+ // CAST CAST int_col AS INTEGER AS DECIMAL
+ // This is unnecessary and problematic in the case of a null value.
+ // TODO: test case for this
+ if (!isTopLevel() && indexColType != dataColType) {
+ indexColNode = FACTORY.cast(indexColNode, dataColType);
+ }
+ return indexColNode;
+ }
+
+ @Override
+ public ParseNode visit(WildcardParseNode node) throws SQLException {
+ return WildcardParseNode.REWRITE_INSTANCE;
+ }
+
+ @Override
+ public ParseNode visit(FamilyWildcardParseNode node) throws SQLException {
+ return new FamilyWildcardParseNode(node, true);
+ }
+
+}