You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2016/12/16 13:58:31 UTC
[5/6] hive git commit: HIVE-14496: Enable Calcite rewriting with
materialized views (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f7b2ed7..d0a66b0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -18,25 +18,44 @@
package org.apache.hadoop.hive.metastore;
-import org.apache.hadoop.hive.metastore.api.ClientCapabilities;
-import org.apache.hadoop.hive.metastore.api.ClientCapability;
-import org.apache.hadoop.hive.metastore.api.CompactionResponse;
-import org.apache.hadoop.hive.metastore.api.GetTableRequest;
-import org.apache.hadoop.hive.metastore.api.GetTableResult;
-import org.apache.hadoop.hive.metastore.api.GetTablesRequest;
-import org.apache.hadoop.hive.metastore.api.GetTablesResult;
-import org.apache.hadoop.hive.metastore.api.MetaException;
+import static org.apache.commons.lang.StringUtils.join;
+import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_COMMENT;
+import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
+import static org.apache.hadoop.hive.metastore.MetaStoreUtils.validateName;
-import com.facebook.fb303.FacebookBase;
-import com.facebook.fb303.fb_status;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimaps;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Formatter;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Timer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+import javax.jdo.JDOException;
import org.apache.commons.cli.OptionBuilder;
import org.apache.hadoop.conf.Configuration;
@@ -45,9 +64,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.JvmPauseMonitor;
import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
-import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.common.cli.CommonCliOptions;
@@ -57,119 +76,7 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
-import org.apache.hadoop.hive.metastore.api.AbortTxnsRequest;
-import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
-import org.apache.hadoop.hive.metastore.api.AddForeignKeyRequest;
-import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
-import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
-import org.apache.hadoop.hive.metastore.api.AddPrimaryKeyRequest;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.CacheFileMetadataRequest;
-import org.apache.hadoop.hive.metastore.api.CacheFileMetadataResult;
-import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
-import org.apache.hadoop.hive.metastore.api.ClearFileMetadataRequest;
-import org.apache.hadoop.hive.metastore.api.ClearFileMetadataResult;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
-import org.apache.hadoop.hive.metastore.api.CompactionRequest;
-import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.DropConstraintRequest;
-import org.apache.hadoop.hive.metastore.api.DropPartitionsExpr;
-import org.apache.hadoop.hive.metastore.api.DropPartitionsRequest;
-import org.apache.hadoop.hive.metastore.api.DropPartitionsResult;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.FireEventRequest;
-import org.apache.hadoop.hive.metastore.api.FireEventResponse;
-import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest;
-import org.apache.hadoop.hive.metastore.api.ForeignKeysResponse;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse;
-import org.apache.hadoop.hive.metastore.api.GetFileMetadataByExprRequest;
-import org.apache.hadoop.hive.metastore.api.GetFileMetadataByExprResult;
-import org.apache.hadoop.hive.metastore.api.GetFileMetadataRequest;
-import org.apache.hadoop.hive.metastore.api.GetFileMetadataResult;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest;
-import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleResponse;
-import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest;
-import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalResponse;
-import org.apache.hadoop.hive.metastore.api.GrantRevokePrivilegeRequest;
-import org.apache.hadoop.hive.metastore.api.GrantRevokePrivilegeResponse;
-import org.apache.hadoop.hive.metastore.api.GrantRevokeRoleRequest;
-import org.apache.hadoop.hive.metastore.api.GrantRevokeRoleResponse;
-import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
-import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
-import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.LockRequest;
-import org.apache.hadoop.hive.metastore.api.LockResponse;
-import org.apache.hadoop.hive.metastore.api.MetadataPpdResult;
-import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
-import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD;
-import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD;
-import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest;
-import org.apache.hadoop.hive.metastore.api.PartitionsByExprResult;
-import org.apache.hadoop.hive.metastore.api.PartitionsStatsRequest;
-import org.apache.hadoop.hive.metastore.api.PartitionsStatsResult;
-import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
-import org.apache.hadoop.hive.metastore.api.PrimaryKeysResponse;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.PutFileMetadataRequest;
-import org.apache.hadoop.hive.metastore.api.PutFileMetadataResult;
-import org.apache.hadoop.hive.metastore.api.RequestPartsSpec;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
-import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
-import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
-import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.TableStatsRequest;
-import org.apache.hadoop.hive.metastore.api.TableStatsResult;
-import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
-import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
-import org.apache.hadoop.hive.metastore.api.TxnOpenException;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.api.UnlockRequest;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.events.AddIndexEvent;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterIndexEvent;
@@ -238,43 +145,16 @@ import org.apache.thrift.transport.TTransportFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.jdo.JDOException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Formatter;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.Timer;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.regex.Pattern;
-
-import static org.apache.commons.lang.StringUtils.join;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_COMMENT;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.validateName;
+import com.facebook.fb303.FacebookBase;
+import com.facebook.fb303.fb_status;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimaps;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* TODO:pc remove application logic to a separate interface.
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 4774899..9eec56a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -18,9 +18,36 @@
package org.apache.hadoop.hive.metastore;
-import com.google.common.collect.Lists;
+import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
+import static org.apache.hadoop.hive.metastore.MetaStoreUtils.isIndexTable;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.security.auth.login.LoginException;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
@@ -30,7 +57,6 @@ import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.conf.HiveConfUtil;
-import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
@@ -45,42 +71,14 @@ import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSSLTransportFactory;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.security.auth.login.LoginException;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.NoSuchElementException;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.isIndexTable;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
/**
* Hive Metastore Client.
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 5ea000a..8ba7352 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -19,6 +19,12 @@
package org.apache.hadoop.hive.metastore;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
@@ -90,12 +96,6 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.thrift.TException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
/**
* Wrapper around hive metastore thrift api
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index d4024d2..1aa3e1a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -56,7 +56,6 @@ import javax.jdo.Transaction;
import javax.jdo.datastore.DataStoreCache;
import javax.jdo.identity.IntIdentity;
-import com.google.common.collect.Maps;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -165,6 +164,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* This class is the interface between the application logic and the database
@@ -1489,11 +1489,13 @@ public class ObjectStore implements RawStore, Configurable {
tableType = TableType.MANAGED_TABLE.toString();
}
}
- return new Table(mtbl.getTableName(), mtbl.getDatabase().getName(), mtbl
- .getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl
- .getRetention(), convertToStorageDescriptor(mtbl.getSd()),
- convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()),
- mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType);
+ final Table table = new Table(mtbl.getTableName(), mtbl.getDatabase().getName(), mtbl
+ .getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl
+ .getRetention(), convertToStorageDescriptor(mtbl.getSd()),
+ convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()),
+ mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType);
+ table.setRewriteEnabled(mtbl.isRewriteEnabled());
+ return table;
}
private MTable convertToMTable(Table tbl) throws InvalidObjectException,
@@ -1530,7 +1532,7 @@ public class ObjectStore implements RawStore, Configurable {
convertToMStorageDescriptor(tbl.getSd()), tbl.getOwner(), tbl
.getCreateTime(), tbl.getLastAccessTime(), tbl.getRetention(),
convertToMFieldSchemas(tbl.getPartitionKeys()), tbl.getParameters(),
- tbl.getViewOriginalText(), tbl.getViewExpandedText(),
+ tbl.getViewOriginalText(), tbl.getViewExpandedText(), tbl.isRewriteEnabled(),
tableType);
}
@@ -3297,6 +3299,7 @@ public class ObjectStore implements RawStore, Configurable {
oldt.setLastAccessTime(newt.getLastAccessTime());
oldt.setViewOriginalText(newt.getViewOriginalText());
oldt.setViewExpandedText(newt.getViewExpandedText());
+ oldt.setRewriteEnabled(newt.isRewriteEnabled());
// commit the changes
success = commitTransaction();
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
index 4546d43..94087b1 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
@@ -1062,6 +1062,7 @@ public class HBaseUtils {
if (table.getViewExpandedText() != null) {
builder.setViewExpandedText(table.getViewExpandedText());
}
+ builder.setIsRewriteEnabled(table.isRewriteEnabled());
if (table.getTableType() != null) builder.setTableType(table.getTableType());
if (table.getPrivileges() != null) {
builder.setPrivileges(buildPrincipalPrivilegeSet(table.getPrivileges()));
@@ -1115,6 +1116,7 @@ public class HBaseUtils {
table.setParameters(buildParameters(proto.getParameters()));
if (proto.hasViewOriginalText()) table.setViewOriginalText(proto.getViewOriginalText());
if (proto.hasViewExpandedText()) table.setViewExpandedText(proto.getViewExpandedText());
+ table.setRewriteEnabled(proto.getIsRewriteEnabled());
table.setTableType(proto.getTableType());
if (proto.hasPrivileges()) {
table.setPrivileges(buildPrincipalPrivilegeSet(proto.getPrivileges()));
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTable.java
----------------------------------------------------------------------
diff --git a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTable.java b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTable.java
index 2a78ce9..6cc7157 100644
--- a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTable.java
+++ b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTable.java
@@ -34,6 +34,7 @@ public class MTable {
private Map<String, String> parameters;
private String viewOriginalText;
private String viewExpandedText;
+ private boolean rewriteEnabled;
private String tableType;
public MTable() {}
@@ -54,8 +55,8 @@ public class MTable {
*/
public MTable(String tableName, MDatabase database, MStorageDescriptor sd, String owner,
int createTime, int lastAccessTime, int retention, List<MFieldSchema> partitionKeys,
- Map<String, String> parameters,
- String viewOriginalText, String viewExpandedText, String tableType) {
+ Map<String, String> parameters, String viewOriginalText, String viewExpandedText,
+ boolean rewriteEnabled, String tableType) {
this.tableName = tableName;
this.database = database;
this.sd = sd;
@@ -67,6 +68,7 @@ public class MTable {
this.parameters = parameters;
this.viewOriginalText = viewOriginalText;
this.viewExpandedText = viewExpandedText;
+ this.rewriteEnabled = rewriteEnabled;
this.tableType = tableType;
}
@@ -155,6 +157,20 @@ public class MTable {
}
/**
+ * @return whether the view can be used for rewriting queries
+ */
+ public boolean isRewriteEnabled() {
+ return rewriteEnabled;
+ }
+
+ /**
+ * @param rewriteEnabled whether the view can be used for rewriting queries
+ */
+ public void setRewriteEnabled(boolean rewriteEnabled) {
+ this.rewriteEnabled = rewriteEnabled;
+ }
+
+ /**
* @return the owner
*/
public String getOwner() {
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/model/package.jdo
----------------------------------------------------------------------
diff --git a/metastore/src/model/package.jdo b/metastore/src/model/package.jdo
index bfd6ddd..daee72c 100644
--- a/metastore/src/model/package.jdo
+++ b/metastore/src/model/package.jdo
@@ -179,6 +179,9 @@
<field name="viewExpandedText" default-fetch-group="false">
<column name="VIEW_EXPANDED_TEXT" jdbc-type="LONGVARCHAR"/>
</field>
+ <field name="rewriteEnabled">
+ <column name="IS_REWRITE_ENABLED"/>
+ </field>
<field name="tableType">
<column name="TBL_TYPE" length="128" jdbc-type="VARCHAR"/>
</field>
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
----------------------------------------------------------------------
diff --git a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto b/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
index 3f9e4c5..6499ac6 100644
--- a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
+++ b/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
@@ -255,6 +255,7 @@ message Table {
optional string table_type = 12;
optional PrincipalPrivilegeSet privileges = 13;
optional bool is_temporary = 14;
+ optional bool is_rewrite_enabled = 15;
}
message Index {
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index aef1149..1f87eeb 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -149,14 +149,14 @@ public class TestObjectStore {
StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
HashMap<String,String> params = new HashMap<String,String>();
params.put("EXTERNAL", "false");
- Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+ Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, null, null, "MANAGED_TABLE");
objectStore.createTable(tbl1);
List<String> tables = objectStore.getAllTables(DB1);
Assert.assertEquals(1, tables.size());
Assert.assertEquals(TABLE1, tables.get(0));
- Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+ Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, null, null, "MANAGED_TABLE");
objectStore.alterTable(DB1, TABLE1, newTbl1);
tables = objectStore.getTables(DB1, "new*");
Assert.assertEquals(1, tables.size());
@@ -181,7 +181,7 @@ public class TestObjectStore {
tableParams.put("EXTERNAL", "false");
FieldSchema partitionKey1 = new FieldSchema("Country", serdeConstants.STRING_TYPE_NAME, "");
FieldSchema partitionKey2 = new FieldSchema("State", serdeConstants.STRING_TYPE_NAME, "");
- Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+ Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, null, null, "MANAGED_TABLE");
objectStore.createTable(tbl1);
HashMap<String, String> partitionParams = new HashMap<String, String>();
partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true");
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
index 6cd3a46..c6a134c 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
@@ -19,8 +19,14 @@
package org.apache.hadoop.hive.metastore.hbase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -41,14 +47,8 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestHBaseAggregateStatsCache {
private static final Logger LOG = LoggerFactory.getLogger(TestHBaseAggregateStatsCache.class.getName());
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
index e0c4094..ecc99c3 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
@@ -18,13 +18,18 @@
*/
package org.apache.hadoop.hive.metastore.hbase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
@@ -40,14 +45,8 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestHBaseAggregateStatsCacheWithBitVector {
private static final Logger LOG = LoggerFactory
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
index f4e55ed..99ce96c 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
@@ -18,14 +18,19 @@
*/
package org.apache.hadoop.hive.metastore.hbase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.StatObjectConverter;
import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
@@ -44,14 +49,8 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestHBaseAggregateStatsExtrapolation {
private static final Logger LOG = LoggerFactory
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
index 62918be..87b1ac8 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
@@ -18,14 +18,19 @@
*/
package org.apache.hadoop.hive.metastore.hbase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.StatObjectConverter;
import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
@@ -43,14 +48,8 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestHBaseAggregateStatsNDVUniformDist {
private static final Logger LOG = LoggerFactory
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
index a34f8ac..0cf56e5 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
@@ -27,7 +27,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
-import java.util.SortedSet;
import java.util.TreeMap;
import org.apache.hadoop.hbase.Cell;
@@ -67,7 +66,6 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
index cfe9cd0..4ccb7dd 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
@@ -18,59 +18,38 @@
*/
package org.apache.hadoop.hive.metastore.hbase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Table;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.Mock;
-import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java
index 78715d8..6dfaa9f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java
@@ -18,11 +18,9 @@
package org.apache.hadoop.hive.ql;
-import java.sql.Timestamp;
import java.util.Map;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 8a25399..b12fa9b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -124,6 +124,7 @@ import org.apache.hadoop.hive.ql.metadata.CheckResult;
import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreChecker;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
@@ -136,9 +137,9 @@ import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatter;
import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.AbortTxnsDesc;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc;
@@ -2120,7 +2121,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
needsLocation = doesTableNeedLocation(tbl);
if (tbl.isView()) {
- String createTab_stmt = "CREATE VIEW `" + tableName + "` AS " + tbl.getViewExpandedText();
+ String createTab_stmt = "CREATE VIEW `" + tableName + "` AS " +
+ tbl.getViewExpandedText();
outStream.write(createTab_stmt.getBytes(StandardCharsets.UTF_8));
return 0;
}
@@ -3958,12 +3960,13 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
}
- int partitionBatchSize = HiveConf.getIntVar(conf,
- ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX);
-
// drop the table
db.dropTable(dropTbl.getTableName(), dropTbl.getIfPurge());
if (tbl != null) {
+ // Remove from cache if it is a materialized view
+ if (tbl.isMaterializedView()) {
+ HiveMaterializedViewsRegistry.get().dropMaterializedView(tbl);
+ }
// We have already locked the table in DDLSemanticAnalyzer, don't do it again here
addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
}
@@ -4347,17 +4350,16 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
} else {
// create new view
Table tbl = db.newTable(crtView.getViewName());
+ tbl.setViewOriginalText(crtView.getViewOriginalText());
if (crtView.isMaterialized()) {
+ tbl.setRewriteEnabled(crtView.isRewriteEnabled());
tbl.setTableType(TableType.MATERIALIZED_VIEW);
} else {
+ tbl.setViewExpandedText(crtView.getViewExpandedText());
tbl.setTableType(TableType.VIRTUAL_VIEW);
}
tbl.setSerializationLib(null);
tbl.clearSerDeInfo();
- tbl.setViewOriginalText(crtView.getViewOriginalText());
- if (!crtView.isMaterialized()) {
- tbl.setViewExpandedText(crtView.getViewExpandedText());
- }
tbl.setFields(crtView.getSchema());
if (crtView.getComment() != null) {
tbl.setProperty("comment", crtView.getComment());
@@ -4391,6 +4393,10 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
db.createTable(tbl, crtView.getIfNotExists());
+ // Add to cache if it is a materialized view
+ if (tbl.isMaterializedView()) {
+ HiveMaterializedViewsRegistry.get().addMaterializedView(tbl);
+ }
addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
}
return 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 750fdef..f62d5f3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -44,19 +45,16 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.ConcurrentHashMap;
-
-import com.google.common.collect.ImmutableMap;
import javax.jdo.JDODataStoreException;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import org.apache.calcite.plan.RelOptMaterialization;
import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -81,6 +79,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.PartitionDropOptions;
import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
+import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.AggrStats;
@@ -104,8 +103,8 @@ import org.apache.hadoop.hive.metastore.api.HiveObjectType;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.InsertEventRequestData;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetadataPpdResult;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.MetadataPpdResult;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
@@ -126,13 +125,12 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.FunctionTask;
import org.apache.hadoop.hive.ql.exec.FunctionUtils;
+import org.apache.hadoop.hive.ql.exec.InPlaceUpdates;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.InPlaceUpdates;
import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.log.PerfLogger;
-import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient;
import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
import org.apache.hadoop.hive.ql.plan.DropTableDesc;
@@ -151,6 +149,9 @@ import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -307,7 +308,7 @@ public class Hive {
Hive db = hiveDB.get();
if (db == null || !db.isCurrentUserOwner() || needsRefresh
|| (c != null && db.metaStoreClient != null && !isCompatible(db, c, isFastCheck))) {
- return create(c, false, db, doRegisterAllFns);
+ db = create(c, false, db, doRegisterAllFns);
}
if (c != null) {
db.conf = c;
@@ -1331,6 +1332,27 @@ public class Hive {
}
/**
+ * Get all tables for the specified database.
+ * @param dbName
+ * @return List of table names
+ * @throws HiveException
+ */
+ public List<Table> getAllTableObjects(String dbName) throws HiveException {
+ try {
+ return Lists.transform(getMSC().getTableObjectsByName(dbName, getMSC().getAllTables(dbName)),
+ new com.google.common.base.Function<org.apache.hadoop.hive.metastore.api.Table, Table>() {
+ @Override
+ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) {
+ return new Table(table);
+ }
+ }
+ );
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+ }
+
+ /**
* Returns all existing tables from default database which match the given
* pattern. The matching occurs as per Java regular expressions
*
@@ -1407,6 +1429,55 @@ public class Hive {
}
/**
+ * Get the materialized views that have been enabled for rewriting from the
+ * metastore. If the materialized view is in the cache, we do not need to
+ * parse it to generate a logical plan for the rewriting. Instead, we
+ * return the version present in the cache.
+ *
+ * @return the list of materialized views available for rewriting
+ * @throws HiveException
+ */
+ public List<RelOptMaterialization> getRewritingMaterializedViews() throws HiveException {
+ try {
+ // Final result
+ List<RelOptMaterialization> result = new ArrayList<>();
+ for (String dbName : getMSC().getAllDatabases()) {
+ // From metastore (for security)
+ List<String> tables = getMSC().getAllTables(dbName);
+ // Cached views (includes all)
+ Collection<RelOptMaterialization> cachedViews =
+ HiveMaterializedViewsRegistry.get().getRewritingMaterializedViews(dbName);
+ if (cachedViews.isEmpty()) {
+ // Bail out: empty list
+ continue;
+ }
+ Map<String, RelOptMaterialization> qualifiedNameToView =
+ new HashMap<String, RelOptMaterialization>();
+ for (RelOptMaterialization materialization : cachedViews) {
+ qualifiedNameToView.put(materialization.table.getQualifiedName().get(0), materialization);
+ }
+ for (String table : tables) {
+ // Compose qualified name
+ String fullyQualifiedName = dbName;
+ if (fullyQualifiedName != null && !fullyQualifiedName.isEmpty()) {
+ fullyQualifiedName = fullyQualifiedName + "." + table;
+ } else {
+ fullyQualifiedName = table;
+ }
+ RelOptMaterialization materialization = qualifiedNameToView.get(fullyQualifiedName);
+ if (materialization != null) {
+ // Add to final result set
+ result.add(materialization);
+ }
+ }
+ }
+ return result;
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+ }
+
+ /**
* Get all existing database names.
*
* @return List of database names.
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
new file mode 100644
index 0000000..89c87cd
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -0,0 +1,393 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.metadata;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.calcite.adapter.druid.DruidQuery;
+import org.apache.calcite.adapter.druid.DruidSchema;
+import org.apache.calcite.adapter.druid.DruidTable;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptMaterialization;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rel.type.RelDataTypeImpl;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.hadoop.hive.conf.Constants;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
+import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveVolcanoPlanner;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.CalcitePlanner;
+import org.apache.hadoop.hive.ql.parse.ParseDriver;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
+import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.joda.time.Interval;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Registry for materialized views. The goal of this cache is to avoid parsing and creating
+ * logical plans for the materialized views at query runtime. When a query arrives, we will
+ * just need to consult this cache and extract the logical plans for the views (which had
+ * already been parsed) from it.
+ */
+public final class HiveMaterializedViewsRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveMaterializedViewsRegistry.class);
+
+ /* Singleton */
+ private static final HiveMaterializedViewsRegistry SINGLETON = new HiveMaterializedViewsRegistry();
+
+ /* Key is the database name. Value a map from a unique identifier for the view comprising
+ * the qualified name and the creation time, to the view object.
+ * Since currently we cannot alter a materialized view, that should suffice to identify
+ * whether the cached view is up to date or not.
+ * Creation time is useful to ensure correctness in case multiple HS2 instances are used. */
+ private final ConcurrentMap<String, ConcurrentMap<ViewKey, RelOptMaterialization>> materializedViews =
+ new ConcurrentHashMap<String, ConcurrentMap<ViewKey, RelOptMaterialization>>();
+ private final ExecutorService pool = Executors.newCachedThreadPool();
+
+ private HiveMaterializedViewsRegistry() {
+ }
+
+ /**
+ * Get instance of HiveMaterializedViewsRegistry.
+ *
+ * @return the singleton
+ */
+ public static HiveMaterializedViewsRegistry get() {
+ return SINGLETON;
+ }
+
+ /**
+ * Initialize the registry for the given database. It will extract the materialized views
+ * that are enabled for rewriting from the metastore for the current user, parse them,
+ * and register them in this cache.
+ *
+ * The loading process runs on the background; the method returns in the moment that the
+ * runnable task is created, thus the views will still not be loaded in the cache when
+ * it does.
+ */
+ public void init(final Hive db) {
+ try {
+ List<Table> tables = new ArrayList<Table>();
+ for (String dbName : db.getAllDatabases()) {
+ // TODO: We should enhance metastore API such that it returns only
+ // materialized views instead of all tables
+ tables.addAll(db.getAllTableObjects(dbName));
+ }
+ pool.submit(new Loader(tables));
+ } catch (HiveException e) {
+ LOG.error("Problem connecting to the metastore when initializing the view registry");
+ }
+ }
+
+ private class Loader implements Runnable {
+ private final List<Table> tables;
+
+ private Loader(List<Table> tables) {
+ this.tables = tables;
+ }
+
+ @Override
+ public void run() {
+ for (Table table : tables) {
+ if (table.isMaterializedView()) {
+ addMaterializedView(table);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds the materialized view to the cache.
+ *
+ * @param materializedViewTable the materialized view
+ */
+ public RelOptMaterialization addMaterializedView(Table materializedViewTable) {
+ // Bail out if it is not enabled for rewriting
+ if (!materializedViewTable.isRewriteEnabled()) {
+ return null;
+ }
+ ConcurrentMap<ViewKey, RelOptMaterialization> cq =
+ new ConcurrentHashMap<ViewKey, RelOptMaterialization>();
+ final ConcurrentMap<ViewKey, RelOptMaterialization> prevCq = materializedViews.putIfAbsent(
+ materializedViewTable.getDbName(), cq);
+ if (prevCq != null) {
+ cq = prevCq;
+ }
+ // Bail out if it already exists
+ final ViewKey vk = new ViewKey(
+ materializedViewTable.getTableName(), materializedViewTable.getCreateTime());
+ if (cq.containsKey(vk)) {
+ return null;
+ }
+ // Add to cache
+ final String viewQuery = materializedViewTable.getViewOriginalText();
+ final RelNode tableRel = createTableScan(materializedViewTable);
+ if (tableRel == null) {
+ LOG.warn("Materialized view " + materializedViewTable.getCompleteName() +
+ " ignored; error creating view replacement");
+ return null;
+ }
+ final RelNode queryRel = parseQuery(viewQuery);
+ if (queryRel == null) {
+ LOG.warn("Materialized view " + materializedViewTable.getCompleteName() +
+ " ignored; error parsing original query");
+ return null;
+ }
+ RelOptMaterialization materialization = new RelOptMaterialization(tableRel, queryRel, null);
+ cq.put(vk, materialization);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cached materialized view for rewriting: " + tableRel.getTable().getQualifiedName());
+ }
+ return materialization;
+ }
+
+ /**
+ * Removes the materialized view from the cache.
+ *
+ * @param materializedViewTable the materialized view to remove
+ */
+ public void dropMaterializedView(Table materializedViewTable) {
+ // Bail out if it is not enabled for rewriting
+ if (!materializedViewTable.isRewriteEnabled()) {
+ return;
+ }
+ final ViewKey vk = new ViewKey(
+ materializedViewTable.getTableName(), materializedViewTable.getCreateTime());
+ materializedViews.get(materializedViewTable.getDbName()).remove(vk);
+ }
+
+ /**
+ * Returns the materialized views in the cache for the given database.
+ *
+ * @param dbName the database
+ * @return the collection of materialized views, or the empty collection if none
+ */
+ Collection<RelOptMaterialization> getRewritingMaterializedViews(String dbName) {
+ if (materializedViews.get(dbName) != null) {
+ return Collections.unmodifiableCollection(materializedViews.get(dbName).values());
+ }
+ return ImmutableList.of();
+ }
+
+ private static RelNode createTableScan(Table viewTable) {
+ // 0. Recreate cluster
+ final RelOptPlanner planner = HiveVolcanoPlanner.createPlanner(null);
+ final RexBuilder rexBuilder = new RexBuilder(new JavaTypeFactoryImpl());
+ final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
+
+ // 1. Create column schema
+ final RowResolver rr = new RowResolver();
+ // 1.1 Add Column info for non partion cols (Object Inspector fields)
+ StructObjectInspector rowObjectInspector;
+ try {
+ rowObjectInspector = (StructObjectInspector) viewTable.getDeserializer()
+ .getObjectInspector();
+ } catch (SerDeException e) {
+ // Bail out
+ return null;
+ }
+ List<? extends StructField> fields = rowObjectInspector.getAllStructFieldRefs();
+ ColumnInfo colInfo;
+ String colName;
+ ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>();
+ for (int i = 0; i < fields.size(); i++) {
+ colName = fields.get(i).getFieldName();
+ colInfo = new ColumnInfo(
+ fields.get(i).getFieldName(),
+ TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()),
+ null, false);
+ rr.put(null, colName, colInfo);
+ cInfoLst.add(colInfo);
+ }
+ ArrayList<ColumnInfo> nonPartitionColumns = new ArrayList<ColumnInfo>(cInfoLst);
+
+ // 1.2 Add column info corresponding to partition columns
+ ArrayList<ColumnInfo> partitionColumns = new ArrayList<ColumnInfo>();
+ for (FieldSchema part_col : viewTable.getPartCols()) {
+ colName = part_col.getName();
+ colInfo = new ColumnInfo(colName,
+ TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true);
+ rr.put(null, colName, colInfo);
+ cInfoLst.add(colInfo);
+ partitionColumns.add(colInfo);
+ }
+
+ // 1.3 Build row type from field <type, name>
+ RelDataType rowType;
+ try {
+ rowType = TypeConverter.getType(cluster, rr, null);
+ } catch (CalciteSemanticException e) {
+ // Bail out
+ return null;
+ }
+
+ // 2. Build RelOptAbstractTable
+ String fullyQualifiedTabName = viewTable.getDbName();
+ if (fullyQualifiedTabName != null && !fullyQualifiedTabName.isEmpty()) {
+ fullyQualifiedTabName = fullyQualifiedTabName + "." + viewTable.getTableName();
+ }
+ else {
+ fullyQualifiedTabName = viewTable.getTableName();
+ }
+ RelOptHiveTable optTable = new RelOptHiveTable(null, fullyQualifiedTabName,
+ rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<VirtualColumn>(),
+ SessionState.get().getConf(), new HashMap<String, PrunedPartitionList>(),
+ new AtomicInteger());
+ RelNode tableRel;
+
+ // 3. Build operator
+ if (obtainTableType(viewTable) == TableType.DRUID) {
+ // Build Druid query
+ String address = HiveConf.getVar(SessionState.get().getConf(),
+ HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS);
+ String dataSource = viewTable.getParameters().get(Constants.DRUID_DATA_SOURCE);
+ Set<String> metrics = new HashSet<>();
+ List<RelDataType> druidColTypes = new ArrayList<>();
+ List<String> druidColNames = new ArrayList<>();
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ druidColTypes.add(field.getType());
+ druidColNames.add(field.getName());
+ if (field.getName().equals(DruidTable.DEFAULT_TIMESTAMP_COLUMN)) {
+ // timestamp
+ continue;
+ }
+ if (field.getType().getSqlTypeName() == SqlTypeName.VARCHAR) {
+ // dimension
+ continue;
+ }
+ metrics.add(field.getName());
+ }
+ List<Interval> intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
+
+ DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false),
+ dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals);
+ final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
+ optTable, viewTable.getTableName(), null, false, false);
+ tableRel = DruidQuery.create(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
+ optTable, druidTable, ImmutableList.<RelNode>of(scan));
+ } else {
+ // Build Hive Table Scan Rel
+ tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable,
+ viewTable.getTableName(), null, false, false);
+ }
+ return tableRel;
+ }
+
+ private static RelNode parseQuery(String viewQuery) {
+ try {
+ final ParseDriver pd = new ParseDriver();
+ final ASTNode node = ParseUtils.findRootNonNullToken(pd.parse(viewQuery));
+ final QueryState qs = new QueryState(SessionState.get().getConf());
+ CalcitePlanner analyzer = new CalcitePlanner(qs);
+ analyzer.initCtx(new Context(SessionState.get().getConf()));
+ analyzer.init(false);
+ return analyzer.genLogicalPlan(node);
+ } catch (Exception e) {
+ // We could not parse the view
+ return null;
+ }
+ }
+
+ private static class ViewKey {
+ private String viewName;
+ private int creationDate;
+
+ private ViewKey(String viewName, int creationTime) {
+ this.viewName = viewName;
+ this.creationDate = creationTime;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(this == obj) {
+ return true;
+ }
+ if((obj == null) || (obj.getClass() != this.getClass())) {
+ return false;
+ }
+ ViewKey viewKey = (ViewKey) obj;
+ return creationDate == viewKey.creationDate &&
+ (viewName == viewKey.viewName || (viewName != null && viewName.equals(viewKey.viewName)));
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 31 * hash + creationDate;
+ hash = 31 * hash + viewName.hashCode();
+ return hash;
+ }
+
+ @Override
+ public String toString() {
+ return "ViewKey{" + viewName + "," + creationDate + "}";
+ }
+ }
+
+ private static TableType obtainTableType(Table tabMetaData) {
+ if (tabMetaData.getStorageHandler() != null &&
+ tabMetaData.getStorageHandler().toString().equals(
+ Constants.DRUID_HIVE_STORAGE_HANDLER_ID)) {
+ return TableType.DRUID;
+ }
+ return TableType.NATIVE;
+ }
+
+ private enum TableType {
+ DRUID,
+ NATIVE
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index ea90889..c6ae6f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -180,7 +180,6 @@ public class Table implements Serializable {
t.setOwner(SessionState.getUserFromAuthenticator());
// set create time
t.setCreateTime((int) (System.currentTimeMillis() / 1000));
-
}
return t;
}
@@ -809,9 +808,6 @@ public class Table implements Serializable {
return tTable.getViewExpandedText();
}
- public void clearSerDeInfo() {
- tTable.getSd().getSerdeInfo().getParameters().clear();
- }
/**
* @param viewExpandedText
* the expanded view text to set
@@ -821,6 +817,25 @@ public class Table implements Serializable {
}
/**
+ * @return whether this view can be used for rewriting queries
+ */
+ public boolean isRewriteEnabled() {
+ return tTable.isRewriteEnabled();
+ }
+
+ /**
+ * @param rewriteEnabled
+ * whether this view can be used for rewriting queries
+ */
+ public void setRewriteEnabled(boolean rewriteEnabled) {
+ tTable.setRewriteEnabled(rewriteEnabled);
+ }
+
+ public void clearSerDeInfo() {
+ tTable.getSd().getSerdeInfo().getParameters().clear();
+ }
+
+ /**
* @return whether this table is actually a view
*/
public boolean isView() {
@@ -863,6 +878,10 @@ public class Table implements Serializable {
return new Table(tTable.deepCopy());
}
+ public int getCreateTime() {
+ return tTable.getCreateTime();
+ }
+
public void setCreateTime(int createTime) {
tTable.setCreateTime(createTime);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
index c850e43..03f74dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
@@ -365,7 +365,7 @@ public final class MetaDataFormatUtils {
tableInfo.append(LINE_DELIM).append("# Storage Information").append(LINE_DELIM);
getStorageDescriptorInfo(tableInfo, table.getTTable().getSd());
- if (table.isView()) {
+ if (table.isView() || table.isMaterializedView()) {
tableInfo.append(LINE_DELIM).append("# View Information").append(LINE_DELIM);
getViewInfo(tableInfo, table);
}
@@ -376,6 +376,7 @@ public final class MetaDataFormatUtils {
private static void getViewInfo(StringBuilder tableInfo, Table tbl) {
formatOutput("View Original Text:", tbl.getViewOriginalText(), tableInfo);
formatOutput("View Expanded Text:", tbl.getViewExpandedText(), tableInfo);
+ formatOutput("View Rewrite Enabled:", tbl.isRewriteEnabled() ? "Yes" : "No", tableInfo);
}
private static void getStorageDescriptorInfo(StringBuilder tableInfo,
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
index f7958c6..c6ac056 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
@@ -22,8 +22,8 @@ import java.util.HashSet;
import java.util.List;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexExecutorImpl;
import org.apache.calcite.rex.RexNode;
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
@@ -36,15 +36,15 @@ import org.slf4j.LoggerFactory;
-public class HiveRexExecutorImpl implements RelOptPlanner.Executor {
+public class HiveRexExecutorImpl extends RexExecutorImpl {
- private final RelOptCluster cluster;
+ private static final Logger LOG = LoggerFactory.getLogger(HiveRexExecutorImpl.class);
- protected final Logger LOG;
+ private final RelOptCluster cluster;
public HiveRexExecutorImpl(RelOptCluster cluster) {
+ super(null);
this.cluster = cluster;
- LOG = LoggerFactory.getLogger(this.getClass().getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
index 4ebbb13..009d9e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
@@ -481,4 +481,18 @@ public class RelOptHiveTable extends RelOptAbstractTable {
public Map<Integer, ColumnInfo> getNonPartColInfoMap() {
return hiveNonPartitionColsMap;
}
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof RelOptHiveTable
+ && this.rowType.equals(((RelOptHiveTable) obj).getRowType())
+ && this.getHiveTableMD().equals(((RelOptHiveTable) obj).getHiveTableMD());
+ }
+
+ @Override
+ public int hashCode() {
+ return (this.getHiveTableMD() == null)
+ ? super.hashCode() : this.getHiveTableMD().hashCode();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
index 6df6026..63bbdac 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
@@ -23,8 +23,6 @@ import java.util.Set;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
@@ -63,11 +61,6 @@ public class HiveAggregate extends Aggregate implements HiveRelNode {
public void implement(Implementor implementor) {
}
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
- return mq.getNonCumulativeCost(this);
- }
-
// getRows will call estimateRowCount
@Override
public double estimateRowCount(RelMetadataQuery mq) {
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
index 0410c91..d5fa856 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
@@ -18,12 +18,9 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexNode;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
@@ -43,9 +40,4 @@ public class HiveFilter extends Filter implements HiveRelNode {
public void implement(Implementor implementor) {
}
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
- return mq.getNonCumulativeCost(this);
- }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
index ba9483e..dc2fa86 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
@@ -24,7 +24,6 @@ import java.util.Set;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelCollation;
@@ -214,14 +213,6 @@ public class HiveJoin extends Join implements HiveRelNode {
this.joinCost = joinCost;
}
- /**
- * Model cost of join as size of Inputs.
- */
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
- return mq.getNonCumulativeCost(this);
- }
-
@Override
public RelWriter explainTerms(RelWriter pw) {
return super.explainTerms(pw)
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
index 3e0a9a6..447db8e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
@@ -22,13 +22,10 @@ import java.util.Collections;
import java.util.List;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
@@ -174,11 +171,6 @@ public class HiveProject extends Project implements HiveRelNode {
}
@Override
- public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
- return mq.getNonCumulativeCost(this);
- }
-
- @Override
public void implement(Implementor implementor) {
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
index d899667..65211cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
@@ -21,15 +21,12 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.util.ImmutableIntList;
@@ -107,9 +104,4 @@ public class HiveSemiJoin extends SemiJoin implements HiveRelNode {
public void implement(Implementor implementor) {
}
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
- return mq.getNonCumulativeCost(this);
- }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3da29fe7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
index cccbd2f..fed1664 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
@@ -24,7 +24,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
@@ -124,11 +123,6 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
newRowtype, this.useQBIdInDigest, this.insideView);
}
- @Override
- public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
- return mq.getNonCumulativeCost(this);
- }
-
@Override public RelWriter explainTerms(RelWriter pw) {
if (this.useQBIdInDigest) {
// TODO: Only the qualified name should be left here