You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ke...@apache.org on 2012/05/02 00:52:40 UTC
svn commit: r1332873 [1/4] - in
/hive/trunk/contrib/src/java/org/apache/hadoop/hive: metastore/
metastore/hooks/ ql/ ql/hooks/ ql/hooks/conf/ ql/stats/
Author: kevinwilfong
Date: Tue May 1 22:52:38 2012
New Revision: 1332873
URL: http://svn.apache.org/viewvc?rev=1332873&view=rev
Log:
HIVE-2986 Create the hooks (njain via kevinwilfong)
Added:
hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/
hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/
hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ExternalInputsHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/FbUpdateInputAccessTimeHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/FifoPoolHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/HiveConfigLoggingHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/JobStatsHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/JobTrackerHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/LineageHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/Pair.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/PyRulesHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/QueryDroppedPartitionsHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/QueryPlanHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/RegressionTestHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ReplicationHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SMCStatsDBHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SampleConcurrencyHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SmcConfigDriverRunHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SmcConfigHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SplitSizeHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/StartFinishHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SuggestionPrintingHook.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/Triple.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/conf/
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/conf/FBHiveConf.java
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/stats/
hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/stats/HiveStatsMetricsPublisher.java
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java Tue May 1 22:52:38 2012
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
+import org.apache.hadoop.hive.ql.hooks.BaseReplicationHook;
+import org.apache.hadoop.hive.ql.hooks.ConnectionUrlFactory;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.ReplicationHook;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TSimpleJSONProtocol;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/*
+ * MetaStoreEventListener that logs metastore operations to the audit log.
+ * The operations that this listener logs are only those from the thrift server,
+ * and not the CLI, because ReplicationHook currently logs queries from the CLI.
+ */
+public class AuditMetaStoreEventListener extends MetaStoreEventListener {
+ public static final Log LOG = LogFactory.getLog(AuditMetaStoreEventListener.class);
+
+ private static final String COMMAND_TYPE = "METASTORE_API";
+ private static final String COMMAND_NAME = "name";
+ private static final String ADD_PARTITION_COMMAND = "ADD_PARTITION";
+ private static final String ALTER_PARTITION_COMMAND = "ALTER_PARTITION";
+ private static final String ALTER_TABLE_COMMAND = "ALTER_TABLE";
+ private static final String CREATE_TABLE_COMMAND = "CREATE_TABLE";
+ private static final String DROP_PARTITION_COMMAND = "DROP_PARTITION";
+ private static final String DROP_DATABASE_COMMAND = "DROP_DATABASE";
+ private static final String DROP_TABLE_COMMAND = "DROP_TABLE";
+ private static final String NEW_TABLE = "new_table";
+ private static final String OLD_TABLE = "old_table";
+ private static final String NEW_PARTITION = "new_partition";
+ private static final String OLD_PARTITION = "old_partition";
+
+ private final TSerializer jsonSerializer;
+
+
+ protected ConnectionUrlFactory urlFactory = null;
+
+ public AuditMetaStoreEventListener(Configuration config) throws Exception{
+ super(config);
+ urlFactory = BaseReplicationHook.getReplicationMySqlUrl();
+ jsonSerializer = new TSerializer(new TSimpleJSONProtocol.Factory());
+ }
+
+ private void insertToDB(Set<ReadEntity> inputs,
+ Set<WriteEntity> outputs, ListenerEvent event, String command) throws MetaException {
+ HiveConf conf = event.getHandler().getHiveConf();
+ //if HIVEQUERYID is set, then this command came from a CLI
+ // (and will execute posthooks). We don't want to log such a command
+ if (conf.getVar(HiveConf.ConfVars.HIVEQUERYID) == null ||
+ conf.getVar(HiveConf.ConfVars.HIVEQUERYID).isEmpty()) {
+ try {
+ ArrayList<Object> sqlParams = new ArrayList<Object>();
+ sqlParams.add(command);
+ sqlParams.add(StringEscapeUtils.escapeJava(ReplicationHook.entitiesToString(inputs)));
+ sqlParams.add(ReplicationHook.entitiesToString(outputs));
+ sqlParams.add(COMMAND_TYPE);
+
+ // Assertion at beginning of method guarantees this string will remain empty
+ String sql = "insert into snc1_command_log " +
+ " set command = ?, inputs = ?, outputs = ?, command_type = ?";
+
+ String ipAddress = HMSHandler.getIpAddress();
+ if (ipAddress != null) {
+ if (ipAddress.startsWith("/")) {
+ ipAddress = ipAddress.replaceFirst("/", "");
+ }
+ sql += ", user_info = ?";
+ sqlParams.add(ipAddress);
+ }
+
+ HookUtils.runInsert(conf, urlFactory, sql, sqlParams, HookUtils
+ .getSqlNumRetry(conf));
+ } catch (Exception e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+ }
+
+ private Table getTableFromPart(Partition p, ListenerEvent event) throws MetaException {
+ try {
+ return event.getHandler().get_table(p.getDbName(), p.getTableName());
+ } catch (Exception e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+
+ private org.apache.hadoop.hive.ql.metadata.Table getQlTable(Table t) {
+ return new org.apache.hadoop.hive.ql.metadata.Table(t);
+ }
+
+ private org.apache.hadoop.hive.ql.metadata.Partition getQlPartition(Table t, Partition p)
+ throws MetaException{
+ try {
+ org.apache.hadoop.hive.ql.metadata.Table qlTable = getQlTable(t);
+ return new org.apache.hadoop.hive.ql.metadata.Partition(qlTable, p);
+ } catch (HiveException e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+
+ private ReadEntity getPartitionInput(Partition p, ListenerEvent event) throws MetaException {
+ Table mTable = getTableFromPart(p, event);
+ ReadEntity input = new ReadEntity(getQlPartition(mTable, p));
+ return input;
+ }
+
+ private WriteEntity getPartitionOutput(Partition p, ListenerEvent event) throws MetaException {
+ try {
+ Table mTable = event.getHandler().get_table(p.getDbName(), p.getTableName());
+ WriteEntity output = new WriteEntity(getQlPartition(mTable, p));
+ return output;
+ } catch (Exception e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+
+ private void logNoSuccess() {
+ LOG.info("ListenerEvent success is false");
+ }
+
+ private void addCommandNameToCommand(JSONObject command, String name) {
+ try {
+ command.put(COMMAND_NAME, name);
+ } catch (JSONException e) {
+ LOG.error("Could not add command name to JSON object", e);
+ }
+ }
+
+ private void addTBaseToCommand(JSONObject command, TBase object, String objectName) {
+ try {
+ command.put(objectName, new JSONObject(jsonSerializer.toString(object)));
+ } catch (JSONException e) {
+ LOG.error("Could not add " + objectName + " to JSON object", e);
+ } catch (TException e) {
+ LOG.error("Could not serialize " + objectName + " to JSON", e);
+ }
+ }
+
+ @Override
+ public void onAddPartition(AddPartitionEvent event) throws MetaException {
+ if (!event.getStatus()) {
+ logNoSuccess();
+ return;
+ }
+
+ Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+ Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+ JSONObject command = new JSONObject();
+ inputs.add(new ReadEntity(getQlTable(getTableFromPart(event.getPartition(), event))));
+ outputs.add(getPartitionOutput(event.getPartition(), event));
+ addCommandNameToCommand(command, ADD_PARTITION_COMMAND);
+
+ insertToDB(inputs, outputs, event, command.toString());
+ }
+
+ @Override
+ public void onAlterPartition(AlterPartitionEvent event) throws MetaException {
+ if (!event.getStatus()) {
+ logNoSuccess();
+ return;
+ }
+
+ Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+ Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+ JSONObject command = new JSONObject();
+ inputs.add(getPartitionInput(event.getOldPartition(), event));
+ outputs.add(getPartitionOutput(event.getNewPartition(), event));
+ addCommandNameToCommand(command, ALTER_PARTITION_COMMAND);
+ addTBaseToCommand(command, event.getOldPartition(), OLD_PARTITION);
+ addTBaseToCommand(command, event.getNewPartition(), NEW_PARTITION);
+
+ insertToDB(inputs, outputs, event, command.toString());
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent event) throws MetaException {
+ if (!event.getStatus()) {
+ logNoSuccess();
+ return;
+ }
+
+ Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+ Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+ JSONObject command = new JSONObject();
+ inputs.add(new ReadEntity(getQlTable(getTableFromPart(event.getPartition(), event))));
+ outputs.add(getPartitionOutput(event.getPartition(), event));
+ addCommandNameToCommand(command, DROP_PARTITION_COMMAND);
+
+ insertToDB(inputs, outputs, event, command.toString());
+ }
+
+ @Override
+ /*
+ * Currently, on the create database CLI command, nothing gets logged.
+ */
+ public void onCreateDatabase(CreateDatabaseEvent event) throws MetaException {
+ }
+
+ @Override
+ public void onDropDatabase(DropDatabaseEvent event) throws MetaException {
+ if (!event.getStatus()) {
+ logNoSuccess();
+ return;
+ }
+
+ Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+ Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+ JSONObject command = new JSONObject();
+ addCommandNameToCommand(command, DROP_DATABASE_COMMAND);
+
+ insertToDB(inputs, outputs, event, command.toString());
+ }
+
+ @Override
+ public void onCreateTable(CreateTableEvent event) throws MetaException {
+ if (!event.getStatus()) {
+ logNoSuccess();
+ return;
+ }
+
+ Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+ Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+ JSONObject command = new JSONObject();
+ outputs.add(new WriteEntity(getQlTable(event.getTable())));
+ addCommandNameToCommand(command, CREATE_TABLE_COMMAND);
+
+ insertToDB(inputs, outputs, event, command.toString());
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent event) throws MetaException {
+ if (!event.getStatus()) {
+ logNoSuccess();
+ return;
+ }
+
+ Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+ Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+ JSONObject command = new JSONObject();
+ inputs.add(new ReadEntity(getQlTable(event.getTable())));
+ outputs.add(new WriteEntity(getQlTable(event.getTable())));
+ addCommandNameToCommand(command, DROP_TABLE_COMMAND);
+
+ insertToDB(inputs, outputs, event, command.toString());
+ }
+
+ @Override
+ public void onAlterTable(AlterTableEvent event) throws MetaException {
+ if (!event.getStatus()) {
+ logNoSuccess();
+ return;
+ }
+
+ Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+ Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+ JSONObject command = new JSONObject();
+ inputs.add(new ReadEntity(getQlTable(event.getOldTable())));
+ outputs.add(new WriteEntity(getQlTable(event.getOldTable())));
+ outputs.add(new WriteEntity(getQlTable(event.getNewTable())));
+ addCommandNameToCommand(command, ALTER_TABLE_COMMAND);
+ addTBaseToCommand(command, event.getOldTable(), OLD_TABLE);
+ addTBaseToCommand(command, event.getNewTable(), NEW_TABLE);
+
+ insertToDB(inputs, outputs, event, command.toString());
+ }
+
+ @Override
+ public void onLoadPartitionDone(LoadPartitionDoneEvent lpe) throws MetaException {
+ }
+
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java Tue May 1 22:52:38 2012
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.AbstractMap;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.MetaStoreEndFunctionContext;
+import org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+/*
+ * MetaStoreEndFunctionListener that uses the StatsManager to collect fb303 counters for
+ * the number of successes and failures for each metastore thrift function, bucketed by time.
+ */
+public class CounterMetaStoreEndFunctionListener extends MetaStoreEndFunctionListener {
+
+ StatsManager stats = null;
+
+ public CounterMetaStoreEndFunctionListener(Configuration config) {
+ super(config);
+ String statsMgr = config.get(FBHiveConf.METASTORE_LISTENER_STATS_MANAGER);
+ if ((statsMgr == null) || (statsMgr.isEmpty())) {
+ return;
+ }
+
+ stats = HookUtils.getObject(config, statsMgr);
+ }
+
+ @Override
+ public void onEndFunction(String functionName, MetaStoreEndFunctionContext context) {
+ if (stats == null) {
+ return;
+ }
+
+ // Construct the counter name, as <functionName> for success
+ // and <functionName.failure> for failure
+ String statName = functionName + (context.isSuccess() ? "" : ".failure");
+
+ // If this is the first time this counter name has been seen, initialize it
+ if (!stats.containsKey(statName)) {
+ stats.addCountStatType(statName);
+ }
+
+ stats.addStatValue(statName, 1);
+ }
+
+ @Override
+ public void exportCounters(AbstractMap<String, Long> counters) {
+ if (stats == null) {
+ return;
+ }
+
+ // For each counter the StatsManager has collected, add it to the map of fb303 counters
+ for (Entry<String, Long> entry : stats.getCounters().entrySet()) {
+ counters.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java Tue May 1 22:52:38 2012
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.HiveAlterHandler;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+/*
+ * Subclass of HiveAlterHandler. Checks that if the table, or partition's table has a
+ * creation_cluster set, that cluster matches the current cluster where the metastore is running.
+ * If so, or if the table or partition's table does not have creation_cluster set, it calls its
+ * super classes implementation of the alter method, i.e. it behaves normally. If not, it throws
+ * a MetaException.
+ */
+public class FbhiveAlterHandler extends HiveAlterHandler {
+ public static final Log LOG = LogFactory.getLog(FbhiveAlterHandler.class);
+
+ @Override
+ public Partition alterPartition(RawStore ms, Warehouse wh, String dbname,
+ String name, List<String> part_vals, Partition new_part)
+ throws InvalidOperationException, InvalidObjectException,
+ AlreadyExistsException, MetaException {
+
+ String exception = "Partition in table " + name + " cannot be altered.";
+ checkTableCluster(ms, dbname, name, exception);
+
+ return super.alterPartition(ms, wh, dbname, name, part_vals, new_part);
+ }
+
+ @Override
+ public void alterTable(RawStore msdb, Warehouse wh, String dbname,
+ String name, Table newt) throws InvalidOperationException, MetaException {
+
+ String exception = "Table " + name + " cannot be altered.";
+ checkTableCluster(msdb, dbname, name, exception);
+
+ super.alterTable(msdb, wh, dbname, name, newt);
+ }
+
+ private void checkTableCluster(RawStore msdb, String dbName, String tableName,
+ String exception) throws MetaException{
+
+ Table oldt = msdb.getTable(dbName.toLowerCase(), tableName.toLowerCase());
+ if (oldt != null) {
+ String creationCluster = oldt.getParameters().get(HookUtils.TABLE_CREATION_CLUSTER);
+ String currentCluster = hiveConf.get(FBHiveConf.FB_CURRENT_CLUSTER);
+ if (creationCluster != null &&
+ currentCluster != null &&
+ !creationCluster.equals(currentCluster)) {
+ throw new MetaException(exception +
+ " Table's cluster is " + creationCluster + "," +
+ " whereas current package is " + currentCluster);
+ }
+ }
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.hooks.ConnectionUrlFactory;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+public class MysqlSmcHook implements JDOConnectionURLHook {
+
+ static final private Log LOG =
+ LogFactory.getLog("hive.metastore.hooks.MysqlSmcHook");
+
+ ConnectionUrlFactory urlFactory = null;
+
+ @Override
+ public String getJdoConnectionUrl(Configuration conf)
+ throws Exception {
+
+ String smcUrl = conf.get(FBHiveConf.METASTORE_SMC_URL);
+ if (smcUrl == null) {
+ throw new Exception(FBHiveConf.METASTORE_SMC_URL + " is not defined");
+ }
+ String mysqlTier = conf.get(FBHiveConf.METASTORE_MYSQL_TIER_VAR_NAME);
+ if (mysqlTier == null) {
+ throw new Exception(FBHiveConf.METASTORE_MYSQL_TIER_VAR_NAME + " is not defined");
+ }
+ String mysqlProps = conf.get(FBHiveConf.METASTORE_MYSQL_PROPS);
+ if (mysqlProps == null) {
+ throw new Exception(FBHiveConf.METASTORE_MYSQL_PROPS + " is not defined");
+ }
+ if (urlFactory == null) {
+ urlFactory = HookUtils.getUrlFactory(
+ conf,
+ FBHiveConf.CONNECTION_FACTORY,
+ null,
+ FBHiveConf.METASTORE_MYSQL_TIER_VAR_NAME,
+ null,
+ FBHiveConf.METASTORE_MYSQL_PROPS);
+ }
+
+ urlFactory.updateProps(smcUrl, mysqlTier, mysqlProps);
+ return urlFactory.getUrl();
+ }
+
+ @Override
+ public void notifyBadConnectionUrl(String url) {
+ LOG.error("Notified of a bad URL: " + url);
+ }
+
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java Tue May 1 22:52:38 2012
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.Map;
+
+
+public interface StatsManager {
+ public boolean containsKey(String statName);
+ public void addCountStatType(String statName);
+ public void addStatValue(String statName, int value);
+ public Map<String, Long> getCounters();
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.thrift.TException;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * An abstract class which should be extended by hooks which read configurations from an SMC
+ * config tier.
+ */
+public abstract class AbstractSmcConfigHook {
+
+ static final private Log LOG = LogFactory.getLog(AbstractSmcConfigHook.class);
+
+ private static final String CONFIG_FIELD = "config";
+ private static final String ENABLED_FIELD = "enabled";
+ private ThreadLocal<ConnectionUrlFactory> urlFactory = null;
+
+ /**
+ * Given a HiveConf, checks if the SMC hook enabled config is set to true
+ *
+ * @param conf
+ * @return
+ */
+ protected boolean isEnabled(HiveConf conf) {
+ boolean enabled = conf.getBoolean(FBHiveConf.ENABLED_CONFIG, false);
+
+ if (!enabled) {
+ LOG.error("SMC hook is not enabled.");
+ }
+
+ return enabled;
+ }
+
+ /**
+ * In each top level config object (jo) there is an enabled field. This method checks that that
+ * field exists, is set properly, and is set to true.
+ *
+ * @param jo
+ * @param packageName
+ * @return
+ * @throws JSONException
+ */
+ protected boolean isConfigEnabled(JSONObject jo, String packageName) throws JSONException {
+ boolean enabled = false;
+
+ Object enabledObj = null;
+
+ if (jo.has(ENABLED_FIELD)) {
+ enabledObj = jo.get(ENABLED_FIELD);
+ }
+
+ if (enabledObj == null || !(enabledObj instanceof Boolean) ) {
+ LOG.error("enabled not properly set!");
+ return false;
+ }
+
+ enabled = enabledObj.equals(Boolean.TRUE);
+
+ if (!enabled) {
+ LOG.error("package " + packageName + " is not enabled");
+ }
+
+ return enabled;
+ }
+
+ /**
+ * Given a HiveConf object, this method goes to the config tier and retrieves the underlying
+ * config object (whether that's an array, object, or any other type of JSON). It also performs
+ * checks that the tier can be retrieved, the package name is set, the config is enabled, etc.
+ *
+ * @param conf
+ * @return
+ * @throws JSONException
+ * @throws ServiceException
+ * @throws TException
+ */
+ protected Object getConfigObject(HiveConf conf)
+ throws JSONException, Exception, TException {
+
+ // Get the properties for this package
+ String packageName = conf.get(FBHiveConf.FB_CURRENT_CLUSTER);
+ if (packageName == null) {
+ LOG.error("Unable to use configs stored in SMC - no hive package set.");
+ return null;
+ }
+
+ if (urlFactory == null) {
+ urlFactory = new ThreadLocal<ConnectionUrlFactory>();
+ urlFactory.set(HookUtils.getUrlFactory(conf,
+ FBHiveConf.CONNECTION_FACTORY, null, null, null));
+ }
+
+ String s = urlFactory.get().getValue(conf.get(FBHiveConf.HIVE_CONFIG_TIER), packageName);
+ JSONObject jo = new JSONObject(s);
+
+ Object configObj = null;
+
+ if (!isConfigEnabled(jo, packageName)) {
+ return null;
+ }
+
+ if (jo.has(CONFIG_FIELD)) {
+ configObj = jo.get(CONFIG_FIELD);
+ }
+
+ return configObj;
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Adding a hook to stop platinum tables from getting modified in silver
+ */
+public class AlterTableRestrictHook implements ExecuteWithHookContext {
+ static final private Log LOG = LogFactory.getLog("hive.ql.hooks.AlterTableRestrictHook");
+
+ String current_cluster = null;
+
+ public AlterTableRestrictHook() throws Exception {
+ HiveConf conf = new HiveConf(AlterTableRestrictHook.class);
+ current_cluster = conf.get(FBHiveConf.FB_CURRENT_CLUSTER);
+ }
+
+ /**
+ * Restrict the alter table command if the current cluster is not the same
+ * as the creation cluster
+ *
+ */
+ public void run(HookContext hookContext) throws Exception {
+ SessionState ss = SessionState.get();
+
+ if ((current_cluster == null) || (ss == null)) {
+ return;
+ }
+
+ HiveOperation commandType = ss.getHiveOperation();
+
+ // This check is only for alter table
+ if (!((commandType == HiveOperation.ALTERTABLE_ADDCOLS) ||
+ (commandType == HiveOperation.ALTERTABLE_REPLACECOLS) ||
+ (commandType == HiveOperation.ALTERTABLE_RENAMECOL) ||
+ (commandType == HiveOperation.ALTERTABLE_RENAMEPART) ||
+ (commandType == HiveOperation.ALTERTABLE_RENAME) ||
+ (commandType == HiveOperation.ALTERTABLE_PROPERTIES) ||
+ (commandType == HiveOperation.ALTERTABLE_SERIALIZER) ||
+ (commandType == HiveOperation.ALTERTABLE_SERDEPROPERTIES) ||
+ (commandType == HiveOperation.ALTERTABLE_CLUSTER_SORT) ||
+ (commandType == HiveOperation.ALTERTABLE_FILEFORMAT))) {
+ return;
+ }
+
+ // If the creation cluster is being modified to be the current cluster the alter should not be
+ // restricted
+ if (commandType == HiveOperation.ALTERTABLE_PROPERTIES) {
+ Map<String, String> newProps =
+ ((DDLWork)(hookContext.getQueryPlan().getRootTasks().get(0).getWork()))
+ .getAlterTblDesc().getProps();
+ if (newProps.containsKey(HookUtils.TABLE_CREATION_CLUSTER) &&
+ (newProps.get(HookUtils.TABLE_CREATION_CLUSTER).equals(current_cluster))) {
+ return;
+ }
+ }
+
+ Set<WriteEntity> outputs = hookContext.getOutputs();
+ for (WriteEntity output : outputs) {
+ Table table = output.getT();
+ if (table != null) {
+ String tableCreationCluster = table.getProperty(HookUtils.TABLE_CREATION_CLUSTER);
+ if (tableCreationCluster != null &&
+ !tableCreationCluster.isEmpty() &&
+ !tableCreationCluster.equals(current_cluster)) {
+ String exception = "Table " + table.getTableName() + " cannot be modified.";
+ exception += " Table's cluster is " + tableCreationCluster + ",";
+ exception += "whereas current package is " + current_cluster;
+ throw new Exception(exception);
+ }
+ }
+ }
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Implementation of a pre execute hook that checks whether
+ * a partition is archived or not and also sets that query
+ * time for the partition.
+ */
+public class ArchiverHook implements PreExecute {
+
+ private static final String ARCHIVE_FLAG = "archiveFlag";
+ private static final String LAST_QUERY_TIME = "lastQueryTime";
+
+ static final private Log LOG = LogFactory.getLog("hive.ql.hooks.ArchiverHook");
+
+ /**
+ * The metastore client.
+ */
+ private HiveMetaStoreClient ms;
+
+ /**
+ * The archiver hook constructor.
+ */
+ public ArchiverHook() throws Exception {
+ ms = new HiveMetaStoreClient(new HiveConf(this.getClass()));
+ }
+
+ private Map<String, String> modifyParams(Map<String, String> old_map, String key, String value) {
+ Map<String, String> new_map = old_map;
+ if (new_map == null)
+ new_map = new LinkedHashMap<String, String>();
+ new_map.put(key, value);
+ return new_map;
+ }
+
+ private boolean setLastQueryTime(Table t) throws Exception {
+ Map<String, String> old_map = t.getParameters();
+ if (old_map != null) {
+ String timeStr = old_map.get(LAST_QUERY_TIME);
+ if (timeStr != null) {
+ long time = Long.parseLong(timeStr);
+ long cur_time = System.currentTimeMillis();
+ if (cur_time - time < 1*60*60*1000) {
+ // lastQueryTime was recently set
+ return false;
+ }
+ }
+ }
+ t.setParameters(modifyParams(old_map, LAST_QUERY_TIME,
+ Long.toString(System.currentTimeMillis())));
+ return true;
+ }
+
+ private boolean setArchiveFlag(Table t) {
+ Map<String, String> old_map = t.getParameters();
+ if (old_map != null) {
+ String archF = old_map.get(ARCHIVE_FLAG);
+ if (archF != null) {
+ if(archF.equals("false")) {
+ return false;
+ }
+ }
+ }
+ t.setParameters(modifyParams(t.getParameters(), ARCHIVE_FLAG, "false"));
+ return true;
+ }
+
+ private boolean setLastQueryTime(Partition p) throws Exception {
+ Map<String, String> old_map = p.getParameters();
+ if (old_map != null) {
+ String timeStr = old_map.get(LAST_QUERY_TIME);
+ if (timeStr != null) {
+ long time = Long.parseLong(timeStr);
+ long cur_time = System.currentTimeMillis();
+ if (cur_time - time < 1*60*60*1000) {
+ // lastQueryTime was recently set
+ return false;
+ }
+ }
+ }
+ p.setParameters(modifyParams(old_map, LAST_QUERY_TIME,
+ Long.toString(System.currentTimeMillis())));
+ return true;
+ }
+
+ private boolean setArchiveFlag(Partition p) {
+ Map<String, String> old_map = p.getParameters();
+ if (old_map != null) {
+ String archF = old_map.get(ARCHIVE_FLAG);
+ if (archF != null) {
+ if(archF.equals("false")) {
+ return false;
+ }
+ }
+ }
+ p.setParameters(modifyParams(p.getParameters(), ARCHIVE_FLAG, "false"));
+ return true;
+ }
+
+ public void run(SessionState sess, Set<ReadEntity> inputs,
+ Set<WriteEntity> outputs, UserGroupInformation ugi)
+ throws Exception {
+
+ //Go over the input paths and check if they are archived or not
+ for(ReadEntity re: inputs) {
+ boolean isArchived = false;
+ if (re.getParameters() != null) {
+ String archF = re.getParameters().get(ARCHIVE_FLAG);
+ if (archF != null) {
+ isArchived = archF.equals("true");
+ }
+ }
+
+ if (isArchived)
+ throw new Exception("Path: " + re.getLocation().toString() + " needs to be unarchived.");
+
+ // Set the last query time
+ ReadEntity.Type typ = re.getType();
+ switch(typ) {
+ case TABLE:
+ Table t = re.getTable().getTTable();
+ if(setLastQueryTime(t)) {
+ LOG.debug("Setting LastQueryTime for table " + re);
+ ms.alter_table(MetaStoreUtils.DEFAULT_DATABASE_NAME, t.getTableName(), t);
+ }
+ break;
+ case PARTITION:
+ Partition p = re.getPartition().getTPartition();
+ if (setLastQueryTime(p)) {
+ LOG.debug("Setting LastQueryTime for partition " + re);
+ ms.alter_partition(MetaStoreUtils.DEFAULT_DATABASE_NAME, p.getTableName(), p);
+ }
+ break;
+ default:
+ throw new Exception("Unknown type for input: " + re.toString());
+ }
+ }
+
+ // Go over the write paths and set the archived flag to false
+ for(WriteEntity we: outputs) {
+ WriteEntity.Type typ = we.getType();
+ boolean q, a;
+
+ switch(typ) {
+ case TABLE:
+ Table t = we.getTable().getTTable();
+ q = setLastQueryTime(t);
+ a = setArchiveFlag(t);
+ if(q || a) {
+ LOG.debug("Altering dest table for archiver " + we);
+ ms.alter_table(MetaStoreUtils.DEFAULT_DATABASE_NAME, t.getTableName(), t);
+ }
+ break;
+ case PARTITION:
+ Partition p = we.getPartition().getTPartition();
+ q = setLastQueryTime(p);
+ a = setArchiveFlag(p);
+ if(q || a) {
+ if (ms.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ p.getTableName(), p.getValues()) != null) {
+ LOG.debug("Altering dest partition for archiver " + we);
+ ms.alter_partition(MetaStoreUtils.DEFAULT_DATABASE_NAME, p.getTableName(), p);
+ }
+ }
+ break;
+ case DFS_DIR:
+ case LOCAL_DIR:
+ break;
+ default:
+ throw new Exception("Unknown type for output: " + we.toString());
+ }
+ }
+ }
+
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskRunner;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Implementation of a post execute hook that checks whether a partition is
+ * archived or not and also sets that query time for the partition.
+ */
+public class AuditJoinHook implements ExecuteWithHookContext {
+ static final private Log LOG = LogFactory
+ .getLog("hive.ql.hooks.AuditJoinHook");
+
+ ConnectionUrlFactory urlFactory = null;
+
+ public AuditJoinHook() throws Exception {
+ HiveConf conf = new HiveConf(AuditJoinHook.class);
+ urlFactory = HookUtils.getUrlFactory(conf,
+ FBHiveConf.CONNECTION_FACTORY,
+ FBHiveConf.AUDIT_CONNECTION_FACTORY,
+ FBHiveConf.AUDIT_MYSQL_TIER_VAR_NAME,
+ FBHiveConf.AUDIT_HOST_DATABASE_VAR_NAME);
+ }
+
+ public void run(HookContext hookContext) throws Exception {
+ HiveConf conf = hookContext.getConf();
+ boolean enableConvert = HiveConf.getBoolVar(conf,
+ HiveConf.ConfVars.HIVECONVERTJOIN);
+ if (!enableConvert) {
+ return;
+ }
+ String command = StringEscapeUtils.escapeJava(SessionState.get()
+ .getCmd());
+ QueryPlan plan = hookContext.getQueryPlan();
+ String queryID = StringEscapeUtils.escapeJava(plan.getQueryId());
+ // String query = SessionState.get().getCmd();
+
+ int convertedMapJoin = 0;
+ int commonJoin = 0;
+ int backupCommonJoin = 0;
+ int convertedLocalMapJoin = 0;
+ int localMapJoin = 0;
+
+ List<TaskRunner> list = hookContext.getCompleteTaskList();
+ for (TaskRunner tskRunner : list) {
+ Task tsk = tskRunner.getTask();
+ int tag = tsk.getTaskTag();
+ switch (tag) {
+ case Task.COMMON_JOIN:
+ commonJoin++;
+ break;
+ case Task.CONVERTED_LOCAL_MAPJOIN:
+ convertedLocalMapJoin++;
+ break;
+ case Task.CONVERTED_MAPJOIN:
+ convertedMapJoin++;
+ break;
+ case Task.BACKUP_COMMON_JOIN:
+ backupCommonJoin++;
+ break;
+ case Task.LOCAL_MAPJOIN:
+ localMapJoin++;
+ break;
+ }
+ }
+
+ // nothing to do
+ if ((convertedMapJoin == 0) &&
+ (commonJoin == 0) &&
+ (backupCommonJoin == 0) &&
+ (convertedLocalMapJoin == 0)
+ && (localMapJoin == 0)) {
+ return;
+ }
+
+ ArrayList<Object> sqlParams = new ArrayList<Object>();
+ sqlParams.add(StringEscapeUtils.escapeJava(command));
+ sqlParams.add(StringEscapeUtils.escapeJava(queryID));
+ sqlParams.add(new Integer(convertedLocalMapJoin));
+ sqlParams.add(new Integer(convertedMapJoin));
+ sqlParams.add(new Integer(localMapJoin));
+ sqlParams.add(new Integer(commonJoin));
+ sqlParams.add(new Integer(backupCommonJoin));
+
+ String sql = "insert into audit_join set " +
+ " command = ?, query_id = ?, converted_local_mapjoin = ?, converted_map_join = ?," +
+ " local_mapjoin = ?, common_join = ?, backup_common_join = ?";
+
+ if (urlFactory == null) {
+ urlFactory = HookUtils.getUrlFactory(
+ conf,
+ FBHiveConf.CONNECTION_FACTORY,
+ FBHiveConf.AUDIT_CONNECTION_FACTORY,
+ FBHiveConf.AUDIT_MYSQL_TIER_VAR_NAME,
+ FBHiveConf.AUDIT_HOST_DATABASE_VAR_NAME);
+ if (urlFactory == null) {
+ throw new RuntimeException("DB parameters not set!");
+ }
+ }
+
+ HookUtils.runInsert(conf,
+ urlFactory, sql, sqlParams, HookUtils
+ .getSqlNumRetry(conf));
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskRunner;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Implementation of a post execute hook that checks whether a partition is
+ * archived or not and also sets that query time for the partition.
+ */
+public class AuditLocalModeHook implements ExecuteWithHookContext {
+ static final private Log LOG = LogFactory
+ .getLog("hive.ql.hooks.AuditLocalModeHook");
+ final static String MYSQL_TIER_VAR_NAME = "fbhive.audit.mysql.tier";
+ final static String HOST_DATABASE_VAR_NAME = "fbhive.audit.mysql";
+
+ ConnectionUrlFactory urlFactory = null;
+
+ public AuditLocalModeHook() throws Exception {
+ HiveConf conf = new HiveConf(AuditLocalModeHook.class);
+ urlFactory = HookUtils.getUrlFactory(conf,
+ FBHiveConf.CONNECTION_FACTORY,
+ FBHiveConf.AUDIT_CONNECTION_FACTORY,
+ FBHiveConf.AUDIT_MYSQL_TIER_VAR_NAME,
+ FBHiveConf.AUDIT_HOST_DATABASE_VAR_NAME);
+ }
+
+ public void run(HookContext hookContext) throws Exception {
+ HiveConf conf = hookContext.getConf();
+ String command = StringEscapeUtils.escapeJava(SessionState.get().getCmd());
+ QueryPlan plan = hookContext.getQueryPlan();
+ String queryID = StringEscapeUtils.escapeJava(plan.getQueryId());
+ int numLocalModeTask = 0;
+ int numMapRedTask = 0;
+ int numTask = 0;
+ List<TaskRunner> list = hookContext.getCompleteTaskList();
+ numTask = list.size();
+ for (TaskRunner tskRunner : list) {
+ Task tsk = tskRunner.getTask();
+ if(tsk.isMapRedTask()){
+ if(tsk.isLocalMode()) {
+ numLocalModeTask++;
+ }
+ numMapRedTask++;
+ }
+ }
+ if(numLocalModeTask == 0){
+ return ;
+ }
+ ArrayList<Object> sqlParams = new ArrayList<Object>();
+ sqlParams.add(StringEscapeUtils.escapeJava(command));
+ sqlParams.add(StringEscapeUtils.escapeJava(queryID));
+ sqlParams.add(new Integer(numTask));
+ sqlParams.add(new Integer(numMapRedTask));
+ sqlParams.add(new Integer(numLocalModeTask));
+
+ String sql = "insert into audit_local set " +
+ " command = ?, query_id = ?, num_tasks = ?, num_mapred_tasks = ?," +
+ " num_local_mapred_tasks = ?";
+ HookUtils.runInsert(conf, urlFactory, sql, sqlParams, HookUtils
+ .getSqlNumRetry(conf));
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+/**
+ * Seperating out some functionality for so that Hive1 can share code.
+ */
+public class BaseReplicationHook {
+ static final private Log LOG = LogFactory.getLog("hive.ql.hooks.BaseReplicationHook");
+
+ protected ConnectionUrlFactory urlFactory = null;
+ HiveConf conf = null;
+
+ public static ConnectionUrlFactory getReplicationMySqlUrl() {
+ HiveConf conf = new HiveConf(BaseReplicationHook.class);
+ return HookUtils.getUrlFactory(conf,
+ FBHiveConf.CONNECTION_FACTORY,
+ FBHiveConf.REPLICATION_CONNECTION_FACTORY,
+ FBHiveConf.REPLICATION_MYSQL_TIER_VAR_NAME,
+ FBHiveConf.REPLICATION_HOST_DATABASE_VAR_NAME);
+ }
+
+ public BaseReplicationHook() throws Exception {
+ urlFactory = getReplicationMySqlUrl();
+ conf = new HiveConf(BaseReplicationHook.class);
+ }
+
+ /**
+ * Simplified call used by hive1 to insert into the audit log
+ *
+ * @param command
+ * @param commandType
+ * @param inputs
+ * @param outputs
+ * @param userInfo
+ * @throws Exception
+ */
+ public void run(String command, String commandType, String inputs,
+ String outputs, String userInfo) throws Exception {
+ ArrayList<Object> sqlParams = new ArrayList<Object>();
+ sqlParams.add(StringEscapeUtils.escapeJava(commandType));
+ sqlParams.add(StringEscapeUtils.escapeJava(inputs));
+ sqlParams.add(outputs);
+ sqlParams.add(StringEscapeUtils.escapeJava(userInfo));
+
+ String sql = "insert into snc1_command_log set command_type = ?, " +
+ "inputs = ?, outputs = ?, user_info = ?";
+ if (conf == null) {
+ conf = new HiveConf(BaseReplicationHook.class);
+ }
+ HookUtils.runInsert(conf, urlFactory, sql, sqlParams);
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.Set;
+
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+/**
+ * Implementation of a pre execute hook that checks whether
+ * a partition is archived or not
+ */
+public class CheckArchivedDataHook {
+
+ private static final String ARCHIVE_FLAG = "archivedFlag";
+ final static String DISABLE_CHECK_ARCHIVAL_HOOK = "fbhive.disable.checkArchival.hook";
+
+ public static class PreExec implements PreExecute {
+
+ public void run(SessionState sess, Set<ReadEntity> inputs,
+ Set<WriteEntity> outputs, UserGroupInformation ugi)
+ throws Exception {
+
+ // Did the user explicitly ask to disable the hook
+ HiveConf conf = sess.getConf();
+ String disableArch = conf.get(DISABLE_CHECK_ARCHIVAL_HOOK);
+ if ((disableArch != null) && (disableArch.compareToIgnoreCase("false") == 0)) {
+ return;
+ }
+
+ //Go over the input paths and check if they are archived or not
+ for(ReadEntity re: inputs) {
+ boolean isArchived = false;
+ if (re.getParameters() != null) {
+ String archF = re.getParameters().get(ARCHIVE_FLAG);
+ if (archF != null) {
+ isArchived = archF.equals("true");
+ if (isArchived)
+ throw new Exception("Path: " + re.getLocation().toString() +
+ " needs to be unarchived.");
+ }
+ }
+ }
+ }
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.DDLTask;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+
+
+/**
+ * Implementation of a pre execute hook that checks the table RETENTION is set.
+ */
+public class CheckRetentionsHook {
+
+ private static final Log LOG = LogFactory.getLog(CheckRetentionsHook.class.getName());
+ private static HiveConf conf;
+
+ // If warningOnly = true, we print out some warnnings without fail
+ // the CREATE TABLE DDL.
+ private static boolean warningOnly = false;
+
+ // required table parameters
+ private static final String RETENTION_FLAG = "RETENTION";
+ private static final String RETENTION_PLATINUM_FLAG = "RETENTION_PLATINUM";
+
+ // wiki page URL that explains the policies
+ private static final String wikiURL =
+ "https://www.intern.facebook.com/intern/wiki/index.php/Data/Hive/" +
+ "Retention_on_new_tables";
+
+ private static String retentionKey = null;
+
+ private static String ErrMsg(String str) {
+ return str + "\n Here's how to add retention: " + wikiURL;
+ }
+
+ public static class PreExec implements ExecuteWithHookContext {
+
+ public void run(HookContext hookContext)
+ throws Exception {
+
+ assert(hookContext.getHookType() == HookContext.HookType.PRE_EXEC_HOOK);
+
+ SessionState sess = SessionState.get();
+ Set<ReadEntity> inputs = hookContext.getInputs();
+ Set<WriteEntity> outputs = hookContext.getOutputs();
+ UserGroupInformation ugi = hookContext.getUgi();
+ conf = sess.getConf();
+
+ warningOnly = conf.getBoolean(FBHiveConf.NO_RETENTION_WARNING_ONLY, true);
+
+ // figure out if we are on silver or platinum
+ String whDir = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE);
+ if (whDir == null) {
+ throw new Exception(ErrMsg("Cannot determine which cluster this query is running on: " +
+ "hive.metastore.warehouse.dir is not set!"));
+ }
+
+ Path p = new Path(whDir);
+ String hostName = p.toUri().getHost();
+
+ if (hostName.equals(conf.get(FBHiveConf.FBHIVE_SILVER_DFS_PREFIX)) ||
+ hostName.equals(conf.get(FBHiveConf.FBHIVE_SILVER_DFS_PREFIX2)) ||
+ hostName.equals(conf.get(FBHiveConf.FBHIVE_SILVER_DFS_PREFIX3))) {
+ retentionKey = RETENTION_FLAG;
+ } else if (hostName.equals(conf.get(FBHiveConf.FBHIVE_PLATINUM_DFS_PREFIX))) {
+ retentionKey = RETENTION_PLATINUM_FLAG;
+ } else {
+ throw new Exception(ErrMsg("Cannot determine which cluster this query is running on: " +
+ "hive.metastore.warehouse.dir=" + whDir +
+ "; does not seems to belong to either silver or platinum!"));
+ }
+
+ Set<Task<? extends Serializable>> tasks = new HashSet<Task<? extends Serializable>>();
+ getReachableTasks(tasks, hookContext.getQueryPlan());
+
+ for (Task task: tasks) {
+ if (task instanceof DDLTask) {
+ DDLWork work = (DDLWork) task.getWork();
+ if (work.getCreateTblDesc() != null) {
+ checkRetention(work.getCreateTblDesc(), retentionKey);
+ }
+ }
+ }
+ }
+
+ private void getReachableTasks(Set<Task<? extends Serializable>> tasks, QueryPlan qp) {
+ ArrayList<Task<? extends Serializable>> rootTasks = qp.getRootTasks();
+ for (Task<? extends Serializable> task: rootTasks) {
+ getReachableTasks(tasks, task);
+ }
+ }
+
+ /**
+ * Recursively traverse the task dependence tree and gather all tasks into
+ * the set.
+ */
+ private void getReachableTasks(Set<Task<? extends Serializable>> tasks,
+ Task<? extends Serializable> rootTask) {
+ if (!tasks.contains(rootTask)) {
+ tasks.add(rootTask);
+ if (rootTask.getDependentTasks() != null) {
+ for (Task<? extends Serializable> child: rootTask.getDependentTasks()) {
+ getReachableTasks(tasks, child);
+ }
+ }
+ }
+ }
+
+ private void warnOrFail(boolean warning, String mesg) throws Exception {
+ if (warning) {
+ // shout loud on stderr!
+ System.err.println("\n ----------");
+ System.err.println("| WARNING: | ");
+ System.err.println(" ----------");
+ System.err.println(" This command does NOT comply with the RETENTION " +
+ "policies. This command will fail in the near future. \n" +
+ mesg);
+ } else {
+ throw new Exception(mesg);
+ }
+ }
+
+
+ /**
+ * Check if the CREATE TABLE statement has retention and data growth
+ * estimation set. If not throw an exception.
+ */
+ private void checkRetention(CreateTableDesc desc, String retentionKey)
+ throws Exception {
+
+ // exclude EXTERNAL tables
+ if (desc.isExternal()) {
+ return;
+ }
+
+ // TODO: remove this whenever it becomes feasible
+ // exclude table name starts with tmp, temp, or test: tmp tables should be set
+ // default retention
+ if (desc.getTableName().startsWith("tmp") ||
+ desc.getTableName().startsWith("temp") ||
+ desc.getTableName().startsWith("test")) {
+ return;
+ }
+
+ // check if table already exists
+ if (tableExists(desc.getTableName())) {
+ return;
+ }
+
+ String tableNeedsRetention = "Newly created tables have to have " + retentionKey +
+ " set unless the table name has one of the prefixes \"tmp\", \"temp\", or \"test\".";
+
+ String tableRetentionFormat = "The value of the " + retentionKey + " parameter must be an " +
+ "integer greater than or equal to -1, i.e. -1,0,1,...";
+
+ // check 'RETENTION' parameter exists
+ String retentionValue = "";
+ if (desc.getTblProps() == null ||
+ (retentionValue = desc.getTblProps().get(retentionKey)) == null) {
+ warnOrFail(warningOnly, ErrMsg("Table " + desc.getTableName() + " does not have " +
+ retentionKey + " parameter set. "
+ + tableNeedsRetention + " " + tableRetentionFormat));
+ return;
+ }
+
+ // check 'RETENTION' parameter is set to a value in the range -1,0,1,...
+ int retentionIntValue;
+ try {
+ retentionIntValue = Integer.parseInt(retentionValue);
+ } catch (Exception e) {
+ // retentionValue is not a valid integer, set retentionIntValue to an invalid value
+ retentionIntValue = Integer.MIN_VALUE;
+ }
+
+ if (retentionIntValue < -1) {
+ warnOrFail(warningOnly, ErrMsg("Table " + desc.getTableName() + " has an invalid value " +
+ retentionValue + " for the parameter " + retentionKey + ". " +
+ tableRetentionFormat + " " + tableNeedsRetention));
+ }
+ }
+
+ private boolean tableExists(String tabName) throws Exception {
+ Hive db = Hive.get(conf);
+ Table table = db.getTable("default", tabName, false);
+ return table != null;
+ }
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java Tue May 1 22:52:38 2012
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ *
+ * This factory creates the connection URL from the supplied configuration.
+ *
+ */
+
+public class ConfUrlFactory implements ConnectionUrlFactory {
+
+ HiveConf conf = null;
+ String confVarName = null;
+ public ConfUrlFactory() {
+ this(new HiveConf(ConfUrlFactory.class), "");
+ }
+
+ public ConfUrlFactory(String confVarName) {
+ this(new HiveConf(ConfUrlFactory.class), confVarName);
+ }
+
+ public ConfUrlFactory(HiveConf conf, String confVarName) {
+ this.conf = conf;
+ this.confVarName = confVarName;
+ }
+
+ public boolean init(Configuration hconf) {
+ this.conf = (HiveConf)hconf;
+ return true;
+ }
+
+ @Override
+ public void init(String param1Name, String param2Name) {
+ this.confVarName = param1Name;
+ }
+
+ @Override
+ public String getUrl() throws Exception {
+ String dbstr = conf.get(confVarName);
+ String[] hostDatabases = dbstr.split(":");
+ return "jdbc:mysql://" + hostDatabases[0] + "/" + hostDatabases[1];
+ }
+
+ @Override
+ public String getUrl(boolean isWrite) throws Exception {
+ return getUrl();
+ }
+
+ @Override
+ public String getValue(String param1, String param2) throws Exception {
+ return null;
+ }
+
+ @Override
+ public void updateProps(String param1, String param2, String param3) {
+ return;
+ }
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java Tue May 1 22:52:38 2012
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+/**
+ *
+ * Classes implementing this interface create JDBC connection URL's.
+ * This can also be used to store a parameters array
+ */
+public interface ConnectionUrlFactory {
+
+ public void init(String param1Name, String param2Name);
+
+ /**
+ * @return the JDBC connection URL
+ * @throws Exception
+ */
+ String getUrl() throws Exception;
+
+ String getUrl(boolean isWrite) throws Exception;
+
+ String getValue(String param1, String param2) throws Exception;
+ void updateProps(String param1, String param2, String param3);
+}
Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java Tue May 1 22:52:38 2012
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
+import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+
+/**
+ * Implementation of a pre execute hook that is used to change
+ * the location of the DFS.
+ * This is only applicable to new tables - this can be used to
+ * eventually spread the load evenly on more than 1 DFS.
+ */
+public class CreateTableChangeDFSHook implements ExecuteWithHookContext {
+ static final private Log LOG = LogFactory.getLog(CreateTableChangeDFSHook.class.getName());
+
+ public void run(HookContext hookContext) throws Exception {
+ assert(hookContext.getHookType() == HookContext.HookType.PRE_EXEC_HOOK);
+
+ QueryPlan queryPlan = hookContext.getQueryPlan();
+
+ // This change is only needed when a new table is being created
+ ArrayList<Task<? extends Serializable>> rootTasks = queryPlan.getRootTasks();
+
+ if ((rootTasks == null) || (rootTasks.size() != 1)) {
+ return;
+ }
+
+ Task<? extends Serializable> tsk = rootTasks.get(0);
+
+ if (!(tsk.getWork() instanceof DDLWork)) {
+ return;
+ }
+
+ HiveConf conf = hookContext.getConf();
+ DDLWork ddlWork = (DDLWork)tsk.getWork();
+
+ float pubPercent = conf.getFloat(FBHiveConf.ENABLE_PARTIAL_CHANGEDFS, 0);
+
+ // if pubPercent == 0, make sure it returns.
+ if (!HookUtils.rollDice(pubPercent)) {
+ return;
+ }
+
+ String newDir = conf.get(FBHiveConf.SECONDARYMETASTOREWAREHOUSE);
+
+ if (ddlWork.getCreateTblDesc() != null) {
+ CreateTableDesc crtTblDesc = ddlWork.getCreateTblDesc();
+ // The user has already specified the location
+ if (crtTblDesc.getLocation() != null) {
+ return;
+ }
+
+ // This is only for tmp tables right now
+ if ((crtTblDesc.getTableName() == null) ||
+ ((!crtTblDesc.getTableName().startsWith("tmp_")) &&
+ (!crtTblDesc.getTableName().startsWith("temp_")))) {
+ return;
+ }
+
+ String locn =
+ (new Warehouse(conf)).getTablePath(newDir, crtTblDesc.getTableName()).toString();
+ crtTblDesc.setLocation(locn);
+ LOG.info("change location for table " + crtTblDesc.getTableName());
+ return;
+ }
+
+ if (ddlWork.getCreateTblLikeDesc() != null) {
+ CreateTableLikeDesc crtTblLikeDesc = ddlWork.getCreateTblLikeDesc();
+ // The user has already specified the location
+ if (crtTblLikeDesc.getLocation() != null) {
+ return;
+ }
+
+ // This is only for tmp tables right now
+ if ((crtTblLikeDesc.getTableName() == null) ||
+ ((!crtTblLikeDesc.getTableName().startsWith("tmp_")) &&
+ (!crtTblLikeDesc.getTableName().startsWith("temp_")))) {
+ return;
+ }
+
+ String locn =
+ (new Warehouse(conf)).getTablePath(newDir, crtTblLikeDesc.getTableName()).toString();
+ crtTblLikeDesc.setLocation(locn);
+ LOG.info("change location for table " + crtTblLikeDesc.getTableName());
+ return;
+ }
+ }
+}