You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2014/10/16 19:18:13 UTC
svn commit: r1632391 - in
/hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore:
MetaStoreDirectSql.java ObjectStore.java
Author: khorgath
Date: Thu Oct 16 17:18:13 2014
New Revision: 1632391
URL: http://svn.apache.org/r1632391
Log:
HIVE-7366 : getDatabase using direct sql (Sushanth Sowmyan, reviewed by Sergei Shelukhin)
Modified:
hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
Modified: hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1632391&r1=1632390&r2=1632391&view=diff
==============================================================================
--- hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Thu Oct 16 17:18:13 2014
@@ -45,10 +45,13 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
@@ -143,13 +146,18 @@ class MetaStoreDirectSql {
}
/**
- * See {@link #trySetAnsiQuotesForMysql()}.
+ * This function is intended to be called by functions before they put together a query
+ * Thus, any query-specific instantiation to be done from within the transaction is done
+ * here - for eg., for MySQL, we signal that we want to use ANSI SQL quoting behaviour
*/
- private void setAnsiQuotesForMysql() throws MetaException {
- try {
- trySetAnsiQuotesForMysql();
- } catch (SQLException sqlEx) {
- throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage());
+ private void doDbSpecificInitializationsBeforeQuery() throws MetaException {
+ if (isMySql){
+ try {
+ assert pm.currentTransaction().isActive(); // must be inside tx together with queries
+ trySetAnsiQuotesForMysql();
+ } catch (SQLException sqlEx) {
+ throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage());
+ }
}
}
@@ -171,6 +179,78 @@ class MetaStoreDirectSql {
}
}
+ public Database getDatabase(String dbName) throws MetaException{
+ Query queryDbSelector = null;
+ Query queryDbParams = null;
+ try {
+ dbName = dbName.toLowerCase();
+
+ doDbSpecificInitializationsBeforeQuery();
+
+ String queryTextDbSelector= "select "
+ + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", "
+ + "\"OWNER_NAME\", \"OWNER_TYPE\" "
+ + "FROM \"DBS\" where \"NAME\" = ? ";
+ Object[] params = new Object[] { dbName };
+ queryDbSelector = pm.newQuery("javax.jdo.query.SQL", queryTextDbSelector);
+
+ LOG.debug("getDatabase:query instantiated : " + queryTextDbSelector + " with param ["+params[0]+"]");
+
+ List<Object[]> sqlResult = (List<Object[]>)queryDbSelector.executeWithArray(params);
+ if ((sqlResult == null) || sqlResult.isEmpty()) {
+ LOG.debug("getDatabase:queryDbSelector ran, returned no/empty results, returning NoSuchObjectException");
+ throw new MetaException("There is no database named " + dbName);
+ }
+
+ assert(sqlResult.size() == 1);
+ if (sqlResult.get(0) == null){
+ LOG.debug("getDatabase:queryDbSelector ran, returned results, but the result entry was null, returning NoSuchObjectException");
+ throw new MetaException("There is no database named " + dbName);
+ }
+
+ Object[] dbline = sqlResult.get(0);
+ Long dbid = StatObjectConverter.extractSqlLong(dbline[0]);
+
+ String queryTextDbParams = "select \"PARAM_KEY\", \"PARAM_VALUE\" "
+ + " FROM \"DATABASE_PARAMS\" "
+ + " WHERE \"DB_ID\" = ? "
+ + " AND \"PARAM_KEY\" IS NOT NULL";
+ Object[] params2 = new Object[] { dbid };
+ queryDbParams = pm.newQuery("javax.jdo.query.SQL",queryTextDbParams);
+ LOG.debug("getDatabase:query2 instantiated : " + queryTextDbParams + " with param ["+params2[0]+"]");
+
+ Map<String,String> dbParams = new HashMap<String,String>();
+ List<Object[]> sqlResult2 = ensureList(queryDbParams.executeWithArray(params2));
+ if (!sqlResult2.isEmpty()){
+ for (Object[] line : sqlResult2){
+ dbParams.put(extractSqlString(line[0]),extractSqlString(line[1]));
+ }
+ }
+ LOG.debug("getDatabase: instantiating db object to return");
+ Database db = new Database();
+ db.setName(extractSqlString(dbline[1]));
+ db.setLocationUri(extractSqlString(dbline[2]));
+ db.setDescription(extractSqlString(dbline[3]));
+ db.setOwnerName(extractSqlString(dbline[4]));
+ String type = extractSqlString(dbline[5]);
+ db.setOwnerType((null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type));
+ db.setParameters(dbParams);
+ if (LOG.isDebugEnabled()){
+ LOG.debug("getDatabase: directsql returning db " + db.getName()
+ + " locn["+db.getLocationUri() +"] desc [" +db.getDescription()
+ + "] owner [" + db.getOwnerName() + "] ownertype ["+ db.getOwnerType() +"]");
+ }
+ return db;
+ } finally {
+ if (queryDbSelector != null){
+ queryDbSelector.closeAll();
+ }
+ if (queryDbParams != null){
+ queryDbParams.closeAll();
+ }
+ }
+ }
+
/**
* Gets partitions by using direct SQL queries.
* @param dbName Metastore db name.
@@ -260,10 +340,8 @@ class MetaStoreDirectSql {
tblName = tblName.toLowerCase();
// We have to be mindful of order during filtering if we are not returning all partitions.
String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : "";
- if (isMySql) {
- assert pm.currentTransaction().isActive();
- setAnsiQuotesForMysql(); // must be inside tx together with queries
- }
+
+ doDbSpecificInitializationsBeforeQuery();
// Get all simple fields for partitions and related objects, which we can map one-on-one.
// We will do this in 2 queries to use different existing indices for each one.
@@ -621,6 +699,11 @@ class MetaStoreDirectSql {
return ((Number)field).intValue();
}
+ private String extractSqlString(Object value) {
+ if (value == null) return null;
+ return value.toString();
+ }
+
private static String trimCommaList(StringBuilder sb) {
if (sb.length() > 0) {
sb.setLength(sb.length() - 1);
Modified: hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1632391&r1=1632390&r2=1632391&view=diff
==============================================================================
--- hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Oct 16 17:18:13 2014
@@ -523,6 +523,34 @@ public class ObjectStore implements RawS
@Override
public Database getDatabase(String name) throws NoSuchObjectException {
+ try {
+ return getDatabaseInternal(name);
+ } catch (MetaException e) {
+ // Signature restriction to NSOE, and NSOE being a flat exception prevents us from
+ // setting the cause of the NSOE as the MetaException. We should not lose the info
+ // we got here, but it's very likely that the MetaException is irrelevant and is
+ // actually an NSOE message, so we should log it and throw an NSOE with the msg.
+ LOG.warn("Got a MetaException trying to call getDatabase("
+ +name+"), returning NoSuchObjectException", e);
+ throw new NoSuchObjectException(e.getMessage());
+ }
+ }
+
+ public Database getDatabaseInternal(String name) throws MetaException, NoSuchObjectException {
+ return new GetDbHelper(name, null, true, true) {
+ @Override
+ protected Database getSqlResult(GetHelper<Database> ctx) throws MetaException {
+ return directSql.getDatabase(dbName);
+ }
+
+ @Override
+ protected Database getJdoResult(GetHelper<Database> ctx) throws MetaException, NoSuchObjectException {
+ return getJDODatabase(dbName);
+ }
+ }.run(false);
+ }
+
+ public Database getJDODatabase(String name) throws NoSuchObjectException {
MDatabase mdb = null;
boolean commited = false;
try {
@@ -2282,7 +2310,14 @@ public class ObjectStore implements RawS
assert allowSql || allowJdo;
this.allowJdo = allowJdo;
this.dbName = dbName.toLowerCase();
- this.tblName = tblName.toLowerCase();
+ if (tblName != null){
+ this.tblName = tblName.toLowerCase();
+ } else {
+ // tblName can be null in cases of Helper being used at a higher
+ // abstraction level, such as with datbases
+ this.tblName = null;
+ this.table = null;
+ }
this.doTrace = LOG.isDebugEnabled();
this.isInTxn = isActiveTransaction();
@@ -2331,7 +2366,7 @@ public class ObjectStore implements RawS
private void start(boolean initTable) throws MetaException, NoSuchObjectException {
start = doTrace ? System.nanoTime() : 0;
openTransaction();
- if (initTable) {
+ if (initTable && (tblName != null)) {
table = ensureGetTable(dbName, tblName);
}
}
@@ -2398,6 +2433,27 @@ public class ObjectStore implements RawS
}
}
+ private abstract class GetDbHelper extends GetHelper<Database> {
+ /**
+ * GetHelper for returning db info using directSql/JDO.
+ * Since this is a db-level call, tblName is ignored, and null is passed irrespective of what is passed in.
+ * @param dbName The Database Name
+ * @param tblName Placeholder param to match signature, always ignored.
+ * @param allowSql Whether or not we allow DirectSQL to perform this query.
+ * @param allowJdo Whether or not we allow ORM to perform this query.
+ * @throws MetaException
+ */
+ public GetDbHelper(
+ String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException {
+ super(dbName,null,allowSql,allowJdo);
+ }
+
+ @Override
+ protected String describeResult() {
+ return "db details for db " + dbName;
+ }
+ }
+
private abstract class GetStatHelper extends GetHelper<ColumnStatistics> {
public GetStatHelper(
String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException {