You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/03/16 07:21:17 UTC
[1/2] incubator-lens git commit: LENS-376 ColumnarSQLRewriter always
ends up making metastore lookup (Amareshwari via jdhok)
Repository: incubator-lens
Updated Branches:
refs/heads/master 6a5e5111c -> 8578c8fc3
LENS-376 ColumnarSQLRewriter always ends up making metastore lookup (Amareshwari via jdhok)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d2d80790
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d2d80790
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d2d80790
Branch: refs/heads/master
Commit: d2d80790d085d7eaaf0df552dbbfc4b862b382e4
Parents: 6a5e511
Author: jdhok <ja...@inmobi.com>
Authored: Mon Mar 16 11:41:54 2015 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Mon Mar 16 11:41:54 2015 +0530
----------------------------------------------------------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 26 +++++++++-----------
.../driver/jdbc/TestColumnarSQLRewriter.java | 10 ++++----
2 files changed, 17 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d2d80790/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 28dd662..df01ba7 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
import java.util.*;
import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
import org.apache.lens.cube.parse.CubeSemanticAnalyzer;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.server.api.LensConfConstants;
@@ -33,7 +34,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.*;
@@ -784,7 +784,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* @throws HiveException
*/
- public void replaceAliasInAST() throws HiveException {
+ public void replaceAliasInAST() {
updateAliasFromAST(fromAST);
if (fromTree != null) {
replaceAlias(fromAST);
@@ -819,9 +819,9 @@ public class ColumnarSQLRewriter implements QueryRewriter {
/**
* Builds the query.
*
- * @throws HiveException the hive exception
+ * @throws SemanticException
*/
- public void buildQuery(HiveConf queryConf) throws HiveException {
+ public void buildQuery(HiveConf queryConf) throws SemanticException {
analyzeInternal();
replaceWithUnderlyingStorage(queryConf, fromAST);
replaceAliasInAST();
@@ -1064,8 +1064,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
throw new LensException(e);
} catch (SemanticException e) {
throw new LensException(e);
- } catch (HiveException e) {
- throw new LensException(e);
}
return queryReplacedUdf;
}
@@ -1092,8 +1090,9 @@ public class ColumnarSQLRewriter implements QueryRewriter {
ASTNode dbIdentifier = (ASTNode) tree.getChild(0);
ASTNode tableIdentifier = (ASTNode) tree.getChild(1);
String lensTable = dbIdentifier.getText() + "." + tableIdentifier.getText();
- String table = getUnderlyingTableName(queryConf, lensTable);
- String db = getUnderlyingDBName(queryConf, lensTable);
+ Table tbl = CubeMetastoreClient.getInstance(queryConf).getHiveTable(lensTable);
+ String table = getUnderlyingTableName(queryConf, tbl);
+ String db = getUnderlyingDBName(queryConf, tbl);
// Replace both table and db names
if ("default".equalsIgnoreCase(db)) {
@@ -1109,14 +1108,15 @@ public class ColumnarSQLRewriter implements QueryRewriter {
} else {
ASTNode tableIdentifier = (ASTNode) tree.getChild(0);
String lensTable = tableIdentifier.getText();
- String table = getUnderlyingTableName(queryConf, lensTable);
+ Table tbl = CubeMetastoreClient.getInstance(queryConf).getHiveTable(lensTable);
+ String table = getUnderlyingTableName(queryConf, tbl);
// Replace table name
if (StringUtils.isNotBlank(table)) {
tableIdentifier.getToken().setText(table);
}
// Add db name as a new child
- String dbName = getUnderlyingDBName(queryConf, lensTable);
+ String dbName = getUnderlyingDBName(queryConf, tbl);
if (StringUtils.isNotBlank(dbName) && !"default".equalsIgnoreCase(dbName)) {
ASTNode dbIdentifier = new ASTNode(new CommonToken(HiveParser.Identifier, dbName));
dbIdentifier.setParent(tree);
@@ -1140,8 +1140,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* @return the underlying db name
* @throws HiveException the hive exception
*/
- String getUnderlyingDBName(HiveConf queryConf, String table) throws HiveException {
- Table tbl = Hive.get(queryConf).getTable(table);
+ String getUnderlyingDBName(HiveConf queryConf, Table tbl) throws HiveException {
return tbl == null ? null : tbl.getProperty(LensConfConstants.NATIVE_DB_NAME);
}
@@ -1152,8 +1151,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* @return the underlying table name
* @throws HiveException the hive exception
*/
- String getUnderlyingTableName(HiveConf queryConf, String table) throws HiveException {
- Table tbl = Hive.get(queryConf).getTable(table);
+ String getUnderlyingTableName(HiveConf queryConf, Table tbl) throws HiveException {
return tbl == null ? null : tbl.getProperty(LensConfConstants.NATIVE_TABLE_NAME);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d2d80790/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index f62170f..0bcbe7b 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -26,7 +26,6 @@ import java.net.URLClassLoader;
import java.util.*;
import org.apache.lens.api.LensException;
-import org.apache.lens.cube.metadata.MetastoreConstants;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.server.api.LensConfConstants;
@@ -795,14 +794,14 @@ public class TestColumnarSQLRewriter {
database.setName("mydb");
Hive.get(conf).createDatabase(database);
+ SessionState.get().setCurrentDatabase("mydb");
createTable(conf, "mydb", "mytable", "testDB", "testTable_1");
createTable(conf, "mydb", "mytable_2", "testDB", "testTable_2");
createTable(conf, "default", "mytable_3", "testDB", "testTable_3");
- String query = "SELECT * FROM mydb.mytable t1 JOIN mydb.mytable_2 t2 ON t1.t2id = t2.id "
- + " left outer join mytable_3 t3 on t2.t3id = t3.id " + "WHERE A = 100";
+ String query = "SELECT * FROM mydb.mytable t1 JOIN mytable_2 t2 ON t1.t2id = t2.id "
+ + " left outer join default.mytable_3 t3 on t2.t3id = t3.id " + "WHERE A = 100";
- queryConf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, false);
// Test fails without setting this class loader as now metastore lookup is done using queryConf
queryConf.setClassLoader(createTableClassLoader);
@@ -818,7 +817,7 @@ public class TestColumnarSQLRewriter {
// Rewrite
rewriter.replaceWithUnderlyingStorage(queryConf, rewriter.fromAST);
String joinTreeAfterRewrite = HQLParser.getString(rewriter.fromAST);
- System.out.println(joinTreeAfterRewrite);
+ System.out.println("joinTreeAfterRewrite:" + joinTreeAfterRewrite);
// Tests
assertTrue(joinTreeBeforeRewrite.contains("mydb"));
@@ -875,6 +874,7 @@ public class TestColumnarSQLRewriter {
Hive.get().dropTable("mydb", "mytable_4");
Hive.get().dropDatabase("mydb", true, true, true);
Hive.get().dropDatabase("examples", true, true, true);
+ SessionState.get().setCurrentDatabase("default");
}
/**
[2/2] incubator-lens git commit: LENS-407 Driver side rewrite
happening twice for execute call on JDBCDriver (Amareshwari via jdhok)
Posted by jd...@apache.org.
LENS-407 Driver side rewrite happening twice for execute call on JDBCDriver (Amareshwari via jdhok)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/8578c8fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/8578c8fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/8578c8fc
Branch: refs/heads/master
Commit: 8578c8fc310cf7ecaf1ce86e0a3626d5587abfe5
Parents: d2d8079
Author: jdhok <ja...@inmobi.com>
Authored: Mon Mar 16 11:46:31 2015 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Mon Mar 16 11:46:31 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 27 +++++++------
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 41 ++++++++++++++++----
.../server/api/query/AbstractQueryContext.java | 7 ++++
.../api/query/DriverSelectorQueryContext.java | 9 +++++
4 files changed, 65 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8578c8fc/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 6f9dd5b..1d68eb5 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -509,7 +509,14 @@ public class JDBCDriver implements LensDriver {
* @return the string
* @throws LensException the lens exception
*/
- protected String rewriteQuery(String query, Configuration conf) throws LensException {
+ protected String rewriteQuery(AbstractQueryContext ctx) throws LensException {
+ if (ctx.getFinalDriverQuery(this) != null) {
+ return ctx.getFinalDriverQuery(this);
+ }
+ String query = ctx.getDriverQuery(this);
+ Configuration driverQueryConf = ctx.getDriverConf(this);
+ MethodMetricsContext checkForAllowedQuery = MethodMetricsFactory.createMethodGauge(driverQueryConf, true,
+ CHECK_ALLOWED_QUERY);
// check if it is select query
try {
ASTNode ast = HQLParser.parseHQL(query);
@@ -526,12 +533,11 @@ public class JDBCDriver implements LensDriver {
} catch (ParseException e) {
throw new LensException(e);
}
+ checkForAllowedQuery.markSuccess();
QueryRewriter rewriter = getQueryRewriter();
- String rewrittenQuery = rewriter.rewrite(query, conf);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Query: " + query + " rewrittenQuery: " + rewrittenQuery);
- }
+ String rewrittenQuery = rewriter.rewrite(query, driverQueryConf);
+ ctx.setFinalDriverQuery(this, rewrittenQuery);
return rewrittenQuery;
}
@@ -555,6 +561,7 @@ public class JDBCDriver implements LensDriver {
private static final String VALIDATE_GAUGE = "validate-thru-prepare";
private static final String COLUMNAR_SQL_REWRITE_GAUGE = "columnar-sql-rewrite";
private static final String JDBC_PREPARE_GAUGE = "jdbc-prepare-statement";
+ private static final String CHECK_ALLOWED_QUERY = "jdbc-check-allowed-query";
@Override
public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
@@ -583,7 +590,7 @@ public class JDBCDriver implements LensDriver {
}
checkConfigured();
String explainQuery;
- String rewrittenQuery = rewriteQuery(explainCtx.getDriverQuery(this), explainCtx.getDriverConf(this));
+ String rewrittenQuery = rewriteQuery(explainCtx);
Configuration explainConf = new Configuration(explainCtx.getDriverConf(this));
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER,
false);
@@ -741,7 +748,7 @@ public class JDBCDriver implements LensDriver {
// Only create a prepared statement and then close it
MethodMetricsContext sqlRewriteGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
COLUMNAR_SQL_REWRITE_GAUGE);
- String rewrittenQuery = rewriteQuery(pContext.getDriverQuery(this), pContext.getDriverConf(this));
+ String rewrittenQuery = rewriteQuery(pContext);
sqlRewriteGauge.markSuccess();
MethodMetricsContext jdbcPrepareGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
JDBC_PREPARE_GAUGE);
@@ -827,8 +834,7 @@ public class JDBCDriver implements LensDriver {
public LensResultSet execute(QueryContext context) throws LensException {
checkConfigured();
- String rewrittenQuery = rewriteQuery(context.getSelectedDriverQuery(), context
- .getSelectedDriverConf());
+ String rewrittenQuery = rewriteQuery(context);
LOG.info("Execute " + context.getQueryHandle());
QueryResult result = executeInternal(context, rewrittenQuery);
return result.getLensResultSet(true);
@@ -864,8 +870,7 @@ public class JDBCDriver implements LensDriver {
checkConfigured();
// Always use the driver rewritten query not user query. Since the
// conf we are passing here is query context conf, we need to add jdbc xml in resource path
- String rewrittenQuery = rewriteQuery(context.getSelectedDriverQuery(), context.getDriverContext()
- .getSelectedDriverConf());
+ String rewrittenQuery = rewriteQuery(context);
JdbcQueryContext jdbcCtx = new JdbcQueryContext(context);
jdbcCtx.setRewrittenQuery(rewrittenQuery);
try {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8578c8fc/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index f3be3fb..912e201 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -119,7 +119,11 @@ public class TestJdbcDriver {
}
private QueryContext createQueryContext(final String query) throws LensException {
- QueryContext context = new QueryContext(query, "SA", new LensConf(), baseConf, drivers);
+ return createQueryContext(query, baseConf);
+ }
+
+ private QueryContext createQueryContext(final String query, Configuration conf) throws LensException {
+ QueryContext context = new QueryContext(query, "SA", new LensConf(), conf, drivers);
return context;
}
@@ -202,7 +206,7 @@ public class TestJdbcDriver {
Throwable th = null;
try {
- driver.rewriteQuery(query, baseConf);
+ driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
e.printStackTrace();
th = e;
@@ -213,7 +217,7 @@ public class TestJdbcDriver {
th = null;
try {
- driver.rewriteQuery(query, baseConf);
+ driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
e.printStackTrace();
th = e;
@@ -224,7 +228,7 @@ public class TestJdbcDriver {
th = null;
try {
- driver.rewriteQuery(query, baseConf);
+ driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
e.printStackTrace();
th = e;
@@ -235,7 +239,7 @@ public class TestJdbcDriver {
th = null;
try {
- driver.rewriteQuery(query, baseConf);
+ driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
e.printStackTrace();
th = e;
@@ -253,9 +257,12 @@ public class TestJdbcDriver {
createTable("estimate_test", driver.getEstimateConnection()); // Create table
insertData("estimate_test", driver.getEstimateConnection()); // Insert some data into table
String query1 = "SELECT * FROM estimate_test"; // Select query against existing table
- QueryCost cost = driver.estimate(createExplainContext(query1, baseConf));
+ ExplainQueryContext ctx = createExplainContext(query1, baseConf);
+ Assert.assertNull(ctx.getFinalDriverQuery(driver));
+ QueryCost cost = driver.estimate(ctx);
Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0);
Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0);
+ Assert.assertNotNull(ctx.getFinalDriverQuery(driver));
}
/**
@@ -292,7 +299,22 @@ public class TestJdbcDriver {
Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
"lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-columnar-sql-rewrite",
"lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-jdbc-prepare-statement",
- "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-thru-prepare")));
+ "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-thru-prepare",
+ "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-jdbc-check-allowed-query")));
+ }
+
+ @Test
+ public void testMetricsEnabled() throws Exception {
+ createTable("test_metrics", driver.getEstimateConnection()); // Create table
+ insertData("test_metrics", driver.getEstimateConnection()); // Insert some data into table
+ createTable("test_metrics"); // Create table
+ insertData("test_metrics"); // Insert some data into table
+ String query1 = "SELECT * FROM test_metrics"; // Select query against existing table
+ Configuration metricConf = new Configuration(baseConf);
+ metricConf.setBoolean(LensConfConstants.ENABLE_QUERY_METRICS, true);
+ QueryContext ctx = createQueryContext(query1, metricConf);
+ driver.estimate(ctx);
+ driver.execute(ctx);
}
/**
@@ -306,7 +328,10 @@ public class TestJdbcDriver {
insertData("explain_test"); // Insert some data into table
String query1 = "SELECT * FROM explain_test"; // Select query against existing table
String query2 = "SELECT * FROM explain_test1"; // Select query against non existing table
- driver.explain(createExplainContext(query1, baseConf));
+ ExplainQueryContext ctx = createExplainContext(query1, baseConf);
+ Assert.assertNull(ctx.getFinalDriverQuery(driver));
+ driver.explain(ctx);
+ Assert.assertNotNull(ctx.getFinalDriverQuery(driver));
try {
driver.explain(createExplainContext(query2, baseConf));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8578c8fc/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 432c863..0f68cb9 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -203,6 +203,9 @@ public abstract class AbstractQueryContext implements Serializable {
return driverContext.getDriverQuery(driver);
}
+ public String getFinalDriverQuery(LensDriver driver) {
+ return driverContext.getFinalDriverQuery(driver);
+ }
/**
* Get driver conf
*
@@ -317,4 +320,8 @@ public abstract class AbstractQueryContext implements Serializable {
public Exception getDriverRewriteError(LensDriver driver) {
return driverContext.driverQueryContextMap.get(driver).getDriverQueryRewriteError();
}
+
+ public void setFinalDriverQuery(LensDriver driver, String rewrittenQuery) {
+ driverContext.driverQueryContextMap.get(driver).setFinalDriverQuery(rewrittenQuery);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8578c8fc/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 94b5daf..088076e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -123,6 +123,10 @@ public class DriverSelectorQueryContext {
/** driver query */
protected String query;
+ @Getter
+ @Setter
+ /** final driver query - after driver rewrites the query*/
+ protected String finalDriverQuery;
}
/**
@@ -293,6 +297,11 @@ public class DriverSelectorQueryContext {
? driverQueryContextMap.get(driver).getQuery() : null;
}
+ public String getFinalDriverQuery(LensDriver driver) {
+ return driverQueryContextMap.get(driver) != null
+ ? driverQueryContextMap.get(driver).getFinalDriverQuery() : null;
+ }
+
public QueryCost getDriverQueryCost(LensDriver driver) {
return driverQueryContextMap.get(driver) != null
? driverQueryContextMap.get(driver).getDriverCost() : null;