You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2016/10/03 11:26:09 UTC
lens git commit: LENS-1324 : Support more hooks in DriverQueryHook
Repository: lens
Updated Branches:
refs/heads/master 241603cf0 -> 08ce29785
LENS-1324 : Support more hooks in DriverQueryHook
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/08ce2978
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/08ce2978
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/08ce2978
Branch: refs/heads/master
Commit: 08ce29785f886b82b490f269a1ca4a6c6eff9eed
Parents: 241603c
Author: Puneet Gupta <pu...@apache.org>
Authored: Mon Oct 3 16:55:58 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Oct 3 16:55:58 2016 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 1 +
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 1 +
.../server/api/driver/AbstractLensDriver.java | 1 +
.../lens/server/api/driver/DriverQueryHook.java | 54 +++++++++++++++++---
.../server/api/driver/NoOpDriverQueryHook.java | 44 ++++++++++++++--
.../query/cost/FactPartitionBasedQueryCost.java | 5 ++
.../lens/server/api/driver/MockDriver.java | 1 +
.../server/api/user/MockDriverQueryHook.java | 34 +++++++++++-
.../server/query/QueryExecutionServiceImpl.java | 12 ++++-
.../lens/server/query/TestQueryService.java | 4 ++
10 files changed, 144 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 84d9933..0d8810f 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -383,6 +383,7 @@ public class HiveDriver extends AbstractLensDriver {
queryHook = driverConf.getClass(
HIVE_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
).newInstance();
+ queryHook.setDriver(this);
} catch (InstantiationException | IllegalAccessException e) {
throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 3bf5e8f..f805ec6 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -407,6 +407,7 @@ public class JDBCDriver extends AbstractLensDriver {
queryHook = this.conf.getClass(
JDBC_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
).newInstance();
+ queryHook.setDriver(this);
} catch (InstantiationException | IllegalAccessException e) {
throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
index 03079e2..e498479 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
@@ -55,6 +55,7 @@ public abstract class AbstractLensDriver implements LensDriver {
throw new LensException("Driver Type and Name can not be null or empty");
}
fullyQualifiedName = new StringBuilder(driverType).append(SEPARATOR).append(driverName).toString();
+ noOpDriverQueryHook.setDriver(this);
}
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
index da125bd..f8a9ee0 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
@@ -37,25 +37,65 @@ import org.apache.lens.server.api.query.QueryContext;
*
* This interface is expected to evolve for some time as more needs for hook are discovered
*
-
+ * Note: Note if the hook updates any configuration, same should be reflected in QueryContext
+ * via {@link AbstractQueryContext#updateConf(Map)} to ensure the modified configuration is persisted and is available
+ * on server restarts and other bookkeeping needs.
*/
public interface DriverQueryHook {
+
/**
- * Called just before launching the query on the selected driver.
+ * This setter method is called by the driver once hook instance is created. This driver information can be used while
+ * extracting driver specific information form the QueryContext.
+ * @param driver
+ */
+ void setDriver(LensDriver driver);
+
+ /**
+ * Called just before rewrite operation is tried on this driver
+ *
* @param ctx
* @throws LensException
*/
- void preLaunch(QueryContext ctx) throws LensException;
+ void preRewrite(AbstractQueryContext ctx) throws LensException;
/**
- * Called just after driver has been selected to execute a query.
+ * Called just after a successful rewrite operation is tried on this driver
*
- * Note: Note if this method updates any configuration, same should be reflected in QueryContext
- * via {@link AbstractQueryContext#updateConf(Map)} to ensure the modified configration is persisted and is available
- * on server restarts and other bookkeeping needs.
+ * @param ctx
+ * @throws LensException
+ */
+ void postRewrite(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just before estimate operation is tried on this driver
+ * Note : Estimate operation will be skipped if rewrite operation fails for this driver
+ *
+ * @param ctx
+ * @throws LensException
+ */
+ void preEstimate(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just after a successful estimate operation is tried on this driver
+ *
+ * @param ctx
+ * @throws LensException
+ */
+ void postEstimate(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just after driver has been selected to execute a query.
*
* @param ctx
* @throws LensException
*/
void postDriverSelection(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just before launching the query on the selected driver.
+ * @param ctx
+ * @throws LensException
+ */
+ void preLaunch(QueryContext ctx) throws LensException;
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
index 8cd03cb..4f1f2eb 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
@@ -25,19 +25,55 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class NoOpDriverQueryHook implements DriverQueryHook {
+
+ @Getter
+ private LensDriver driver;
+
@Override
- public void preLaunch(QueryContext ctx) {
- log.debug("Pre launch for {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
- ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery());
+ public void setDriver(LensDriver driver) {
+ this.driver = driver;
+ log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName());
+ }
+
+ @Override
+ public void preRewrite(AbstractQueryContext ctx) throws LensException {
+ log.debug("Pre rewrite for user {}, user query: {}, driver: {}", ctx.getSubmittedUser(), ctx.getUserQuery(),
+ driver.getFullyQualifiedName());
+ }
+
+ @Override
+ public void postRewrite(AbstractQueryContext ctx) throws LensException {
+ log.debug("Post rewrite for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(),
+ ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver));
+ }
+
+ @Override
+ public void preEstimate(AbstractQueryContext ctx) throws LensException {
+ log.debug("Pre estimate for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(),
+ ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver));
+ }
+
+ @Override
+ public void postEstimate(AbstractQueryContext ctx) throws LensException {
+ log.debug("Post estimate for user {}, user query: {}, driver: {}, driver query :{}, query cost :{}",
+ ctx.getSubmittedUser(), ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver),
+ ctx.getDriverQueryCost(driver));
}
@Override
public void postDriverSelection(AbstractQueryContext ctx) throws LensException {
- log.debug("Post driver selection for {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
+ log.debug("Post driver selection for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
+ ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery());
+ }
+
+ @Override
+ public void preLaunch(QueryContext ctx) {
+ log.debug("Pre launch for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery());
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
index 792508b..4768550 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
@@ -64,4 +64,9 @@ public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBased
public int compareTo(final FactPartitionBasedQueryCost o) {
return new Double(partitionCost).compareTo(o.partitionCost);
}
+
+ @Override
+ public String toString() {
+ return getQueryCostType() + "(" + getEstimatedResourceUsage() + ")";
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 168b3cc..3c18ac7 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -99,6 +99,7 @@ public class MockDriver extends AbstractLensDriver {
this.conf = conf;
ioTestVal = conf.getInt("mock.driver.test.val", -1);
this.conf.addResource(getDriverResourcePath("failing-query-driver-site.xml"));
+ getQueryHook().setDriver(this);
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
index 0c4a3f0..f70979a 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
@@ -34,6 +34,11 @@ public class MockDriverQueryHook extends NoOpDriverQueryHook {
public static final String UNSAVED_KEY_POST_SELECT = "TEST_UNSAVED__KEY_POST_SELECT";
public static final String UNSAVED_VALUE_POST_SELECT = "TEST_UNSAVED_VALUE_POST_SELECT";
+ public static final String PRE_REWRITE = "PRE_REWRITE";
+ public static final String POST_REWRITE = "POST_REWRITE";
+ public static final String PRE_ESTIMATE = "PRE_ESTIMATE";
+ public static final String POST_ESTIMATE = "POST_ESTIMATE";
+
@Override
public void preLaunch(QueryContext ctx) {
super.preLaunch(ctx);
@@ -46,9 +51,36 @@ public class MockDriverQueryHook extends NoOpDriverQueryHook {
//Updated both in driver config and LensConf(which gets persisted)
ctx.getSelectedDriverConf().set(KEY_POST_SELECT, VALUE_POST_SELECT);
- ctx.updateConf(new HashMap<String, String>(1){{put(KEY_POST_SELECT, VALUE_POST_SELECT); }});
+ ctx.updateConf(new HashMap<String, String>(1) {{
+ put(KEY_POST_SELECT, VALUE_POST_SELECT);
+ }
+ });
//Updated only in driver conf.
ctx.getSelectedDriverConf().set(UNSAVED_KEY_POST_SELECT, UNSAVED_VALUE_POST_SELECT);
}
+
+ @Override
+ public void preRewrite(AbstractQueryContext ctx) throws LensException {
+ super.preRewrite(ctx);
+ ctx.getDriverConf(getDriver()).set(PRE_REWRITE, PRE_REWRITE);
+ }
+
+ @Override
+ public void postRewrite(AbstractQueryContext ctx) throws LensException {
+ super.postRewrite(ctx);
+ ctx.getDriverConf(getDriver()).set(POST_REWRITE, POST_REWRITE);
+ }
+
+ @Override
+ public void preEstimate(AbstractQueryContext ctx) throws LensException {
+ super.preEstimate(ctx);
+ ctx.getDriverConf(getDriver()).set(PRE_ESTIMATE, PRE_ESTIMATE);
+ }
+
+ @Override
+ public void postEstimate(AbstractQueryContext ctx) throws LensException {
+ super.postEstimate(ctx);
+ ctx.getDriverConf(getDriver()).set(POST_ESTIMATE, POST_ESTIMATE);
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 0bb106e..87d7cb0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1689,12 +1689,17 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
acquire(ctx.getLensSessionIdentifier());
MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
REWRITE_GAUGE);
+ log.info("Calling preRewrite hook for driver {}", driver.getFullyQualifiedName());
+ driver.getQueryHook().preRewrite(ctx);
// 1. Rewrite for driver
rewriterRunnable.run();
succeeded = rewriterRunnable.isSucceeded();
if (!succeeded) {
failureCause = rewriterRunnable.getFailureCause();
cause = rewriterRunnable.getCause();
+ } else {
+ log.info("Calling postRewrite hook for driver {}", driver.getFullyQualifiedName());
+ driver.getQueryHook().postRewrite(ctx);
}
rewriteGauge.markSuccess();
@@ -1704,14 +1709,19 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
MethodMetricsContext estimateGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
DRIVER_ESTIMATE_GAUGE);
+ log.info("Calling preEstimate hook for driver {}", driver.getFullyQualifiedName());
+ driver.getQueryHook().preEstimate(ctx);
estimateRunnable.run();
succeeded = estimateRunnable.isSucceeded();
-
if (!succeeded) {
failureCause = estimateRunnable.getFailureCause();
cause = estimateRunnable.getCause();
log.error("Estimate failed for driver {} cause: {}", driver, failureCause);
+ } else {
+ log.info("Calling postRewrite hook for driver {}", driver.getFullyQualifiedName());
+ driver.getQueryHook().postEstimate(ctx);
}
+
estimateGauge.markSuccess();
} else {
log.error("Estimate skipped since rewrite failed for driver {} cause: {}", driver, failureCause);
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index f755508..3f71aef 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -766,6 +766,10 @@ public class TestQueryService extends LensJerseyTest {
Thread.sleep(1000);
}
assertEquals(ctx.getSelectedDriverConf().get(KEY_PRE_LAUNCH), VALUE_PRE_LAUNCH);
+ assertEquals(ctx.getSelectedDriverConf().get(PRE_REWRITE), PRE_REWRITE);
+ assertEquals(ctx.getSelectedDriverConf().get(POST_REWRITE), POST_REWRITE);
+ assertEquals(ctx.getSelectedDriverConf().get(PRE_ESTIMATE), PRE_ESTIMATE);
+ assertEquals(ctx.getSelectedDriverConf().get(POST_ESTIMATE), POST_ESTIMATE);
assertTrue(lensQuery.getSubmissionTime() > 0);
assertTrue(lensQuery.getLaunchTime() > 0);
assertTrue(lensQuery.getDriverStartTime() > 0);